CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 09:12:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

Logstash Import and Template Assignment

Browse Source
This commit is contained in:
Mike Reeves
2020-07-10 13:53:55 -04:00
parent bbef7955b2
commit 5eb33d5ac7
25 changed files with 70 additions and 350 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
pillar/logstash/eval.sls
View File

@@ -18,11 +18,20 @@ logstash:
templates:
- so/so-beats-template.json.jinja
- so/so-common-template.json
- so/so-dhcp-template.json.jinja
- so/so-dns_windows-template.json.jinja
- so/so-esxi-template.json.jinja
- so/so-firewall-template.json.jinja
- so/so-flow-template.json.jinja
- so/so-greensql-template.json.jinja
- so/so-ids-template.json.jinja
- so/so-import-template.json.jinja
- so/so-iss-template.json.jinja
- so/so-mcafee-template.json.jinja
- so/so-osquery-template.json.jinja
- so/so-ossec-template.json.jinja
- so/so-strelka-template.json.jinja
- so/so-switch-template.json.jinja
- so/so-syslog-template.json.jinja
- so/so-windows-template.json.jinja
- so/so-zeek-template.json.jinja

9
pillar/logstash/search.sls
View File

@@ -14,11 +14,20 @@ logstash:
templates:
- so/so-beats-template.json.jinja
- so/so-common-template.json
- so/so-dhcp-template.json.jinja
- so/so-dns_windows-template.json.jinja
- so/so-esxi-template.json.jinja
- so/so-firewall-template.json.jinja
- so/so-flow-template.json.jinja
- so/so-greensql-template.json.jinja
- so/so-ids-template.json.jinja
- so/so-import-template.json.jinja
- so/so-iss-template.json.jinja
- so/so-mcafee-template.json.jinja
- so/so-osquery-template.json.jinja
- so/so-ossec-template.json.jinja
- so/so-strelka-template.json.jinja
- so/so-switch-template.json.jinja
- so/so-syslog-template.json.jinja
- so/so-windows-template.json.jinja
- so/so-zeek-template.json.jinja

4
salt/filebeat/etc/filebeat.yml
View File

@@ -126,6 +126,8 @@ filebeat.inputs:
category: network
imported: true
processors:
- add_tags:
tags: [import]
- dissect:
tokenizer: "/nsm/import/%{import.id}/zeek/logs/%{import.file}"
field: "log.file.path"
@@ -164,6 +166,8 @@ filebeat.inputs:
category: network
imported: true
processors:
- add_tags:
tags: [import]
- dissect:
tokenizer: "/nsm/import/%{import.id}/suricata/%{import.file}"
field: "log.file.path"

15
salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja
View File

@@ -3,21 +3,8 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [module] =~ "zeek" {
mutate {
##add_tag => [ "conf_file_9000"]
}
}
}
output {
if [module] =~ "zeek" {
if [module] =~ "zeek" and "import" not in [tags] {
elasticsearch {
pipeline => "%{module}.%{dataset}"
hosts => "{{ ES }}"

27
salt/logstash/pipelines/config/so/9001_output_switch.conf.jinja
View File

@@ -1,27 +0,0 @@
{%- if grains['role'] == 'so-eval' -%}
{%- set ES = salt['pillar.get']('master:mainip', '') -%}
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if "switch" in [tags] and "test_data" not in [tags] {
mutate {
##add_tag => [ "conf_file_9001"]
}
}
}
output {
if "switch" in [tags] and "test_data" not in [tags] {
#stdout { codec => rubydebug }
elasticsearch {
hosts => "{{ ES }}"
index => "so-switch-%{+YYYY.MM.dd}"
template => "/so-common-template.json"
}
}
}

18
salt/logstash/pipelines/config/so/9002_output_import.conf.jinja
View File

@@ -3,24 +3,14 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
# Updated by: Doug Burks
# Last Update: 5/16/2017
filter {
if "import" in [tags] and "test_data" not in [tags] {
mutate {
##add_tag => [ "conf_file_9002"]
}
}
}
output {
if "import" in [tags] and "test_data" not in [tags] {
# stdout { codec => rubydebug }
if "import" in [tags] {
elasticsearch {
pipeline => "%{module}.%{dataset}"
hosts => "{{ ES }}"
index => "so-import-%{+YYYY.MM.dd}"
template_name => "so-common"
template => "/so-common-template.json"
template_name => "so-import"
template => "/so-import-template.json"
template_overwrite => true
}
}

19
salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja
View File

@@ -3,25 +3,14 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [event_type] == "sflow" and "test_data" not in [tags] {
mutate {
##add_tag => [ "conf_file_9004"]
}
}
}
output {
if [event_type] == "sflow" and "test_data" not in [tags] {
#stdout { codec => rubydebug }
if [event_type] == "sflow" {
elasticsearch {
hosts => "{{ ES }}"
index => "so-flow-%{+YYYY.MM.dd}"
template => "/so-common-template.json"
template_name => "so-flow"
template => "/so-flow-template.json"
template_overwrite => true
}
}
}

26
salt/logstash/pipelines/config/so/9026_output_dhcp.conf.jinja
View File

@@ -1,26 +0,0 @@
{%- if grains['role'] == 'so-eval' -%}
{%- set ES = salt['pillar.get']('master:mainip', '') -%}
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [event_type] == "dhcp" and "test_data" not in [tags] {
mutate {
##add_tag => [ "conf_file_9026"]
}
}
}
output {
if [event_type] == "dhcp" and "test_data" not in [tags] {
#stdout { codec => rubydebug }
elasticsearch {
hosts => "{{ ES }}"
template => "/so-common-template.json"
}
}
}

25
salt/logstash/pipelines/config/so/9029_output_esxi.conf.jinja
View File

@@ -1,25 +0,0 @@
{%- if grains['role'] == 'so-eval' -%}
{%- set ES = salt['pillar.get']('master:mainip', '') -%}
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [event_type] == "esxi" and "test_data" not in [tags] {
mutate {
##add_tag => [ "conf_file_9029"]
}
}
}
output {
if [event_type] == "esxi" and "test_data" not in [tags] {
elasticsearch {
hosts => "{{ ES }}"
template => "/so-common-template.json"
}
}
}

25
salt/logstash/pipelines/config/so/9030_output_greensql.conf.jinja
View File

@@ -1,25 +0,0 @@
{%- if grains['role'] == 'so-eval' -%}
{%- set ES = salt['pillar.get']('master:mainip', '') -%}
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [event_type] == "greensql" and "test_data" not in [tags] {
mutate {
##add_tag => [ "conf_file_9030"]
}
}
}
output {
if [event_type] == "greensql" and "test_data" not in [tags] {
elasticsearch {
hosts => "{{ ES }}"
template => "/so-common-template.json"
}
}
}

26
salt/logstash/pipelines/config/so/9031_output_iis.conf.jinja
View File

@@ -1,26 +0,0 @@
{%- if grains['role'] == 'so-eval' -%}
{%- set ES = salt['pillar.get']('master:mainip', '') -%}
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [event_type] == "iis" and "test_data" not in [tags] {
mutate {
##add_tag => [ "conf_file_9031"]
}
}
}
output {
if [event_type] == "iis" and "test_data" not in [tags] {
#stdout { codec => rubydebug }
elasticsearch {
hosts => "{{ ES }}"
template => "/so-common-template.json"
}
}
}

26
salt/logstash/pipelines/config/so/9032_output_mcafee.conf.jinja
View File

@@ -1,26 +0,0 @@
{%- if grains['role'] == 'so-eval' -%}
{%- set ES = salt['pillar.get']('master:mainip', '') -%}
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [event_type] == "mcafee" and "test_data" not in [tags] {
mutate {
##add_tag => [ "conf_file_9032"]
}
}
}
output {
if [event_type] == "mcafee" and "test_data" not in [tags] {
#stdout { codec => rubydebug }
elasticsearch {
hosts => "{{ ES }}"
template => "/so-common-template.json"
}
}
}

19
salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja
View File

@@ -3,26 +3,13 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [event_type] == "ids" and "test_data" not in [tags] {
mutate {
##add_tag => [ "conf_file_9033"]
}
}
}
output {
if [event_type] == "ids" and "test_data" not in [tags] {
#stdout { codec => rubydebug }
if [event_type] == "ids" and "import" not in [tags] {
elasticsearch {
hosts => "{{ ES }}"
index => "so-ids-%{+YYYY.MM.dd}"
template_name => "so-common"
template => "/so-common-template.json"
template_name => "so-ids"
template => "/so-ids-template.json"
template_overwrite => true
}
}

12
salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja
View File

@@ -3,22 +3,14 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
filter {
if [module] =~ "syslog" {
mutate {
##add_tag => [ "conf_file_9000"]
}
}
}
output {
if [module] =~ "syslog" {
elasticsearch {
pipeline => "%{module}"
hosts => "{{ ES }}"
index => "so-syslog-%{+YYYY.MM.dd}"
template_name => "so-common"
template => "/so-common-template.json"
template_name => "so-syslog"
template => "/so-syslog-template.json"
template_overwrite => true
}
}

9
salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja
View File

@@ -3,18 +3,15 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
# Author: Josh Brower
# Last Update: 12/29/2018
# Output to ES for osquery tagged logs
output {
if [module] =~ "osquery" {
elasticsearch {
pipeline => "%{module}.%{dataset}"
hosts => "{{ ES }}"
index => "so-osquery-%{+YYYY.MM.dd}"
template => "/so-common-template.json"
template_name => "so-osquery"
template => "/so-osquery-template.json"
template_overwrite => true
}
}
}

19
salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja
View File

@@ -3,26 +3,13 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if "firewall" in [tags] and "test_data" not in [tags] {
mutate {
##add_tag => [ "conf_file_9200"]
}
}
}
output {
if "firewall" in [tags] and "test_data" not in [tags] {
# stdout { codec => rubydebug }
if "firewall" in [tags] {
elasticsearch {
hosts => "{{ ES }}"
index => "so-firewall-%{+YYYY.MM.dd}"
template_name => "so-common"
template => "/so-common-template.json"
template_name => "so-firewall"
template => "/so-firewall-template.json"
template_overwrite => true
}
}

27
salt/logstash/pipelines/config/so/9300_output_windows.conf.jinja
View File

@@ -1,27 +0,0 @@
{%- if grains['role'] == 'so-eval' -%}
{%- set ES = salt['pillar.get']('master:mainip', '') -%}
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [event_type] == "windows" and "test_data" not in [tags] {
mutate {
##add_tag => [ "conf_file_9300"]
}
}
}
output {
if [event_type] == "windows" and "test_data" not in [tags] {
#stdout { codec => rubydebug }
elasticsearch {
hosts => "{{ ES }}"
index => "so-windows-%{+YYYY.MM.dd}"
template => "/so-common-template.json"
}
}
}

27
salt/logstash/pipelines/config/so/9301_output_dns_windows.conf.jinja
View File

@@ -1,27 +0,0 @@
{%- if grains['role'] == 'so-eval' -%}
{%- set ES = salt['pillar.get']('master:mainip', '') -%}
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [event_type] == "dns" and "test_data" not in [tags] {
mutate {
##add_tag => [ "conf_file_9301"]
}
}
}
output {
if [event_type] == "dns" and "test_data" not in [tags] {
#stdout { codec => rubydebug }
elasticsearch {
hosts => "{{ ES }}"
index => "so-%{+YYYY.MM.dd}"
template => "/so-common-template.json"
}
}
}

18
salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja
View File

@@ -3,26 +3,14 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [module] == "suricata" {
mutate {
##add_tag => [ "conf_file_9400"]
}
}
}
output {
if [module] =~ "suricata" {
if [module] =~ "suricata" and "import" not in [tags] {
elasticsearch {
pipeline => "%{module}.%{dataset}"
hosts => "{{ ES }}"
index => "so-ids-%{+YYYY.MM.dd}"
template_name => "so-common"
template => "/so-common-template.json"
template_name => "so-ids"
template => "/so-ids-template.json"
}
}
}

7
salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja
View File

@@ -3,15 +3,14 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
output {
if "beat-ext" in [tags] {
if "beat-ext" in [tags] and "import" not in [tags] {
elasticsearch {
pipeline => "beats.common"
hosts => "{{ ES }}"
index => "so-beats-%{+YYYY.MM.dd}"
template_name => "so-common"
template => "/so-common-template.json"
template_name => "so-beats"
template => "/so-beats-template.json"
template_overwrite => true
}
}

17
salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja
View File

@@ -3,27 +3,14 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Updated by: Doug Burks
# Last Update: 9/19/2018
filter {
if [module] =~ "ossec" {
mutate {
##add_tag => [ "conf_file_9600"]
}
}
}
output {
if [module] =~ "ossec" {
elasticsearch {
pipeline => "%{module}.%{dataset}"
hosts => "{{ ES }}"
index => "so-ossec-%{+YYYY.MM.dd}"
template_name => "so-common"
template => "/so-common-template.json"
template_name => "so-ossec"
template => "/so-ossec-template.json"
template_overwrite => true
}
}

17
salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja
View File

@@ -3,27 +3,14 @@
{%- else %}
{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
{%- endif %}
# Author: Justin Henderson
# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
# Email: justin@hasecuritysolution.com
# Last Update: 12/9/2016
filter {
if [module] =~ "strelka" {
mutate {
##add_tag => [ "conf_file_9000"]
}
}
}
output {
if [event_type] =~ "strelka" {
elasticsearch {
pipeline => "%{module}.%{dataset}"
hosts => "{{ ES }}"
index => "so-strelka-%{+YYYY.MM.dd}"
template_name => "so-common"
template => "/so-common-template.json"
template_name => "so-strelka"
template => "/so-strelka-template.json"
template_overwrite => true
}
}

1
salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja
View File

@@ -1,6 +1,5 @@
{% set MASTER = salt['pillar.get']('static:masterip', '') %}
{% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
output {
redis {
host => '{{ MASTER }}'

13
salt/logstash/pipelines/templates/so/so-flow-template.json.jinja Normal file
View File

@@ -0,0 +1,13 @@
{%- set SHARDS = salt['pillar.get']('elasticsearch:index_settings:so-flow:shards', 1) %}
{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %}
{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-flow:refresh', '30s') %}
{
"index_patterns": ["so-flow-*"],
"version": 50001,
"order": 11,
"settings":{
"number_of_replicas":{{ REPLICAS }},
"number_of_shards":{{ SHARDS }},
"index.refresh_interval":"{{ REFRESH }}"
}
}

5
setup/so-functions
View File

@@ -1026,6 +1026,11 @@ master_static() {
" warm: 7"\
" close: 30"\
" delete: 365"\
" so-flow:"\
" shards: 1"\
" warm: 7"\
" close: 30"\
" delete: 365"\
" so-ids:"\
" shards: 1"\
" warm: 7"\