From a413b7594da9790118015156d0dea464da62a133 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 14 Sep 2018 14:32:27 -0400 Subject: [PATCH] Logstash Module - Add Jinja for output plugins --- salt/logstash/files/dynamic/0900_input_redis.conf | 5 +++-- salt/logstash/files/dynamic/9000_output_bro.conf | 3 ++- salt/logstash/files/dynamic/9001_output_switch.conf | 3 ++- salt/logstash/files/dynamic/9002_output_import.conf | 3 ++- salt/logstash/files/dynamic/9004_output_flow.conf | 3 ++- salt/logstash/files/dynamic/9026_output_dhcp.conf | 3 ++- salt/logstash/files/dynamic/9029_output_esxi.conf | 3 ++- salt/logstash/files/dynamic/9030_output_greensql.conf | 3 ++- salt/logstash/files/dynamic/9031_output_iis.conf | 3 ++- salt/logstash/files/dynamic/9032_output_mcafee.conf | 3 ++- salt/logstash/files/dynamic/9033_output_snort.conf | 3 ++- salt/logstash/files/dynamic/9034_output_syslog.conf | 3 ++- salt/logstash/files/dynamic/9200_output_firewall.conf | 3 ++- salt/logstash/files/dynamic/9300_output_windows.conf | 4 ++-- salt/logstash/files/dynamic/9301_output_dns_windows.conf | 4 ++-- salt/logstash/files/dynamic/9400_output_suricata.conf | 3 ++- salt/logstash/files/dynamic/9500_output_beats.conf | 3 ++- salt/logstash/files/dynamic/9998_output_test_data.conf | 3 ++- so-setup-network.sh | 2 ++ 19 files changed, 39 insertions(+), 21 deletions(-) diff --git a/salt/logstash/files/dynamic/0900_input_redis.conf b/salt/logstash/files/dynamic/0900_input_redis.conf index 66492dc5f..8041d6107 100644 --- a/salt/logstash/files/dynamic/0900_input_redis.conf +++ b/salt/logstash/files/dynamic/0900_input_redis.conf @@ -1,9 +1,10 @@ +{%- set master = grains['master'] %} input { redis { - host => '172.18.0.1' + host => '{{ master }}' data_type => 'list' key => 'logstash:redis' type => 'redis-input' # threads => 1 } -} \ No newline at end of file +} diff --git a/salt/logstash/files/dynamic/9000_output_bro.conf b/salt/logstash/files/dynamic/9000_output_bro.conf index 43aacda05..b96bea628 100644 --- a/salt/logstash/files/dynamic/9000_output_bro.conf +++ b/salt/logstash/files/dynamic/9000_output_bro.conf @@ -1,3 +1,4 @@ +{%- set ES = salt['pillar.get']('mainip'. '') %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -15,7 +16,7 @@ output { if "bro" in [tags] and "test_data" not in [tags] and "import" not in [tags] { # stdout { codec => rubydebug } elasticsearch { - hosts => elasticsearch + hosts => {{ ES }} index => "logstash-bro-%{+YYYY.MM.dd}" template_name => "logstash" template => "/logstash-template.json" diff --git a/salt/logstash/files/dynamic/9001_output_switch.conf b/salt/logstash/files/dynamic/9001_output_switch.conf index 2a1b37eba..2261b6651 100644 --- a/salt/logstash/files/dynamic/9001_output_switch.conf +++ b/salt/logstash/files/dynamic/9001_output_switch.conf @@ -1,3 +1,4 @@ +{%- set ES = salt['pillar.get']('mainip'. '') %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -14,7 +15,7 @@ output { if "switch" in [tags] and "test_data" not in [tags] { #stdout { codec => rubydebug } elasticsearch { - hosts => elasticsearch + hosts => {{ ES }} index => "logstash-switch-%{+YYYY.MM.dd}" template => "/logstash-template.json" } diff --git a/salt/logstash/files/dynamic/9002_output_import.conf b/salt/logstash/files/dynamic/9002_output_import.conf index 38f5b0f90..c86c1a47c 100644 --- a/salt/logstash/files/dynamic/9002_output_import.conf +++ b/salt/logstash/files/dynamic/9002_output_import.conf @@ -1,3 +1,4 @@ +{%- set ES = salt['pillar.get']('mainip'. '') %} # Updated by: Doug Burks # Last Update: 5/16/2017 @@ -12,7 +13,7 @@ output { if "import" in [tags] and "test_data" not in [tags] { # stdout { codec => rubydebug } elasticsearch { - hosts => elasticsearch + hosts => {{ ES }} index => "logstash-import-%{+YYYY.MM.dd}" template_name => "logstash-*" template => "/logstash-template.json" diff --git a/salt/logstash/files/dynamic/9004_output_flow.conf b/salt/logstash/files/dynamic/9004_output_flow.conf index be125243e..be117ff40 100644 --- a/salt/logstash/files/dynamic/9004_output_flow.conf +++ b/salt/logstash/files/dynamic/9004_output_flow.conf @@ -1,3 +1,4 @@ +{%- set ES = salt['pillar.get']('mainip'. '') %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -14,7 +15,7 @@ output { if [event_type] == "sflow" and "test_data" not in [tags] { #stdout { codec => rubydebug } elasticsearch { - hosts => elasticsearch + hosts => {{ ES }} index => "logstash-flow-%{+YYYY.MM.dd}" template => "/logstash-template.json" } diff --git a/salt/logstash/files/dynamic/9026_output_dhcp.conf b/salt/logstash/files/dynamic/9026_output_dhcp.conf index 7365cea35..c0ee0dfb5 100644 --- a/salt/logstash/files/dynamic/9026_output_dhcp.conf +++ b/salt/logstash/files/dynamic/9026_output_dhcp.conf @@ -1,3 +1,4 @@ +{%- set ES = salt['pillar.get']('mainip'. '') %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -14,7 +15,7 @@ output { if [event_type] == "dhcp" and "test_data" not in [tags] { #stdout { codec => rubydebug } elasticsearch { - hosts => elasticsearch + hosts => {{ ES }} template => "/logstash-template.json" } } diff --git a/salt/logstash/files/dynamic/9029_output_esxi.conf b/salt/logstash/files/dynamic/9029_output_esxi.conf index 09547e248..f0c72429b 100644 --- a/salt/logstash/files/dynamic/9029_output_esxi.conf +++ b/salt/logstash/files/dynamic/9029_output_esxi.conf @@ -1,3 +1,4 @@ +{%- set ES = salt['pillar.get']('mainip'. '') %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -13,7 +14,7 @@ filter { output { if [event_type] == "esxi" and "test_data" not in [tags] { elasticsearch { - hosts => elasticsearch + hosts => {{ ES }} template => "/logstash-template.json" } } diff --git a/salt/logstash/files/dynamic/9030_output_greensql.conf b/salt/logstash/files/dynamic/9030_output_greensql.conf index c08315397..646f7fd0b 100644 --- a/salt/logstash/files/dynamic/9030_output_greensql.conf +++ b/salt/logstash/files/dynamic/9030_output_greensql.conf @@ -1,3 +1,4 @@ +{%- set ES = salt['pillar.get']('mainip'. '') %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -13,7 +14,7 @@ filter { output { if [event_type] == "greensql" and "test_data" not in [tags] { elasticsearch { - hosts => elasticsearch + hosts => {{ ES }} template => "/logstash-template.json" } } diff --git a/salt/logstash/files/dynamic/9031_output_iis.conf b/salt/logstash/files/dynamic/9031_output_iis.conf index c130cc2fb..ffb774183 100644 --- a/salt/logstash/files/dynamic/9031_output_iis.conf +++ b/salt/logstash/files/dynamic/9031_output_iis.conf @@ -1,3 +1,4 @@ +{%- set ES = salt['pillar.get']('mainip'. '') %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -14,7 +15,7 @@ output { if [event_type] == "iis" and "test_data" not in [tags] { #stdout { codec => rubydebug } elasticsearch { - hosts => elasticsearch + hosts => {{ ES }} template => "/logstash-template.json" } } diff --git a/salt/logstash/files/dynamic/9032_output_mcafee.conf b/salt/logstash/files/dynamic/9032_output_mcafee.conf index 537381529..018bf0664 100644 --- a/salt/logstash/files/dynamic/9032_output_mcafee.conf +++ b/salt/logstash/files/dynamic/9032_output_mcafee.conf @@ -1,3 +1,4 @@ +{%- set ES = salt['pillar.get']('mainip'. '') %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -14,7 +15,7 @@ output { if [event_type] == "mcafee" and "test_data" not in [tags] { #stdout { codec => rubydebug } elasticsearch { - hosts => elasticsearch + hosts => {{ ES }} template => "/logstash-template.json" } } diff --git a/salt/logstash/files/dynamic/9033_output_snort.conf b/salt/logstash/files/dynamic/9033_output_snort.conf index 0a3bdcb68..850481c74 100644 --- a/salt/logstash/files/dynamic/9033_output_snort.conf +++ b/salt/logstash/files/dynamic/9033_output_snort.conf @@ -1,3 +1,4 @@ +{%- set ES = salt['pillar.get']('mainip'. '') %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -14,7 +15,7 @@ output { if [event_type] == "snort" and "test_data" not in [tags] { #stdout { codec => rubydebug } elasticsearch { - hosts => elasticsearch + hosts => {{ ES }} index => "logstash-ids-%{+YYYY.MM.dd}" template_name => "logstash" template => "/logstash-template.json" diff --git a/salt/logstash/files/dynamic/9034_output_syslog.conf b/salt/logstash/files/dynamic/9034_output_syslog.conf index 01b462b4e..d58635aa7 100644 --- a/salt/logstash/files/dynamic/9034_output_syslog.conf +++ b/salt/logstash/files/dynamic/9034_output_syslog.conf @@ -1,3 +1,4 @@ +{%- set ES = salt['pillar.get']('mainip'. '') %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Updated by: Doug Burks @@ -13,7 +14,7 @@ filter { output { if "syslog" in [tags] and "test_data" not in [tags] { elasticsearch { - hosts => elasticsearch + hosts => {{ ES }} index => "logstash-syslog-%{+YYYY.MM.dd}" template_name => "logstash" template => "/logstash-template.json" diff --git a/salt/logstash/files/dynamic/9200_output_firewall.conf b/salt/logstash/files/dynamic/9200_output_firewall.conf index ab1019990..84406b0a9 100644 --- a/salt/logstash/files/dynamic/9200_output_firewall.conf +++ b/salt/logstash/files/dynamic/9200_output_firewall.conf @@ -1,3 +1,4 @@ +{%- set ES = salt['pillar.get']('mainip'. '') %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -14,7 +15,7 @@ output { if "firewall" in [tags] and "test_data" not in [tags] { # stdout { codec => rubydebug } elasticsearch { - hosts => elasticsearch + hosts => {{ ES }} index => "logstash-firewall-%{+YYYY.MM.dd}" template_name => "logstash" template => "/logstash-template.json" diff --git a/salt/logstash/files/dynamic/9300_output_windows.conf b/salt/logstash/files/dynamic/9300_output_windows.conf index 9c6470cb7..78f2ac237 100644 --- a/salt/logstash/files/dynamic/9300_output_windows.conf +++ b/salt/logstash/files/dynamic/9300_output_windows.conf @@ -1,3 +1,4 @@ +{%- set ES = salt['pillar.get']('mainip'. '') %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -14,10 +15,9 @@ output { if [event_type] == "windows" and "test_data" not in [tags] { #stdout { codec => rubydebug } elasticsearch { - hosts => elasticsearch + hosts => {{ ES } index => "logstash-windows-%{+YYYY.MM.dd}" template => "/logstash-template.json" } } } - diff --git a/salt/logstash/files/dynamic/9301_output_dns_windows.conf b/salt/logstash/files/dynamic/9301_output_dns_windows.conf index 97d3cb872..835b0c1e2 100644 --- a/salt/logstash/files/dynamic/9301_output_dns_windows.conf +++ b/salt/logstash/files/dynamic/9301_output_dns_windows.conf @@ -1,3 +1,4 @@ +{%- set ES = salt['pillar.get']('mainip'. '') %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -14,10 +15,9 @@ output { if [event_type] == "dns" and "test_data" not in [tags] { #stdout { codec => rubydebug } elasticsearch { - hosts => elasticsearch + hosts => {{ ES }} index => "logstash-%{+YYYY.MM.dd}" template => "/logstash-template.json" } } } - diff --git a/salt/logstash/files/dynamic/9400_output_suricata.conf b/salt/logstash/files/dynamic/9400_output_suricata.conf index 76b4526ec..13efcf9cf 100644 --- a/salt/logstash/files/dynamic/9400_output_suricata.conf +++ b/salt/logstash/files/dynamic/9400_output_suricata.conf @@ -1,3 +1,4 @@ +{%- set ES = salt['pillar.get']('mainip'. '') %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -14,7 +15,7 @@ output { if [event_type] == "suricata" and "test_data" not in [tags] { #stdout { codec => rubydebug } elasticsearch { - hosts => elasticsearch + hosts => {{ ES }} index => "logstash-ids-%{+YYYY.MM.dd}" template => "/logstash-template.json" } diff --git a/salt/logstash/files/dynamic/9500_output_beats.conf b/salt/logstash/files/dynamic/9500_output_beats.conf index be231c8e3..003ea9460 100644 --- a/salt/logstash/files/dynamic/9500_output_beats.conf +++ b/salt/logstash/files/dynamic/9500_output_beats.conf @@ -1,3 +1,4 @@ +{%- set ES = salt['pillar.get']('mainip'. '') %} # Author: Wes Lambert # Last Update: 12/11/2017 filter { @@ -10,7 +11,7 @@ filter { output { if "beat" in [tags] { elasticsearch { - hosts => elasticsearch + hosts => {{ ES }} index => "logstash-beats-%{+YYYY.MM.dd}" template_name => "logstash-beats" template => "/beats-template.json" diff --git a/salt/logstash/files/dynamic/9998_output_test_data.conf b/salt/logstash/files/dynamic/9998_output_test_data.conf index fd11e32d7..031302bf3 100644 --- a/salt/logstash/files/dynamic/9998_output_test_data.conf +++ b/salt/logstash/files/dynamic/9998_output_test_data.conf @@ -1,3 +1,4 @@ +{%- set ES = salt['pillar.get']('mainip'. '') %} # Author: Justin Henderson # SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics # Email: justin@hasecuritysolution.com @@ -13,7 +14,7 @@ filter { output { if "test_data" in [tags] { elasticsearch { - hosts => elasticsearch + hosts => {{ ES }} index => "logstash-test-%{+YYYY.MM.dd}" template => "/logstash-template.json" } diff --git a/so-setup-network.sh b/so-setup-network.sh index 2e57b74a2..7ccb01fd7 100644 --- a/so-setup-network.sh +++ b/so-setup-network.sh @@ -389,6 +389,7 @@ node_pillar() { # Create the node pillar touch $TMP/$HOSTNAME.sls echo "node:" > $TMP/$HOSTNAME.sls + echo " mainip: $MAINIP" >> $TMP/$HOSTNAME.sls echo " esaccessip: 127.0.0.1" >> $TMP/$HOSTNAME.sls echo " esheap: $NODE_ES_HEAP_SIZE" >> $TMP/$HOSTNAME.sls echo " esclustername: {{ grains.host }}" >> $TMP/$HOSTNAME.sls @@ -543,6 +544,7 @@ sensor_pillar() { touch $TMP/$HOSTNAME.sls echo "sensor:" > $TMP/$HOSTNAME.sls echo " interface: bond0" >> $TMP/$HOSTNAME.sls + echo " mainip: $MAINIP" >> $TMP/$HOSTNAME.sls if [ $NSMSETUP == 'ADVANCED' ]; then echo " bro_pins:" >> $TMP/$HOSTNAME.sls for PIN in $BROPINS; do