From f9e07709f2cef88c13f3773cebdd87bf68742d36 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 18 Feb 2020 16:36:20 -0500 Subject: [PATCH 1/9] bpf fix --- salt/pcap/init.sls | 2 +- salt/suricata/init.sls | 2 +- salt/zeek/init.sls | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/pcap/init.sls b/salt/pcap/init.sls index 11732ad29..17162fb16 100644 --- a/salt/pcap/init.sls +++ b/salt/pcap/init.sls @@ -41,7 +41,7 @@ stenoconfdir: - makedirs: True {% if BPF_STENO %} - {% set BPF_CALC = salt['cmd.script']('/usr/sbin/so-bpf-compile', INTERFACE + ' ' + BPF_STENO|join(" ") ) %} + {% set BPF_CALC = salt['cmd.script']('/usr/sbin/so-bpf-compile', INTERFACE + ' ' + BPF_STENO|join(" "),cwd='/root') %} {% if BPF_CALC['stderr'] == "" %} {% set BPF_COMPILED = ",\\\"--filter=" + BPF_CALC['stdout'] + "\\\"" %} {% else %} diff --git a/salt/suricata/init.sls b/salt/suricata/init.sls index dcea927ae..da1220e63 100644 --- a/salt/suricata/init.sls +++ b/salt/suricata/init.sls @@ -84,7 +84,7 @@ surithresholding: # BPF compilation and configuration {% if BPF_NIDS %} - {% set BPF_CALC = salt['cmd.script']('/usr/sbin/so-bpf-compile', interface + ' ' + BPF_NIDS|join(" ") ) %} + {% set BPF_CALC = salt['cmd.script']('/usr/sbin/so-bpf-compile', interface + ' ' + BPF_NIDS|join(" "),cwd='/root') %} {% if BPF_CALC['stderr'] == "" %} {% set BPF_STATUS = 1 %} {% else %} diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index e7124727e..db54bf62e 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -95,7 +95,7 @@ plcronscript: # BPF compilation and configuration {% if BPF_ZEEK %} - {% set BPF_CALC = salt['cmd.script']('/usr/sbin/so-bpf-compile', INTERFACE + ' ' + BPF_ZEEK|join(" ") ) %} + {% set BPF_CALC = salt['cmd.script']('/usr/sbin/so-bpf-compile', INTERFACE + ' ' + BPF_ZEEK|join(" "),cwd='/root') %} {% if BPF_CALC['stderr'] == "" %} {% set BPF_STATUS = 1 %} {% else %} From 30a1197b448c2646e818e3fbd0f8c965952d2c6b Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 20 Feb 2020 11:20:06 -0500 Subject: [PATCH 2/9] Rename template to avoid duplication under different name --- salt/logstash/files/dynamic/9002_output_import.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/logstash/files/dynamic/9002_output_import.conf b/salt/logstash/files/dynamic/9002_output_import.conf index 88fbc7551..1b691df6b 100644 --- a/salt/logstash/files/dynamic/9002_output_import.conf +++ b/salt/logstash/files/dynamic/9002_output_import.conf @@ -19,7 +19,7 @@ output { elasticsearch { hosts => "{{ ES }}" index => "logstash-import-%{+YYYY.MM.dd}" - template_name => "logstash-*" + template_name => "logstash" template => "/logstash-template.json" template_overwrite => true } From 6945cbb843560992c72a5a33d21b60645403c96c Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 20 Feb 2020 11:45:50 -0500 Subject: [PATCH 3/9] Change template name --- .../conf/pipelines/eval/templates/9002_output_import.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/logstash/conf/pipelines/eval/templates/9002_output_import.conf b/salt/logstash/conf/pipelines/eval/templates/9002_output_import.conf index 88fbc7551..1b691df6b 100644 --- a/salt/logstash/conf/pipelines/eval/templates/9002_output_import.conf +++ b/salt/logstash/conf/pipelines/eval/templates/9002_output_import.conf @@ -19,7 +19,7 @@ output { elasticsearch { hosts => "{{ ES }}" index => "logstash-import-%{+YYYY.MM.dd}" - template_name => "logstash-*" + template_name => "logstash" template => "/logstash-template.json" template_overwrite => true } From fc9786e54194f5860e0e2a9e64e8119682445977 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 20 Feb 2020 11:46:15 -0500 Subject: [PATCH 4/9] Change template name --- .../conf/pipelines/search/templates/9002_output_import.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/logstash/conf/pipelines/search/templates/9002_output_import.conf b/salt/logstash/conf/pipelines/search/templates/9002_output_import.conf index 88fbc7551..1b691df6b 100644 --- a/salt/logstash/conf/pipelines/search/templates/9002_output_import.conf +++ b/salt/logstash/conf/pipelines/search/templates/9002_output_import.conf @@ -19,7 +19,7 @@ output { elasticsearch { hosts => "{{ ES }}" index => "logstash-import-%{+YYYY.MM.dd}" - template_name => "logstash-*" + template_name => "logstash" template => "/logstash-template.json" template_overwrite => true } From e2ccebd2fab8563479195d0c752713c84c49218f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 20 Feb 2020 17:00:15 -0500 Subject: [PATCH 5/9] resolve conflicts --- salt/logstash/files/dynamic/9002_output_import.conf | 2 +- salt/logstash/pipelines/config/9002_output_import.conf.jinja | 2 +- salt/pcap/init.sls | 2 +- salt/suricata/init.sls | 2 +- salt/zeek/init.sls | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/logstash/files/dynamic/9002_output_import.conf b/salt/logstash/files/dynamic/9002_output_import.conf index 88fbc7551..1b691df6b 100644 --- a/salt/logstash/files/dynamic/9002_output_import.conf +++ b/salt/logstash/files/dynamic/9002_output_import.conf @@ -19,7 +19,7 @@ output { elasticsearch { hosts => "{{ ES }}" index => "logstash-import-%{+YYYY.MM.dd}" - template_name => "logstash-*" + template_name => "logstash" template => "/logstash-template.json" template_overwrite => true } diff --git a/salt/logstash/pipelines/config/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/9002_output_import.conf.jinja index 88fbc7551..1b691df6b 100644 --- a/salt/logstash/pipelines/config/9002_output_import.conf.jinja +++ b/salt/logstash/pipelines/config/9002_output_import.conf.jinja @@ -19,7 +19,7 @@ output { elasticsearch { hosts => "{{ ES }}" index => "logstash-import-%{+YYYY.MM.dd}" - template_name => "logstash-*" + template_name => "logstash" template => "/logstash-template.json" template_overwrite => true } diff --git a/salt/pcap/init.sls b/salt/pcap/init.sls index 11732ad29..17162fb16 100644 --- a/salt/pcap/init.sls +++ b/salt/pcap/init.sls @@ -41,7 +41,7 @@ stenoconfdir: - makedirs: True {% if BPF_STENO %} - {% set BPF_CALC = salt['cmd.script']('/usr/sbin/so-bpf-compile', INTERFACE + ' ' + BPF_STENO|join(" ") ) %} + {% set BPF_CALC = salt['cmd.script']('/usr/sbin/so-bpf-compile', INTERFACE + ' ' + BPF_STENO|join(" "),cwd='/root') %} {% if BPF_CALC['stderr'] == "" %} {% set BPF_COMPILED = ",\\\"--filter=" + BPF_CALC['stdout'] + "\\\"" %} {% else %} diff --git a/salt/suricata/init.sls b/salt/suricata/init.sls index dcea927ae..da1220e63 100644 --- a/salt/suricata/init.sls +++ b/salt/suricata/init.sls @@ -84,7 +84,7 @@ surithresholding: # BPF compilation and configuration {% if BPF_NIDS %} - {% set BPF_CALC = salt['cmd.script']('/usr/sbin/so-bpf-compile', interface + ' ' + BPF_NIDS|join(" ") ) %} + {% set BPF_CALC = salt['cmd.script']('/usr/sbin/so-bpf-compile', interface + ' ' + BPF_NIDS|join(" "),cwd='/root') %} {% if BPF_CALC['stderr'] == "" %} {% set BPF_STATUS = 1 %} {% else %} diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls index e7124727e..db54bf62e 100644 --- a/salt/zeek/init.sls +++ b/salt/zeek/init.sls @@ -95,7 +95,7 @@ plcronscript: # BPF compilation and configuration {% if BPF_ZEEK %} - {% set BPF_CALC = salt['cmd.script']('/usr/sbin/so-bpf-compile', INTERFACE + ' ' + BPF_ZEEK|join(" ") ) %} + {% set BPF_CALC = salt['cmd.script']('/usr/sbin/so-bpf-compile', INTERFACE + ' ' + BPF_ZEEK|join(" "),cwd='/root') %} {% if BPF_CALC['stderr'] == "" %} {% set BPF_STATUS = 1 %} {% else %} From 5d81bf32046b298c27a6622b64b3205d888b79fc Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 25 Feb 2020 12:36:35 +0000 Subject: [PATCH 6/9] remove source --- salt/logstash/etc/beats-template.json | 4 ---- salt/logstash/etc/logstash-ossec-template.json | 8 -------- salt/logstash/etc/logstash-template.json | 8 -------- 3 files changed, 20 deletions(-) diff --git a/salt/logstash/etc/beats-template.json b/salt/logstash/etc/beats-template.json index 0e831aa52..433c0862e 100644 --- a/salt/logstash/etc/beats-template.json +++ b/salt/logstash/etc/beats-template.json @@ -989,10 +989,6 @@ } } }, - "source": { - "ignore_above": 1024, - "type": "keyword" - }, "stream": { "ignore_above": 1024, "type": "keyword" diff --git a/salt/logstash/etc/logstash-ossec-template.json b/salt/logstash/etc/logstash-ossec-template.json index ab3a14a93..b44ae69a9 100644 --- a/salt/logstash/etc/logstash-ossec-template.json +++ b/salt/logstash/etc/logstash-ossec-template.json @@ -2825,14 +2825,6 @@ } } }, - "source":{ - "type":"text", - "fields":{ - "keyword":{ - "type":"keyword" - } - } - }, "source_geo.city_name":{ "type":"text", "fields":{ diff --git a/salt/logstash/etc/logstash-template.json b/salt/logstash/etc/logstash-template.json index 44e519842..f23c7b58a 100644 --- a/salt/logstash/etc/logstash-template.json +++ b/salt/logstash/etc/logstash-template.json @@ -2946,14 +2946,6 @@ } } }, - "source":{ - "type":"text", - "fields":{ - "keyword":{ - "type":"keyword" - } - } - }, "source_geo.city_name":{ "type":"text", "fields":{ From 0c4973ad773ee1477033699254d6a783fd3eb5de Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 25 Feb 2020 17:44:32 -0500 Subject: [PATCH 7/9] pillarize logstash config,parsers,templates and docker port bindings --- pillar/logstash/eval.sls | 107 ++++----- pillar/logstash/helix.sls | 76 +++---- pillar/logstash/init.sls | 11 + pillar/logstash/master.sls | 4 +- pillar/logstash/search.sls | 107 ++++----- pillar/top.sls | 5 +- salt/logstash/defaults.yml | 6 - .../Drop.Your.Custom.Parsers.Here.conf | 2 - .../files/dynamic/0006_input_beats.conf | 23 -- .../files/dynamic/0008_input_eval.conf | 203 ------------------ .../files/dynamic/9000_output_bro.conf | 31 --- .../files/dynamic/9001_output_switch.conf | 27 --- .../files/dynamic/9002_output_import.conf | 27 --- .../files/dynamic/9004_output_flow.conf | 27 --- .../files/dynamic/9026_output_dhcp.conf | 26 --- .../files/dynamic/9029_output_esxi.conf | 25 --- .../files/dynamic/9030_output_greensql.conf | 25 --- .../files/dynamic/9031_output_iis.conf | 26 --- .../files/dynamic/9032_output_mcafee.conf | 26 --- .../files/dynamic/9033_output_snort.conf | 29 --- .../files/dynamic/9034_output_syslog.conf | 28 --- .../files/dynamic/9100_output_osquery.conf | 19 -- .../files/dynamic/9200_output_firewall.conf | 29 --- .../files/dynamic/9300_output_windows.conf | 27 --- .../dynamic/9301_output_dns_windows.conf | 27 --- .../files/dynamic/9400_output_suricata.conf | 27 --- .../files/dynamic/9500_output_beats.conf | 25 --- .../files/dynamic/9600_output_ossec.conf | 29 --- .../files/dynamic/9997_output_helix.conf | 142 ------------ .../files/dynamic/9998_output_test_data.conf | 26 --- salt/logstash/init.sls | 57 +++-- .../pipelines/config/7100_osquery_wel.conf | 23 -- .../config/{ => so}/0010_input_hhbeats.conf | 0 .../config/{ => so}/0800_input_eval.conf | 0 .../{ => so}/0900_input_redis.conf.jinja | 0 .../{ => so}/1000_preprocess_log_elapsed.conf | 0 .../{ => so}/1001_preprocess_syslogng.conf | 0 .../config/{ => so}/1002_preprocess_json.conf | 0 .../1004_preprocess_syslog_types.conf | 0 .../config/{ => so}/1026_preprocess_dhcp.conf | 0 .../config/{ => so}/1029_preprocess_esxi.conf | 0 .../{ => so}/1030_preprocess_greensql.conf | 0 .../config/{ => so}/1031_preprocess_iis.conf | 0 .../{ => so}/1032_preprocess_mcafee.conf | 0 .../{ => so}/1033_preprocess_snort.conf | 0 .../{ => so}/1034_preprocess_syslog.conf | 0 .../{ => so}/1100_preprocess_bro_conn.conf | 0 .../{ => so}/1101_preprocess_bro_dhcp.conf | 0 .../{ => so}/1102_preprocess_bro_dns.conf | 0 .../{ => so}/1103_preprocess_bro_dpd.conf | 0 .../{ => so}/1104_preprocess_bro_files.conf | 0 .../{ => so}/1105_preprocess_bro_ftp.conf | 0 .../{ => so}/1106_preprocess_bro_http.conf | 0 .../{ => so}/1107_preprocess_bro_irc.conf | 0 .../1108_preprocess_bro_kerberos.conf | 0 .../{ => so}/1109_preprocess_bro_notice.conf | 0 .../{ => so}/1110_preprocess_bro_rdp.conf | 0 .../1111_preprocess_bro_signatures.conf | 0 .../{ => so}/1112_preprocess_bro_smtp.conf | 0 .../{ => so}/1113_preprocess_bro_snmp.conf | 0 .../1114_preprocess_bro_software.conf | 0 .../{ => so}/1115_preprocess_bro_ssh.conf | 0 .../{ => so}/1116_preprocess_bro_ssl.conf | 0 .../{ => so}/1117_preprocess_bro_syslog.conf | 0 .../{ => so}/1118_preprocess_bro_tunnel.conf | 0 .../{ => so}/1119_preprocess_bro_weird.conf | 0 .../{ => so}/1121_preprocess_bro_mysql.conf | 0 .../{ => so}/1122_preprocess_bro_socks.conf | 0 .../{ => so}/1123_preprocess_bro_x509.conf | 0 .../{ => so}/1124_preprocess_bro_intel.conf | 0 .../{ => so}/1125_preprocess_bro_modbus.conf | 0 .../{ => so}/1126_preprocess_bro_sip.conf | 0 .../{ => so}/1127_preprocess_bro_radius.conf | 0 .../{ => so}/1128_preprocess_bro_pe.conf | 0 .../{ => so}/1129_preprocess_bro_rfb.conf | 0 .../{ => so}/1130_preprocess_bro_dnp3.conf | 0 .../1131_preprocess_bro_smb_files.conf | 0 .../1132_preprocess_bro_smb_mapping.conf | 0 .../{ => so}/1133_preprocess_bro_ntlm.conf | 0 .../{ => so}/1134_preprocess_bro_dce_rpc.conf | 0 .../config/{ => so}/2000_network_flow.conf | 0 .../pipelines/config/{ => so}/6000_bro.conf | 0 .../config/{ => so}/6001_bro_import.conf | 0 .../config/{ => so}/6002_syslog.conf | 0 .../config/{ => so}/6101_switch_brocade.conf | 0 .../{ => so}/6200_firewall_fortinet.conf | 0 .../{ => so}/6201_firewall_pfsense.conf | 0 .../config/{ => so}/6300_windows.conf | 0 .../config/{ => so}/6301_dns_windows.conf | 0 .../config/{ => so}/6400_suricata.conf | 0 .../pipelines/config/{ => so}/6500_ossec.conf | 0 .../config/{ => so}/6501_ossec_sysmon.conf | 0 .../config/{ => so}/6502_ossec_autoruns.conf | 0 .../{ => so}/6600_winlogbeat_sysmon.conf | 0 .../config/{ => so}/6700_winlogbeat.conf | 0 .../config/so}/7100_osquery_wel.conf | 0 .../config/{ => so}/7200_strelka.conf | 0 .../8000_postprocess_bro_cleanup.conf | 0 ...01_postprocess_common_ip_augmentation.conf | 0 .../config/{ => so}/8006_postprocess_dns.conf | 0 .../{ => so}/8007_postprocess_http.conf | 0 .../{ => so}/8200_postprocess_tagging.conf | 0 .../8998_postprocess_log_elapsed.conf | 0 .../8999_postprocess_rename_type.conf | 0 .../{ => so}/9000_output_bro.conf.jinja | 0 .../{ => so}/9001_output_switch.conf.jinja | 0 .../{ => so}/9002_output_import.conf.jinja | 0 .../{ => so}/9004_output_flow.conf.jinja | 0 .../{ => so}/9026_output_dhcp.conf.jinja | 0 .../{ => so}/9029_output_esxi.conf.jinja | 0 .../{ => so}/9030_output_greensql.conf.jinja | 0 .../{ => so}/9031_output_iis.conf.jinja | 0 .../{ => so}/9032_output_mcafee.conf.jinja | 0 .../{ => so}/9033_output_snort.conf.jinja | 0 .../{ => so}/9034_output_syslog.conf.jinja | 0 .../{ => so}/9100_output_osquery.conf.jinja | 0 .../{ => so}/9200_output_firewall.conf.jinja | 0 .../{ => so}/9300_output_windows.conf.jinja | 0 .../9301_output_dns_windows.conf.jinja | 0 .../{ => so}/9400_output_suricata.conf.jinja | 0 .../{ => so}/9500_output_beats.conf.jinja | 0 .../{ => so}/9600_output_ossec.conf.jinja | 0 .../{ => so}/9700_output_strelka.conf.jinja | 0 .../{ => so}/9997_output_helix.conf.jinja | 0 .../{ => so}/9999_output_redis.conf.jinja | 0 .../Drop.Your.Custom.Templates.Here.conf | 0 .../templates/so}/beats-template.json | 0 .../so}/logstash-ossec-template.json | 0 .../so}/logstash-strelka-template.json | 0 .../templates/so}/logstash-template.json | 0 130 files changed, 191 insertions(+), 1081 deletions(-) create mode 100644 pillar/logstash/init.sls delete mode 100644 salt/logstash/defaults.yml delete mode 100644 salt/logstash/files/custom/parsers/Drop.Your.Custom.Parsers.Here.conf delete mode 100644 salt/logstash/files/dynamic/0006_input_beats.conf delete mode 100644 salt/logstash/files/dynamic/0008_input_eval.conf delete mode 100644 salt/logstash/files/dynamic/9000_output_bro.conf delete mode 100644 salt/logstash/files/dynamic/9001_output_switch.conf delete mode 100644 salt/logstash/files/dynamic/9002_output_import.conf delete mode 100644 salt/logstash/files/dynamic/9004_output_flow.conf delete mode 100644 salt/logstash/files/dynamic/9026_output_dhcp.conf delete mode 100644 salt/logstash/files/dynamic/9029_output_esxi.conf delete mode 100644 salt/logstash/files/dynamic/9030_output_greensql.conf delete mode 100644 salt/logstash/files/dynamic/9031_output_iis.conf delete mode 100644 salt/logstash/files/dynamic/9032_output_mcafee.conf delete mode 100644 salt/logstash/files/dynamic/9033_output_snort.conf delete mode 100644 salt/logstash/files/dynamic/9034_output_syslog.conf delete mode 100644 salt/logstash/files/dynamic/9100_output_osquery.conf delete mode 100644 salt/logstash/files/dynamic/9200_output_firewall.conf delete mode 100644 salt/logstash/files/dynamic/9300_output_windows.conf delete mode 100644 salt/logstash/files/dynamic/9301_output_dns_windows.conf delete mode 100644 salt/logstash/files/dynamic/9400_output_suricata.conf delete mode 100644 salt/logstash/files/dynamic/9500_output_beats.conf delete mode 100644 salt/logstash/files/dynamic/9600_output_ossec.conf delete mode 100644 salt/logstash/files/dynamic/9997_output_helix.conf delete mode 100644 salt/logstash/files/dynamic/9998_output_test_data.conf delete mode 100644 salt/logstash/pipelines/config/7100_osquery_wel.conf rename salt/logstash/pipelines/config/{ => so}/0010_input_hhbeats.conf (100%) rename salt/logstash/pipelines/config/{ => so}/0800_input_eval.conf (100%) rename salt/logstash/pipelines/config/{ => so}/0900_input_redis.conf.jinja (100%) rename salt/logstash/pipelines/config/{ => so}/1000_preprocess_log_elapsed.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1001_preprocess_syslogng.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1002_preprocess_json.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1004_preprocess_syslog_types.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1026_preprocess_dhcp.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1029_preprocess_esxi.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1030_preprocess_greensql.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1031_preprocess_iis.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1032_preprocess_mcafee.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1033_preprocess_snort.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1034_preprocess_syslog.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1100_preprocess_bro_conn.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1101_preprocess_bro_dhcp.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1102_preprocess_bro_dns.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1103_preprocess_bro_dpd.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1104_preprocess_bro_files.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1105_preprocess_bro_ftp.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1106_preprocess_bro_http.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1107_preprocess_bro_irc.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1108_preprocess_bro_kerberos.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1109_preprocess_bro_notice.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1110_preprocess_bro_rdp.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1111_preprocess_bro_signatures.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1112_preprocess_bro_smtp.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1113_preprocess_bro_snmp.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1114_preprocess_bro_software.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1115_preprocess_bro_ssh.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1116_preprocess_bro_ssl.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1117_preprocess_bro_syslog.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1118_preprocess_bro_tunnel.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1119_preprocess_bro_weird.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1121_preprocess_bro_mysql.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1122_preprocess_bro_socks.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1123_preprocess_bro_x509.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1124_preprocess_bro_intel.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1125_preprocess_bro_modbus.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1126_preprocess_bro_sip.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1127_preprocess_bro_radius.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1128_preprocess_bro_pe.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1129_preprocess_bro_rfb.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1130_preprocess_bro_dnp3.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1131_preprocess_bro_smb_files.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1132_preprocess_bro_smb_mapping.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1133_preprocess_bro_ntlm.conf (100%) rename salt/logstash/pipelines/config/{ => so}/1134_preprocess_bro_dce_rpc.conf (100%) rename salt/logstash/pipelines/config/{ => so}/2000_network_flow.conf (100%) rename salt/logstash/pipelines/config/{ => so}/6000_bro.conf (100%) rename salt/logstash/pipelines/config/{ => so}/6001_bro_import.conf (100%) rename salt/logstash/pipelines/config/{ => so}/6002_syslog.conf (100%) rename salt/logstash/pipelines/config/{ => so}/6101_switch_brocade.conf (100%) rename salt/logstash/pipelines/config/{ => so}/6200_firewall_fortinet.conf (100%) rename salt/logstash/pipelines/config/{ => so}/6201_firewall_pfsense.conf (100%) rename salt/logstash/pipelines/config/{ => so}/6300_windows.conf (100%) rename salt/logstash/pipelines/config/{ => so}/6301_dns_windows.conf (100%) rename salt/logstash/pipelines/config/{ => so}/6400_suricata.conf (100%) rename salt/logstash/pipelines/config/{ => so}/6500_ossec.conf (100%) rename salt/logstash/pipelines/config/{ => so}/6501_ossec_sysmon.conf (100%) rename salt/logstash/pipelines/config/{ => so}/6502_ossec_autoruns.conf (100%) rename salt/logstash/pipelines/config/{ => so}/6600_winlogbeat_sysmon.conf (100%) rename salt/logstash/pipelines/config/{ => so}/6700_winlogbeat.conf (100%) rename salt/logstash/{files/dynamic => pipelines/config/so}/7100_osquery_wel.conf (100%) rename salt/logstash/pipelines/config/{ => so}/7200_strelka.conf (100%) rename salt/logstash/pipelines/config/{ => so}/8000_postprocess_bro_cleanup.conf (100%) rename salt/logstash/pipelines/config/{ => so}/8001_postprocess_common_ip_augmentation.conf (100%) rename salt/logstash/pipelines/config/{ => so}/8006_postprocess_dns.conf (100%) rename salt/logstash/pipelines/config/{ => so}/8007_postprocess_http.conf (100%) rename salt/logstash/pipelines/config/{ => so}/8200_postprocess_tagging.conf (100%) rename salt/logstash/pipelines/config/{ => so}/8998_postprocess_log_elapsed.conf (100%) rename salt/logstash/pipelines/config/{ => so}/8999_postprocess_rename_type.conf (100%) rename salt/logstash/pipelines/config/{ => so}/9000_output_bro.conf.jinja (100%) rename salt/logstash/pipelines/config/{ => so}/9001_output_switch.conf.jinja (100%) rename salt/logstash/pipelines/config/{ => so}/9002_output_import.conf.jinja (100%) rename salt/logstash/pipelines/config/{ => so}/9004_output_flow.conf.jinja (100%) rename salt/logstash/pipelines/config/{ => so}/9026_output_dhcp.conf.jinja (100%) rename salt/logstash/pipelines/config/{ => so}/9029_output_esxi.conf.jinja (100%) rename salt/logstash/pipelines/config/{ => so}/9030_output_greensql.conf.jinja (100%) rename salt/logstash/pipelines/config/{ => so}/9031_output_iis.conf.jinja (100%) rename salt/logstash/pipelines/config/{ => so}/9032_output_mcafee.conf.jinja (100%) rename salt/logstash/pipelines/config/{ => so}/9033_output_snort.conf.jinja (100%) rename salt/logstash/pipelines/config/{ => so}/9034_output_syslog.conf.jinja (100%) rename salt/logstash/pipelines/config/{ => so}/9100_output_osquery.conf.jinja (100%) rename salt/logstash/pipelines/config/{ => so}/9200_output_firewall.conf.jinja (100%) rename salt/logstash/pipelines/config/{ => so}/9300_output_windows.conf.jinja (100%) rename salt/logstash/pipelines/config/{ => so}/9301_output_dns_windows.conf.jinja (100%) rename salt/logstash/pipelines/config/{ => so}/9400_output_suricata.conf.jinja (100%) rename salt/logstash/pipelines/config/{ => so}/9500_output_beats.conf.jinja (100%) rename salt/logstash/pipelines/config/{ => so}/9600_output_ossec.conf.jinja (100%) rename salt/logstash/pipelines/config/{ => so}/9700_output_strelka.conf.jinja (100%) rename salt/logstash/pipelines/config/{ => so}/9997_output_helix.conf.jinja (100%) rename salt/logstash/pipelines/config/{ => so}/9999_output_redis.conf.jinja (100%) rename salt/logstash/{files/custom/templates => pipelines/templates/custom}/Drop.Your.Custom.Templates.Here.conf (100%) rename salt/logstash/{etc => pipelines/templates/so}/beats-template.json (100%) rename salt/logstash/{etc => pipelines/templates/so}/logstash-ossec-template.json (100%) rename salt/logstash/{etc => pipelines/templates/so}/logstash-strelka-template.json (100%) rename salt/logstash/{etc => pipelines/templates/so}/logstash-template.json (100%) diff --git a/pillar/logstash/eval.sls b/pillar/logstash/eval.sls index c30e856d3..491dcfbd8 100644 --- a/pillar/logstash/eval.sls +++ b/pillar/logstash/eval.sls @@ -2,54 +2,59 @@ logstash: pipelines: eval: config: - - 0800_input_eval.conf - - 1000_preprocess_log_elapsed.conf - - 1001_preprocess_syslogng.conf - - 1002_preprocess_json.conf - - 1004_preprocess_syslog_types.conf - - 1026_preprocess_dhcp.conf - - 1029_preprocess_esxi.conf - - 1030_preprocess_greensql.conf - - 1031_preprocess_iis.conf - - 1032_preprocess_mcafee.conf - - 1033_preprocess_snort.conf - - 1034_preprocess_syslog.conf - - 2000_network_flow.conf - - 6002_syslog.conf - - 6101_switch_brocade.conf - - 6200_firewall_fortinet.conf - - 6201_firewall_pfsense.conf - - 6300_windows.conf - - 6301_dns_windows.conf - - 6400_suricata.conf - - 6500_ossec.conf - - 6501_ossec_sysmon.conf - - 6502_ossec_autoruns.conf - - 6600_winlogbeat_sysmon.conf - - 6700_winlogbeat.conf - - 7100_osquery_wel.conf - - 7200_strelka.conf - - 8001_postprocess_common_ip_augmentation.conf - - 8007_postprocess_http.conf - - 8200_postprocess_tagging.conf - - 8998_postprocess_log_elapsed.conf - - 8999_postprocess_rename_type.conf - - 9000_output_bro.conf.jinja - - 9001_output_switch.conf.jinja - - 9002_output_import.conf.jinja - - 9004_output_flow.conf.jinja - - 9026_output_dhcp.conf.jinja - - 9029_output_esxi.conf.jinja - - 9030_output_greensql.conf.jinja - - 9031_output_iis.conf.jinja - - 9032_output_mcafee.conf.jinja - - 9033_output_snort.conf.jinja - - 9034_output_syslog.conf.jinja - - 9100_output_osquery.conf.jinja - - 9200_output_firewall.conf.jinja - - 9300_output_windows.conf.jinja - - 9301_output_dns_windows.conf.jinja - - 9400_output_suricata.conf.jinja - - 9500_output_beats.conf.jinja - - 9600_output_ossec.conf.jinja - - 9700_ouptut_strelka.conf.jinja + - so/0800_input_eval.conf + - so/1000_preprocess_log_elapsed.conf + - so/1001_preprocess_syslogng.conf + - so/1002_preprocess_json.conf + - so/1004_preprocess_syslog_types.conf + - so/1026_preprocess_dhcp.conf + - so/1029_preprocess_esxi.conf + - so/1030_preprocess_greensql.conf + - so/1031_preprocess_iis.conf + - so/1032_preprocess_mcafee.conf + - so/1033_preprocess_snort.conf + - so/1034_preprocess_syslog.conf + - so/2000_network_flow.conf + - so/6002_syslog.conf + - so/6101_switch_brocade.conf + - so/6200_firewall_fortinet.conf + - so/6201_firewall_pfsense.conf + - so/6300_windows.conf + - so/6301_dns_windows.conf + - so/6400_suricata.conf + - so/6500_ossec.conf + - so/6501_ossec_sysmon.conf + - so/6502_ossec_autoruns.conf + - so/6600_winlogbeat_sysmon.conf + - so/6700_winlogbeat.conf + - so/7100_osquery_wel.conf + - so/7200_strelka.conf + - so/8001_postprocess_common_ip_augmentation.conf + - so/8007_postprocess_http.conf + - so/8200_postprocess_tagging.conf + - so/8998_postprocess_log_elapsed.conf + - so/8999_postprocess_rename_type.conf + - so/9000_output_bro.conf.jinja + - so/9001_output_switch.conf.jinja + - so/9002_output_import.conf.jinja + - so/9004_output_flow.conf.jinja + - so/9026_output_dhcp.conf.jinja + - so/9029_output_esxi.conf.jinja + - so/9030_output_greensql.conf.jinja + - so/9031_output_iis.conf.jinja + - so/9032_output_mcafee.conf.jinja + - so/9033_output_snort.conf.jinja + - so/9034_output_syslog.conf.jinja + - so/9100_output_osquery.conf.jinja + - so/9200_output_firewall.conf.jinja + - so/9300_output_windows.conf.jinja + - so/9301_output_dns_windows.conf.jinja + - so/9400_output_suricata.conf.jinja + - so/9500_output_beats.conf.jinja + - so/9600_output_ossec.conf.jinja + - so/9700_output_strelka.conf.jinja + templates: + - so/beats-template.json + - so/logstash-ossec-template.json + - so/logstash-strelka-template.json + - so/logstash-template.json diff --git a/pillar/logstash/helix.sls b/pillar/logstash/helix.sls index cdde880e1..ddc1c745b 100644 --- a/pillar/logstash/helix.sls +++ b/pillar/logstash/helix.sls @@ -2,41 +2,41 @@ logstash: pipelines: helix: config: - - 0010_input_hhbeats.conf - - 1033_preprocess_snort.conf - - 1100_preprocess_bro_conn.conf - - 1101_preprocess_bro_dhcp.conf - - 1102_preprocess_bro_dns.conf - - 1103_preprocess_bro_dpd.conf - - 1104_preprocess_bro_files.conf - - 1105_preprocess_bro_ftp.conf - - 1106_preprocess_bro_http.conf - - 1107_preprocess_bro_irc.conf - - 1108_preprocess_bro_kerberos.conf - - 1109_preprocess_bro_notice.conf - - 1110_preprocess_bro_rdp.conf - - 1111_preprocess_bro_signatures.conf - - 1112_preprocess_bro_smtp.conf - - 1113_preprocess_bro_snmp.conf - - 1114_preprocess_bro_software.conf - - 1115_preprocess_bro_ssh.conf - - 1116_preprocess_bro_ssl.conf - - 1117_preprocess_bro_syslog.conf - - 1118_preprocess_bro_tunnel.conf - - 1119_preprocess_bro_weird.conf - - 1121_preprocess_bro_mysql.conf - - 1122_preprocess_bro_socks.conf - - 1123_preprocess_bro_x509.conf - - 1124_preprocess_bro_intel.conf - - 1125_preprocess_bro_modbus.conf - - 1126_preprocess_bro_sip.conf - - 1127_preprocess_bro_radius.conf - - 1128_preprocess_bro_pe.conf - - 1129_preprocess_bro_rfb.conf - - 1130_preprocess_bro_dnp3.conf - - 1131_preprocess_bro_smb_files.conf - - 1132_preprocess_bro_smb_mapping.conf - - 1133_preprocess_bro_ntlm.conf - - 1134_preprocess_bro_dce_rpc.conf - - 8001_postprocess_common_ip_augmentation.conf - - 9997_output_helix.conf.jinja + - so/0010_input_hhbeats.conf + - so/1033_preprocess_snort.conf + - so/1100_preprocess_bro_conn.conf + - so/1101_preprocess_bro_dhcp.conf + - so/1102_preprocess_bro_dns.conf + - so/1103_preprocess_bro_dpd.conf + - so/1104_preprocess_bro_files.conf + - so/1105_preprocess_bro_ftp.conf + - so/1106_preprocess_bro_http.conf + - so/1107_preprocess_bro_irc.conf + - so/1108_preprocess_bro_kerberos.conf + - so/1109_preprocess_bro_notice.conf + - so/1110_preprocess_bro_rdp.conf + - so/1111_preprocess_bro_signatures.conf + - so/1112_preprocess_bro_smtp.conf + - so/1113_preprocess_bro_snmp.conf + - so/1114_preprocess_bro_software.conf + - so/1115_preprocess_bro_ssh.conf + - so/1116_preprocess_bro_ssl.conf + - so/1117_preprocess_bro_syslog.conf + - so/1118_preprocess_bro_tunnel.conf + - so/1119_preprocess_bro_weird.conf + - so/1121_preprocess_bro_mysql.conf + - so/1122_preprocess_bro_socks.conf + - so/1123_preprocess_bro_x509.conf + - so/1124_preprocess_bro_intel.conf + - so/1125_preprocess_bro_modbus.conf + - so/1126_preprocess_bro_sip.conf + - so/1127_preprocess_bro_radius.conf + - so/1128_preprocess_bro_pe.conf + - so/1129_preprocess_bro_rfb.conf + - so/1130_preprocess_bro_dnp3.conf + - so/1131_preprocess_bro_smb_files.conf + - so/1132_preprocess_bro_smb_mapping.conf + - so/1133_preprocess_bro_ntlm.conf + - so/1134_preprocess_bro_dce_rpc.conf + - so/8001_postprocess_common_ip_augmentation.conf + - so/9997_output_helix.conf.jinja diff --git a/pillar/logstash/init.sls b/pillar/logstash/init.sls new file mode 100644 index 000000000..6d51d0471 --- /dev/null +++ b/pillar/logstash/init.sls @@ -0,0 +1,11 @@ +logstash: + docker_options: + port_bindings: + - 0.0.0.0:514:514 + - 0.0.0.0:5044:5044 + - 0.0.0.0:5644:5644 + - 0.0.0.0:6050:6050 + - 0.0.0.0:6051:6051 + - 0.0.0.0:6052:6052 + - 0.0.0.0:6053:6053 + - 0.0.0.0:9600:9600 diff --git a/pillar/logstash/master.sls b/pillar/logstash/master.sls index f3ad90d7e..61d1bb6d1 100644 --- a/pillar/logstash/master.sls +++ b/pillar/logstash/master.sls @@ -2,5 +2,5 @@ logstash: pipelines: master: config: - - 0010_input_hhbeats.conf - - 9999_output_redis.conf.jinja + - so/0010_input_hhbeats.conf + - so/9999_output_redis.conf.jinja diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 3db36320f..4c295df14 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -2,54 +2,59 @@ logstash: pipelines: search: config: - - 1000_preprocess_log_elapsed.conf - - 1001_preprocess_syslogng.conf - - 1002_preprocess_json.conf - - 1004_preprocess_syslog_types.conf - - 1026_preprocess_dhcp.conf - - 1029_preprocess_esxi.conf - - 1030_preprocess_greensql.conf - - 1031_preprocess_iis.conf - - 1032_preprocess_mcafee.conf - - 1033_preprocess_snort.conf - - 1034_preprocess_syslog.conf - - 2000_network_flow.conf - - 6002_syslog.conf - - 6101_switch_brocade.conf - - 6200_firewall_fortinet.conf - - 6201_firewall_pfsense.conf - - 6300_windows.conf - - 6301_dns_windows.conf - - 6400_suricata.conf - - 6500_ossec.conf - - 6501_ossec_sysmon.conf - - 6502_ossec_autoruns.conf - - 6600_winlogbeat_sysmon.conf - - 6700_winlogbeat.conf - - 7100_osquery_wel.conf - - 7200_strelka.conf - - 8001_postprocess_common_ip_augmentation.conf - - 8007_postprocess_http.conf - - 8200_postprocess_tagging.conf - - 8998_postprocess_log_elapsed.conf - - 8999_postprocess_rename_type.conf - - 0900_input_redis.conf.jinja - - 9000_output_bro.conf.jinja - - 9001_output_switch.conf.jinja - - 9002_output_import.conf.jinja - - 9004_output_flow.conf.jinja - - 9026_output_dhcp.conf.jinja - - 9029_output_esxi.conf.jinja - - 9030_output_greensql.conf.jinja - - 9031_output_iis.conf.jinja - - 9032_output_mcafee.conf.jinja - - 9033_output_snort.conf.jinja - - 9034_output_syslog.conf.jinja - - 9100_output_osquery.conf.jinja - - 9200_output_firewall.conf.jinja - - 9300_output_windows.conf.jinja - - 9301_output_dns_windows.conf.jinja - - 9400_output_suricata.conf.jinja - - 9500_output_beats.conf.jinja - - 9600_output_ossec.conf.jinja - - 9700_output_strelka.conf.jinja + - so/1000_preprocess_log_elapsed.conf + - so/1001_preprocess_syslogng.conf + - so/1002_preprocess_json.conf + - so/1004_preprocess_syslog_types.conf + - so/1026_preprocess_dhcp.conf + - so/1029_preprocess_esxi.conf + - so/1030_preprocess_greensql.conf + - so/1031_preprocess_iis.conf + - so/1032_preprocess_mcafee.conf + - so/1033_preprocess_snort.conf + - so/1034_preprocess_syslog.conf + - so/2000_network_flow.conf + - so/6002_syslog.conf + - so/6101_switch_brocade.conf + - so/6200_firewall_fortinet.conf + - so/6201_firewall_pfsense.conf + - so/6300_windows.conf + - so/6301_dns_windows.conf + - so/6400_suricata.conf + - so/6500_ossec.conf + - so/6501_ossec_sysmon.conf + - so/6502_ossec_autoruns.conf + - so/6600_winlogbeat_sysmon.conf + - so/6700_winlogbeat.conf + - so/7100_osquery_wel.conf + - so/7200_strelka.conf + - so/8001_postprocess_common_ip_augmentation.conf + - so/8007_postprocess_http.conf + - so/8200_postprocess_tagging.conf + - so/8998_postprocess_log_elapsed.conf + - so/8999_postprocess_rename_type.conf + - so/0900_input_redis.conf.jinja + - so/9000_output_bro.conf.jinja + - so/9001_output_switch.conf.jinja + - so/9002_output_import.conf.jinja + - so/9004_output_flow.conf.jinja + - so/9026_output_dhcp.conf.jinja + - so/9029_output_esxi.conf.jinja + - so/9030_output_greensql.conf.jinja + - so/9031_output_iis.conf.jinja + - so/9032_output_mcafee.conf.jinja + - so/9033_output_snort.conf.jinja + - so/9034_output_syslog.conf.jinja + - so/9100_output_osquery.conf.jinja + - so/9200_output_firewall.conf.jinja + - so/9300_output_windows.conf.jinja + - so/9301_output_dns_windows.conf.jinja + - so/9400_output_suricata.conf.jinja + - so/9500_output_beats.conf.jinja + - so/9600_output_ossec.conf.jinja + - so/9700_output_strelka.conf.jinja + templates: + - so/beats-template.json + - so/logstash-ossec-template.json + - so/logstash-strelka-template.json + - so/logstash-template.json diff --git a/pillar/top.sls b/pillar/top.sls index 42d40ec10..85f28b22a 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -5,6 +5,7 @@ base: 'G@role:so-mastersearch or G@role:so-heavynode': - match: compound + - logstash - logstash.master - logstash.search @@ -23,6 +24,7 @@ base: - minions.{{ grains.id }} 'G@role:so-master': + - logstash - logstash.master 'G@role:so-eval': @@ -31,6 +33,7 @@ base: - data.* - brologs - auth + - logstash - logstash.eval - minions.{{ grains.id }} @@ -50,6 +53,6 @@ base: - firewall.* - fireeye - brologs + - logstash - logstash.helix - - static - minions.{{ grains.id }} diff --git a/salt/logstash/defaults.yml b/salt/logstash/defaults.yml deleted file mode 100644 index ba6d19534..000000000 --- a/salt/logstash/defaults.yml +++ /dev/null @@ -1,6 +0,0 @@ -logstash: - pipelines: - master: - config: "/usr/share/logstash/pipelines/master/*.conf" - search: - config: "/usr/share/logstash/pipelines/search/*.conf" diff --git a/salt/logstash/files/custom/parsers/Drop.Your.Custom.Parsers.Here.conf b/salt/logstash/files/custom/parsers/Drop.Your.Custom.Parsers.Here.conf deleted file mode 100644 index 6e9bbe36f..000000000 --- a/salt/logstash/files/custom/parsers/Drop.Your.Custom.Parsers.Here.conf +++ /dev/null @@ -1,2 +0,0 @@ -# -# diff --git a/salt/logstash/files/dynamic/0006_input_beats.conf b/salt/logstash/files/dynamic/0006_input_beats.conf deleted file mode 100644 index a7140f859..000000000 --- a/salt/logstash/files/dynamic/0006_input_beats.conf +++ /dev/null @@ -1,23 +0,0 @@ -input { - beats { - port => "5044" - ssl => false - ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"] - ssl_certificate => "/usr/share/logstash/filebeat.crt" - ssl_key => "/usr/share/logstash/filebeat.key" - tags => [ "beat" ] - } -} -filter { - if [type] == "osquery" { - mutate { - rename => { "host" => "beat_host" } - remove_tag => ["beat"] - add_tag => ["osquery"] - } - json { - source => "message" - target => "osquery" - } - } -} diff --git a/salt/logstash/files/dynamic/0008_input_eval.conf b/salt/logstash/files/dynamic/0008_input_eval.conf deleted file mode 100644 index b02f9d516..000000000 --- a/salt/logstash/files/dynamic/0008_input_eval.conf +++ /dev/null @@ -1,203 +0,0 @@ -# Updated by: Mike Reeves -# Last Update: 11/1/2018 - -input { - file { - path => "/suricata/eve.json" - type => "ids" - add_field => { "engine" => "suricata" } - } - file { - path => "/nsm/bro/logs/current/conn*.log" - type => "bro_conn" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/dce_rpc*.log" - type => "bro_dce_rpc" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/dhcp*.log" - type => "bro_dhcp" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/dnp3*.log" - type => "bro_dnp3" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/dns*.log" - type => "bro_dns" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/dpd*.log" - type => "bro_dpd" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/files*.log" - type => "bro_files" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/ftp*.log" - type => "bro_ftp" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/http*.log" - type => "bro_http" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/intel*.log" - type => "bro_intel" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/irc*.log" - type => "bro_irc" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/kerberos*.log" - type => "bro_kerberos" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/modbus*.log" - type => "bro_modbus" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/mysql*.log" - type => "bro_mysql" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/notice*.log" - type => "bro_notice" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/ntlm*.log" - type => "bro_ntlm" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/pe*.log" - type => "bro_pe" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/radius*.log" - type => "bro_radius" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/rdp*.log" - type => "bro_rdp" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/rfb*.log" - type => "bro_rfb" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/signatures*.log" - type => "bro_signatures" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/sip*.log" - type => "bro_sip" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/smb_files*.log" - type => "bro_smb_files" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/smb_mapping*.log" - type => "bro_smb_mapping" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/smtp*.log" - type => "bro_smtp" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/snmp*.log" - type => "bro_snmp" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/socks*.log" - type => "bro_socks" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/software*.log" - type => "bro_software" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/ssh*.log" - type => "bro_ssh" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/ssl*.log" - type => "bro_ssl" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/syslog*.log" - type => "bro_syslog" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/tunnel*.log" - type => "bro_tunnels" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/weird*.log" - type => "bro_weird" - tags => ["bro"] - } - file { - path => "/nsm/bro/logs/current/x509*.log" - type => "bro_x509" - tags => ["bro"] - } - file { - path => "/wazuh/alerts/alerts.json" - type => "ossec" - } - file { - path => "/wazuh/archives/archive.json" - type => "ossec_archive" - } - file { - path => "/osquery/logs/result.log" - type => "osquery" - } - file { - path => "/strelka/strelka.log" - type => "strelka" - } -} -filter { - if "import" in [tags] { - mutate { - #add_tag => [ "conf_file_0007"] - } - } -} diff --git a/salt/logstash/files/dynamic/9000_output_bro.conf b/salt/logstash/files/dynamic/9000_output_bro.conf deleted file mode 100644 index 553500281..000000000 --- a/salt/logstash/files/dynamic/9000_output_bro.conf +++ /dev/null @@ -1,31 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('node:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - - -filter { - if "bro" in [tags] and "test_data" not in [tags] and "import" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9000"] - } - } -} -output { - if "bro" in [tags] and "test_data" not in [tags] and "import" not in [tags] { -# stdout { codec => rubydebug } - elasticsearch { - pipeline => "%{event_type}" - hosts => "{{ ES }}" - index => "logstash-bro-%{+YYYY.MM.dd}" - template_name => "logstash" - template => "/logstash-template.json" - template_overwrite => true - } - } -} diff --git a/salt/logstash/files/dynamic/9001_output_switch.conf b/salt/logstash/files/dynamic/9001_output_switch.conf deleted file mode 100644 index 949a738ab..000000000 --- a/salt/logstash/files/dynamic/9001_output_switch.conf +++ /dev/null @@ -1,27 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('node:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if "switch" in [tags] and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9001"] - } - } -} -output { - if "switch" in [tags] and "test_data" not in [tags] { - #stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - index => "logstash-switch-%{+YYYY.MM.dd}" - template => "/logstash-template.json" - } - } -} diff --git a/salt/logstash/files/dynamic/9002_output_import.conf b/salt/logstash/files/dynamic/9002_output_import.conf deleted file mode 100644 index 1b691df6b..000000000 --- a/salt/logstash/files/dynamic/9002_output_import.conf +++ /dev/null @@ -1,27 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('node:mainip', '') -%} -{%- endif %} -# Updated by: Doug Burks -# Last Update: 5/16/2017 - -filter { - if "import" in [tags] and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9002"] - } - } -} -output { - if "import" in [tags] and "test_data" not in [tags] { -# stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - index => "logstash-import-%{+YYYY.MM.dd}" - template_name => "logstash" - template => "/logstash-template.json" - template_overwrite => true - } - } -} diff --git a/salt/logstash/files/dynamic/9004_output_flow.conf b/salt/logstash/files/dynamic/9004_output_flow.conf deleted file mode 100644 index 3dbd34f16..000000000 --- a/salt/logstash/files/dynamic/9004_output_flow.conf +++ /dev/null @@ -1,27 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('node:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "sflow" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9004"] - } - } -} -output { - if [event_type] == "sflow" and "test_data" not in [tags] { - #stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - index => "logstash-flow-%{+YYYY.MM.dd}" - template => "/logstash-template.json" - } - } -} diff --git a/salt/logstash/files/dynamic/9026_output_dhcp.conf b/salt/logstash/files/dynamic/9026_output_dhcp.conf deleted file mode 100644 index a63ac5f98..000000000 --- a/salt/logstash/files/dynamic/9026_output_dhcp.conf +++ /dev/null @@ -1,26 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('node:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "dhcp" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9026"] - } - } -} -output { - if [event_type] == "dhcp" and "test_data" not in [tags] { - #stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - template => "/logstash-template.json" - } - } -} diff --git a/salt/logstash/files/dynamic/9029_output_esxi.conf b/salt/logstash/files/dynamic/9029_output_esxi.conf deleted file mode 100644 index 229de6b9c..000000000 --- a/salt/logstash/files/dynamic/9029_output_esxi.conf +++ /dev/null @@ -1,25 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('node:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "esxi" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9029"] - } - } -} -output { - if [event_type] == "esxi" and "test_data" not in [tags] { - elasticsearch { - hosts => "{{ ES }}" - template => "/logstash-template.json" - } - } -} diff --git a/salt/logstash/files/dynamic/9030_output_greensql.conf b/salt/logstash/files/dynamic/9030_output_greensql.conf deleted file mode 100644 index a6d16b95d..000000000 --- a/salt/logstash/files/dynamic/9030_output_greensql.conf +++ /dev/null @@ -1,25 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('node:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "greensql" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9030"] - } - } -} -output { - if [event_type] == "greensql" and "test_data" not in [tags] { - elasticsearch { - hosts => "{{ ES }}" - template => "/logstash-template.json" - } - } -} diff --git a/salt/logstash/files/dynamic/9031_output_iis.conf b/salt/logstash/files/dynamic/9031_output_iis.conf deleted file mode 100644 index 6650d8a7d..000000000 --- a/salt/logstash/files/dynamic/9031_output_iis.conf +++ /dev/null @@ -1,26 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('node:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "iis" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9031"] - } - } -} -output { - if [event_type] == "iis" and "test_data" not in [tags] { - #stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - template => "/logstash-template.json" - } - } -} diff --git a/salt/logstash/files/dynamic/9032_output_mcafee.conf b/salt/logstash/files/dynamic/9032_output_mcafee.conf deleted file mode 100644 index ca982967d..000000000 --- a/salt/logstash/files/dynamic/9032_output_mcafee.conf +++ /dev/null @@ -1,26 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('node:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "mcafee" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9032"] - } - } -} -output { - if [event_type] == "mcafee" and "test_data" not in [tags] { - #stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - template => "/logstash-template.json" - } - } -} diff --git a/salt/logstash/files/dynamic/9033_output_snort.conf b/salt/logstash/files/dynamic/9033_output_snort.conf deleted file mode 100644 index 6c310b91e..000000000 --- a/salt/logstash/files/dynamic/9033_output_snort.conf +++ /dev/null @@ -1,29 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('node:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "ids" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9033"] - } - } -} -output { - if [event_type] == "ids" and "test_data" not in [tags] { - #stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - index => "logstash-ids-%{+YYYY.MM.dd}" - template_name => "logstash" - template => "/logstash-template.json" - template_overwrite => true - } - } -} diff --git a/salt/logstash/files/dynamic/9034_output_syslog.conf b/salt/logstash/files/dynamic/9034_output_syslog.conf deleted file mode 100644 index 56a6527b8..000000000 --- a/salt/logstash/files/dynamic/9034_output_syslog.conf +++ /dev/null @@ -1,28 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('node:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Updated by: Doug Burks -# Last Update: 5/15/2017 - -filter { - if "syslog" in [tags] and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9034"] - } - } -} -output { - if "syslog" in [tags] and "test_data" not in [tags] { - elasticsearch { - hosts => "{{ ES }}" - index => "logstash-syslog-%{+YYYY.MM.dd}" - template_name => "logstash" - template => "/logstash-template.json" - template_overwrite => true - } - } -} diff --git a/salt/logstash/files/dynamic/9100_output_osquery.conf b/salt/logstash/files/dynamic/9100_output_osquery.conf deleted file mode 100644 index e95119562..000000000 --- a/salt/logstash/files/dynamic/9100_output_osquery.conf +++ /dev/null @@ -1,19 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('node:mainip', '') -%} -{%- endif %} -# Author: Josh Brower -# Last Update: 12/29/2018 -# Output to ES for osquery tagged logs - - -output { - if "osquery" in [tags] { - elasticsearch { - hosts => "{{ ES }}" - index => "logstash-osquery-%{+YYYY.MM.dd}" - template => "/logstash-template.json" - } - } -} \ No newline at end of file diff --git a/salt/logstash/files/dynamic/9200_output_firewall.conf b/salt/logstash/files/dynamic/9200_output_firewall.conf deleted file mode 100644 index b2ad43963..000000000 --- a/salt/logstash/files/dynamic/9200_output_firewall.conf +++ /dev/null @@ -1,29 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('node:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if "firewall" in [tags] and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9200"] - } - } -} -output { - if "firewall" in [tags] and "test_data" not in [tags] { -# stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - index => "logstash-firewall-%{+YYYY.MM.dd}" - template_name => "logstash" - template => "/logstash-template.json" - template_overwrite => true - } - } -} diff --git a/salt/logstash/files/dynamic/9300_output_windows.conf b/salt/logstash/files/dynamic/9300_output_windows.conf deleted file mode 100644 index d3f9d1919..000000000 --- a/salt/logstash/files/dynamic/9300_output_windows.conf +++ /dev/null @@ -1,27 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('node:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "windows" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9300"] - } - } -} -output { - if [event_type] == "windows" and "test_data" not in [tags] { - #stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - index => "logstash-windows-%{+YYYY.MM.dd}" - template => "/logstash-template.json" - } - } -} diff --git a/salt/logstash/files/dynamic/9301_output_dns_windows.conf b/salt/logstash/files/dynamic/9301_output_dns_windows.conf deleted file mode 100644 index 8a56b7044..000000000 --- a/salt/logstash/files/dynamic/9301_output_dns_windows.conf +++ /dev/null @@ -1,27 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('node:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "dns" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9301"] - } - } -} -output { - if [event_type] == "dns" and "test_data" not in [tags] { - #stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - index => "logstash-%{+YYYY.MM.dd}" - template => "/logstash-template.json" - } - } -} diff --git a/salt/logstash/files/dynamic/9400_output_suricata.conf b/salt/logstash/files/dynamic/9400_output_suricata.conf deleted file mode 100644 index 4bffd7f0a..000000000 --- a/salt/logstash/files/dynamic/9400_output_suricata.conf +++ /dev/null @@ -1,27 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('node:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if [event_type] == "suricata" and "test_data" not in [tags] { - mutate { - ##add_tag => [ "conf_file_9400"] - } - } -} -output { - if [event_type] == "suricata" and "test_data" not in [tags] { - #stdout { codec => rubydebug } - elasticsearch { - hosts => "{{ ES }}" - index => "logstash-ids-%{+YYYY.MM.dd}" - template => "/logstash-template.json" - } - } -} diff --git a/salt/logstash/files/dynamic/9500_output_beats.conf b/salt/logstash/files/dynamic/9500_output_beats.conf deleted file mode 100644 index 30900cb93..000000000 --- a/salt/logstash/files/dynamic/9500_output_beats.conf +++ /dev/null @@ -1,25 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('node:mainip', '') -%} -{%- endif %} -# Author: Wes Lambert -# Last Update: 09/14/2018 -filter { - if "beat" in [tags] { - mutate { - ##add_tag => [ "conf_file_9500"] - } - } -} -output { - if "beat" in [tags] { - elasticsearch { - hosts => "{{ ES }}" - index => "logstash-beats-%{+YYYY.MM.dd}" - template_name => "logstash-beats" - template => "/beats-template.json" - template_overwrite => true - } - } -} diff --git a/salt/logstash/files/dynamic/9600_output_ossec.conf b/salt/logstash/files/dynamic/9600_output_ossec.conf deleted file mode 100644 index 71d0c28aa..000000000 --- a/salt/logstash/files/dynamic/9600_output_ossec.conf +++ /dev/null @@ -1,29 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('node:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Updated by: Doug Burks -# Last Update: 9/19/2018 - -filter { - if [event_type] =~ "ossec" { - mutate { - ##add_tag => [ "conf_file_9600"] - } - } -} - -output { - if [event_type] =~ "ossec" or "ossec" in [tags] { - elasticsearch { - hosts => "{{ ES }}" - index => "logstash-ossec-%{+YYYY.MM.dd}" - template_name => "logstash-ossec" - template => "/logstash-ossec-template.json" - template_overwrite => true - } - } -} diff --git a/salt/logstash/files/dynamic/9997_output_helix.conf b/salt/logstash/files/dynamic/9997_output_helix.conf deleted file mode 100644 index 5dd0036fe..000000000 --- a/salt/logstash/files/dynamic/9997_output_helix.conf +++ /dev/null @@ -1,142 +0,0 @@ -{% set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %} - -filter { - if [type] =~ /^bro_conn|bro_dns|bro_http|bro_files|bro_ssl|bro_dhcp|bro_x509$/ { - grok { - match => [ - "source_ip", "^%{IPV4:srcipv4}$", - "source_ip", "(?^([0-9A-Fa-f]{0,4}:){2,7}([0-9A-Fa-f]{1,4}$|((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.|$)){4})$)" - ] - } - grok { - match => [ - "destination_ip", "(?^([0-9A-Fa-f]{0,4}:){2,7}([0-9A-Fa-f]{1,4}$|((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.|$)){4})$)", - "destination_ip", "^%{IPV4:dstipv4}$" - ] - } - - geoip { - source => "[source_ip]" - target => "source_geo" - } - geoip { - source => "[destination_ip]" - target => "destination_geo" - } - mutate { - #rename => { "%{[source_geo][country_code]}" => "srccountrycode" } - #rename => { "%{[destination_geo][country_code]}" => "dstcountrycode" } - rename => { "[beat_host][name]" => "sensor" } - copy => { "sensor" => "rawmsghostname" } - rename => { "message" => "rawmsg" } - #rename => { "event_type" => "program" } - copy => { "type" => "class" } - copy => { "class" => "program"} - rename => { "source_port" => "srcport" } - rename => { "destination_port" => "dstport" } - remove_field => ["source_ip", "destination_ip"] - remove_field => ["sensorname", "sensor_name", "service", "source", "tags", "syslog-host"] - remove_field => ["sensor_name", "source_ips", "ips", "destination_ips", "syslog-priority", "syslog-file_name", "syslog-facility"] - } - if "bro_conn" in [class] { - mutate { - #add_field => { "metaclass" => "connection" } - rename => { "original_bytes" => "sentbytes" } - rename => { "respond_bytes" => "rcvdbytes" } - rename => { "connection_state" => "connstate" } - rename => { "uid" => "connectionid" } - rename => { "respond_packets" => "rcvdpackets" } - rename => { "original_packets" => "sentpackets" } - rename => { "respond_ip_bytes" => "rcvdipbytes" } - rename => { "original_ip_bytes" => "sentipbytes" } - rename => { "local_respond" => "local_resp" } - rename => { "local_orig" => "localorig" } - rename => { "missed_bytes" => "missingbytes" } - } - } - if "bro_dns" in [class] { - mutate{ - #add_field = { "metaclass" => "dns"} - rename => { "answers" => "answer" } - rename => { "query" => "domain" } - rename => { "query_class" => "queryclass" } - rename => { "query_class_name" => "queryclassname" } - rename => { "query_type" => "querytype" } - rename => { "query_type_name" => "querytypename" } - rename => { "ra" => "recursionavailable" } - rename => { "rd" => "recursiondesired" } - } - } - if "bro_dhcp" in [class] { - mutate{ - #add_field = { "metaclass" => "dhcp"} - rename => { "message_types" => "direction" } - rename => { "lease_time" => "duration" } - } - } - if "bro_files" in [class] { - mutate{ - #add_field = { "metaclass" => "dns"} - rename => { "missing_bytes" => "missingbytes" } - rename => { "fuid" => "fileid" } - rename => { "uid" => "connectionid" } - } - } - if "bro_http" in [class] { - mutate{ - #add_field = { "metaclass" => "dns"} - rename => { "virtual_host" => "hostname" } - rename => { "status_code" => "statuscode" } - rename => { "status_message" => "statusmsg" } - rename => { "resp_mime_types" => "rcvdmimetype" } - rename => { "resp_fuids" => "rcvdfileid" } - rename => { "response_body_len" => "rcvdbodybytes" } - rename => { "request_body_len" => "sentbodybytes" } - rename => { "uid" => "connectionid" } - rename => { "ts"=> "eventtime" } - rename => { "@timestamp"=> "eventtime" } - } - } - if "bro_ssl" in [class] { - mutate{ - #add_field = { "metaclass" => "dns"} - rename => { "status_code" => "statuscode" } - rename => { "status_message" => "statusmsg" } - rename => { "resp_mime_types" => "rcvdmimetype" } - rename => { "resp_fuids" => "rcvdfileid" } - rename => { "response_body_len" => "rcvdbodybytes" } - rename => { "request_body_len" => "sentbodybytes" } - } - } - if "bro_weird" in [class] { - mutate{ - #add_field = { "metaclass" => "dns"} - rename => { "name" => "eventname" } - } - } - if "bro_x509" in [class] { - mutate{ - #add_field = { "metaclass" => "dns"} - rename => { "certificate_common_name" => "certname" } - rename => { "certificate_subject" => "certsubject" } - rename => { "issuer_common_name" => "issuer" } - rename => { "certificate_issuer" => "issuersubject" } - rename => { "certificate_not_valid_before" => "issuetime" } - rename => { "certificate_key_type" => "cert_type" } - } - } - } -} - -output { - if [type] =~ /^bro_conn|bro_dns|bro_http|bro_files|bro_ssl|bro_dhcp|bro_x509$/ { - http { - url => "https://helix-integrations.cloud.aws.apps.fireeye.com/api/upload" - http_method => post - http_compression => true - socket_timeout => 60 - headers => ["Authorization","{{ HELIX_API_KEY }}"] - format => json_batch - } - } -} diff --git a/salt/logstash/files/dynamic/9998_output_test_data.conf b/salt/logstash/files/dynamic/9998_output_test_data.conf deleted file mode 100644 index 4e83aa185..000000000 --- a/salt/logstash/files/dynamic/9998_output_test_data.conf +++ /dev/null @@ -1,26 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('master:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('node:mainip', '') -%} -{%- endif %} -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if "test_data" in [tags] { - mutate { - #add_tag => [ "conf_file_9998"] - } - } -} -output { - if "test_data" in [tags] { - elasticsearch { - hosts => "{{ ES }}" - index => "logstash-test-%{+YYYY.MM.dd}" - template => "/logstash-template.json" - } - } -} diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 0b80caacf..6a12557ec 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -55,6 +55,8 @@ {% endif %} {% set PIPELINES = salt['pillar.get']('logstash:pipelines', {}) %} +{% set TEMPLATES = salt['pillar.get']('logstash:templates', {}) %} +{% set DOCKER_OPTIONS = salt['pillar.get']('logstash:docker_options', {}) %} # Create the logstash group logstashgroup: @@ -69,21 +71,6 @@ logstash: - gid: 931 - home: /opt/so/conf/logstash -# Create a directory for people to drop their own custom parsers into -lscustdir: - file.directory: - - name: /opt/so/conf/logstash/custom - - user: 931 - - group: 939 - - makedirs: True - -lsdyndir: - file.directory: - - name: /opt/so/conf/logstash/dynamic - - user: 931 - - group: 939 - - makedirs: True - lsetcdir: file.directory: - name: /opt/so/conf/logstash/etc @@ -91,13 +78,7 @@ lsetcdir: - group: 939 - makedirs: True -lscustparserdir: - file.directory: - - name: /opt/so/conf/logstash/custom/parsers - - user: 931 - - group: 939 - - makedirs: True - +# templates not specific to pipeline lscusttemplatedir: file.directory: - name: /opt/so/conf/logstash/custom/templates @@ -113,20 +94,35 @@ ls_pipeline_{{PL}}: - group: 939 {% for CONFIGFILE in PIPELINES[PL].config %} -ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0]}}: +ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: file.managed: - source: salt://logstash/pipelines/config/{{CONFIGFILE}} {% if 'jinja' in CONFIGFILE.split('.')[-1] %} - - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE | replace(".jinja", "")}} + - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE.split('/')[1] | replace(".jinja", "")}} - template: jinja {% else %} - - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE}} + - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE.split('/')[1]}} {% endif %} - user: 931 - group: 939 {% endfor %} {% endfor %} +#sync templates to /opt/so/conf/logstash/etc/ here +{% for TEMPLATE in TEMPLATES %} +ls_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}: + file.managed: + - source: salt://logstash/pipelines/templates/{{TEMPLATE}} + {% if 'jinja' in TEMPLATE.split('.')[-1] %} + - name: /opt/so/conf/logstash/etc/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}} + - template: jinja + {% else %} + - name: /opt/so/conf/logstash/etc/{{TEMPLATE.split('/')[1]}} + {% endif %} + - user: 931 + - group: 939 +{% endfor %} + lspipelinesyml: file.managed: - name: /opt/so/conf/logstash/etc/pipelines.yml @@ -193,14 +189,9 @@ so-logstash: - environment: - LS_JAVA_OPTS=-Xms{{ lsheap }} -Xmx{{ lsheap }} - port_bindings: - - 0.0.0.0:514:514 - - 0.0.0.0:5044:5044 - - 0.0.0.0:5644:5644 - - 0.0.0.0:6050:6050 - - 0.0.0.0:6051:6051 - - 0.0.0.0:6052:6052 - - 0.0.0.0:6053:6053 - - 0.0.0.0:9600:9600 +{% for BINDING in DOCKER_OPTIONS.port_bindings %} + - {{ BINDING }} +{% endfor %} - binds: - /opt/so/conf/logstash/etc/log4j2.properties:/usr/share/logstash/config/log4j2.properties:ro - /opt/so/conf/logstash/etc/logstash.yml:/usr/share/logstash/config/logstash.yml:ro diff --git a/salt/logstash/pipelines/config/7100_osquery_wel.conf b/salt/logstash/pipelines/config/7100_osquery_wel.conf deleted file mode 100644 index b4d77d83f..000000000 --- a/salt/logstash/pipelines/config/7100_osquery_wel.conf +++ /dev/null @@ -1,23 +0,0 @@ -# Author: Josh Brower -# Last Update: 12/28/2018 -# If log is tagged osquery and there is an eventid column - then cleanup and parse out the EventData column - -filter { - if "osquery" in [tags] and [osquery][columns][eventid] { - - mutate { - gsub => ["[osquery][columns][data]", "\\x0A", ""] - } - - json { - source => "[osquery][columns][data]" - target => "[osquery][columns][data]" - } - - mutate { - merge => { "[osquery][columns]" => "[osquery][columns][data]" } - remove_field => ["[osquery][columns][data]"] - } - - } -} \ No newline at end of file diff --git a/salt/logstash/pipelines/config/0010_input_hhbeats.conf b/salt/logstash/pipelines/config/so/0010_input_hhbeats.conf similarity index 100% rename from salt/logstash/pipelines/config/0010_input_hhbeats.conf rename to salt/logstash/pipelines/config/so/0010_input_hhbeats.conf diff --git a/salt/logstash/pipelines/config/0800_input_eval.conf b/salt/logstash/pipelines/config/so/0800_input_eval.conf similarity index 100% rename from salt/logstash/pipelines/config/0800_input_eval.conf rename to salt/logstash/pipelines/config/so/0800_input_eval.conf diff --git a/salt/logstash/pipelines/config/0900_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/0900_input_redis.conf.jinja rename to salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja diff --git a/salt/logstash/pipelines/config/1000_preprocess_log_elapsed.conf b/salt/logstash/pipelines/config/so/1000_preprocess_log_elapsed.conf similarity index 100% rename from salt/logstash/pipelines/config/1000_preprocess_log_elapsed.conf rename to salt/logstash/pipelines/config/so/1000_preprocess_log_elapsed.conf diff --git a/salt/logstash/pipelines/config/1001_preprocess_syslogng.conf b/salt/logstash/pipelines/config/so/1001_preprocess_syslogng.conf similarity index 100% rename from salt/logstash/pipelines/config/1001_preprocess_syslogng.conf rename to salt/logstash/pipelines/config/so/1001_preprocess_syslogng.conf diff --git a/salt/logstash/pipelines/config/1002_preprocess_json.conf b/salt/logstash/pipelines/config/so/1002_preprocess_json.conf similarity index 100% rename from salt/logstash/pipelines/config/1002_preprocess_json.conf rename to salt/logstash/pipelines/config/so/1002_preprocess_json.conf diff --git a/salt/logstash/pipelines/config/1004_preprocess_syslog_types.conf b/salt/logstash/pipelines/config/so/1004_preprocess_syslog_types.conf similarity index 100% rename from salt/logstash/pipelines/config/1004_preprocess_syslog_types.conf rename to salt/logstash/pipelines/config/so/1004_preprocess_syslog_types.conf diff --git a/salt/logstash/pipelines/config/1026_preprocess_dhcp.conf b/salt/logstash/pipelines/config/so/1026_preprocess_dhcp.conf similarity index 100% rename from salt/logstash/pipelines/config/1026_preprocess_dhcp.conf rename to salt/logstash/pipelines/config/so/1026_preprocess_dhcp.conf diff --git a/salt/logstash/pipelines/config/1029_preprocess_esxi.conf b/salt/logstash/pipelines/config/so/1029_preprocess_esxi.conf similarity index 100% rename from salt/logstash/pipelines/config/1029_preprocess_esxi.conf rename to salt/logstash/pipelines/config/so/1029_preprocess_esxi.conf diff --git a/salt/logstash/pipelines/config/1030_preprocess_greensql.conf b/salt/logstash/pipelines/config/so/1030_preprocess_greensql.conf similarity index 100% rename from salt/logstash/pipelines/config/1030_preprocess_greensql.conf rename to salt/logstash/pipelines/config/so/1030_preprocess_greensql.conf diff --git a/salt/logstash/pipelines/config/1031_preprocess_iis.conf b/salt/logstash/pipelines/config/so/1031_preprocess_iis.conf similarity index 100% rename from salt/logstash/pipelines/config/1031_preprocess_iis.conf rename to salt/logstash/pipelines/config/so/1031_preprocess_iis.conf diff --git a/salt/logstash/pipelines/config/1032_preprocess_mcafee.conf b/salt/logstash/pipelines/config/so/1032_preprocess_mcafee.conf similarity index 100% rename from salt/logstash/pipelines/config/1032_preprocess_mcafee.conf rename to salt/logstash/pipelines/config/so/1032_preprocess_mcafee.conf diff --git a/salt/logstash/pipelines/config/1033_preprocess_snort.conf b/salt/logstash/pipelines/config/so/1033_preprocess_snort.conf similarity index 100% rename from salt/logstash/pipelines/config/1033_preprocess_snort.conf rename to salt/logstash/pipelines/config/so/1033_preprocess_snort.conf diff --git a/salt/logstash/pipelines/config/1034_preprocess_syslog.conf b/salt/logstash/pipelines/config/so/1034_preprocess_syslog.conf similarity index 100% rename from salt/logstash/pipelines/config/1034_preprocess_syslog.conf rename to salt/logstash/pipelines/config/so/1034_preprocess_syslog.conf diff --git a/salt/logstash/pipelines/config/1100_preprocess_bro_conn.conf b/salt/logstash/pipelines/config/so/1100_preprocess_bro_conn.conf similarity index 100% rename from salt/logstash/pipelines/config/1100_preprocess_bro_conn.conf rename to salt/logstash/pipelines/config/so/1100_preprocess_bro_conn.conf diff --git a/salt/logstash/pipelines/config/1101_preprocess_bro_dhcp.conf b/salt/logstash/pipelines/config/so/1101_preprocess_bro_dhcp.conf similarity index 100% rename from salt/logstash/pipelines/config/1101_preprocess_bro_dhcp.conf rename to salt/logstash/pipelines/config/so/1101_preprocess_bro_dhcp.conf diff --git a/salt/logstash/pipelines/config/1102_preprocess_bro_dns.conf b/salt/logstash/pipelines/config/so/1102_preprocess_bro_dns.conf similarity index 100% rename from salt/logstash/pipelines/config/1102_preprocess_bro_dns.conf rename to salt/logstash/pipelines/config/so/1102_preprocess_bro_dns.conf diff --git a/salt/logstash/pipelines/config/1103_preprocess_bro_dpd.conf b/salt/logstash/pipelines/config/so/1103_preprocess_bro_dpd.conf similarity index 100% rename from salt/logstash/pipelines/config/1103_preprocess_bro_dpd.conf rename to salt/logstash/pipelines/config/so/1103_preprocess_bro_dpd.conf diff --git a/salt/logstash/pipelines/config/1104_preprocess_bro_files.conf b/salt/logstash/pipelines/config/so/1104_preprocess_bro_files.conf similarity index 100% rename from salt/logstash/pipelines/config/1104_preprocess_bro_files.conf rename to salt/logstash/pipelines/config/so/1104_preprocess_bro_files.conf diff --git a/salt/logstash/pipelines/config/1105_preprocess_bro_ftp.conf b/salt/logstash/pipelines/config/so/1105_preprocess_bro_ftp.conf similarity index 100% rename from salt/logstash/pipelines/config/1105_preprocess_bro_ftp.conf rename to salt/logstash/pipelines/config/so/1105_preprocess_bro_ftp.conf diff --git a/salt/logstash/pipelines/config/1106_preprocess_bro_http.conf b/salt/logstash/pipelines/config/so/1106_preprocess_bro_http.conf similarity index 100% rename from salt/logstash/pipelines/config/1106_preprocess_bro_http.conf rename to salt/logstash/pipelines/config/so/1106_preprocess_bro_http.conf diff --git a/salt/logstash/pipelines/config/1107_preprocess_bro_irc.conf b/salt/logstash/pipelines/config/so/1107_preprocess_bro_irc.conf similarity index 100% rename from salt/logstash/pipelines/config/1107_preprocess_bro_irc.conf rename to salt/logstash/pipelines/config/so/1107_preprocess_bro_irc.conf diff --git a/salt/logstash/pipelines/config/1108_preprocess_bro_kerberos.conf b/salt/logstash/pipelines/config/so/1108_preprocess_bro_kerberos.conf similarity index 100% rename from salt/logstash/pipelines/config/1108_preprocess_bro_kerberos.conf rename to salt/logstash/pipelines/config/so/1108_preprocess_bro_kerberos.conf diff --git a/salt/logstash/pipelines/config/1109_preprocess_bro_notice.conf b/salt/logstash/pipelines/config/so/1109_preprocess_bro_notice.conf similarity index 100% rename from salt/logstash/pipelines/config/1109_preprocess_bro_notice.conf rename to salt/logstash/pipelines/config/so/1109_preprocess_bro_notice.conf diff --git a/salt/logstash/pipelines/config/1110_preprocess_bro_rdp.conf b/salt/logstash/pipelines/config/so/1110_preprocess_bro_rdp.conf similarity index 100% rename from salt/logstash/pipelines/config/1110_preprocess_bro_rdp.conf rename to salt/logstash/pipelines/config/so/1110_preprocess_bro_rdp.conf diff --git a/salt/logstash/pipelines/config/1111_preprocess_bro_signatures.conf b/salt/logstash/pipelines/config/so/1111_preprocess_bro_signatures.conf similarity index 100% rename from salt/logstash/pipelines/config/1111_preprocess_bro_signatures.conf rename to salt/logstash/pipelines/config/so/1111_preprocess_bro_signatures.conf diff --git a/salt/logstash/pipelines/config/1112_preprocess_bro_smtp.conf b/salt/logstash/pipelines/config/so/1112_preprocess_bro_smtp.conf similarity index 100% rename from salt/logstash/pipelines/config/1112_preprocess_bro_smtp.conf rename to salt/logstash/pipelines/config/so/1112_preprocess_bro_smtp.conf diff --git a/salt/logstash/pipelines/config/1113_preprocess_bro_snmp.conf b/salt/logstash/pipelines/config/so/1113_preprocess_bro_snmp.conf similarity index 100% rename from salt/logstash/pipelines/config/1113_preprocess_bro_snmp.conf rename to salt/logstash/pipelines/config/so/1113_preprocess_bro_snmp.conf diff --git a/salt/logstash/pipelines/config/1114_preprocess_bro_software.conf b/salt/logstash/pipelines/config/so/1114_preprocess_bro_software.conf similarity index 100% rename from salt/logstash/pipelines/config/1114_preprocess_bro_software.conf rename to salt/logstash/pipelines/config/so/1114_preprocess_bro_software.conf diff --git a/salt/logstash/pipelines/config/1115_preprocess_bro_ssh.conf b/salt/logstash/pipelines/config/so/1115_preprocess_bro_ssh.conf similarity index 100% rename from salt/logstash/pipelines/config/1115_preprocess_bro_ssh.conf rename to salt/logstash/pipelines/config/so/1115_preprocess_bro_ssh.conf diff --git a/salt/logstash/pipelines/config/1116_preprocess_bro_ssl.conf b/salt/logstash/pipelines/config/so/1116_preprocess_bro_ssl.conf similarity index 100% rename from salt/logstash/pipelines/config/1116_preprocess_bro_ssl.conf rename to salt/logstash/pipelines/config/so/1116_preprocess_bro_ssl.conf diff --git a/salt/logstash/pipelines/config/1117_preprocess_bro_syslog.conf b/salt/logstash/pipelines/config/so/1117_preprocess_bro_syslog.conf similarity index 100% rename from salt/logstash/pipelines/config/1117_preprocess_bro_syslog.conf rename to salt/logstash/pipelines/config/so/1117_preprocess_bro_syslog.conf diff --git a/salt/logstash/pipelines/config/1118_preprocess_bro_tunnel.conf b/salt/logstash/pipelines/config/so/1118_preprocess_bro_tunnel.conf similarity index 100% rename from salt/logstash/pipelines/config/1118_preprocess_bro_tunnel.conf rename to salt/logstash/pipelines/config/so/1118_preprocess_bro_tunnel.conf diff --git a/salt/logstash/pipelines/config/1119_preprocess_bro_weird.conf b/salt/logstash/pipelines/config/so/1119_preprocess_bro_weird.conf similarity index 100% rename from salt/logstash/pipelines/config/1119_preprocess_bro_weird.conf rename to salt/logstash/pipelines/config/so/1119_preprocess_bro_weird.conf diff --git a/salt/logstash/pipelines/config/1121_preprocess_bro_mysql.conf b/salt/logstash/pipelines/config/so/1121_preprocess_bro_mysql.conf similarity index 100% rename from salt/logstash/pipelines/config/1121_preprocess_bro_mysql.conf rename to salt/logstash/pipelines/config/so/1121_preprocess_bro_mysql.conf diff --git a/salt/logstash/pipelines/config/1122_preprocess_bro_socks.conf b/salt/logstash/pipelines/config/so/1122_preprocess_bro_socks.conf similarity index 100% rename from salt/logstash/pipelines/config/1122_preprocess_bro_socks.conf rename to salt/logstash/pipelines/config/so/1122_preprocess_bro_socks.conf diff --git a/salt/logstash/pipelines/config/1123_preprocess_bro_x509.conf b/salt/logstash/pipelines/config/so/1123_preprocess_bro_x509.conf similarity index 100% rename from salt/logstash/pipelines/config/1123_preprocess_bro_x509.conf rename to salt/logstash/pipelines/config/so/1123_preprocess_bro_x509.conf diff --git a/salt/logstash/pipelines/config/1124_preprocess_bro_intel.conf b/salt/logstash/pipelines/config/so/1124_preprocess_bro_intel.conf similarity index 100% rename from salt/logstash/pipelines/config/1124_preprocess_bro_intel.conf rename to salt/logstash/pipelines/config/so/1124_preprocess_bro_intel.conf diff --git a/salt/logstash/pipelines/config/1125_preprocess_bro_modbus.conf b/salt/logstash/pipelines/config/so/1125_preprocess_bro_modbus.conf similarity index 100% rename from salt/logstash/pipelines/config/1125_preprocess_bro_modbus.conf rename to salt/logstash/pipelines/config/so/1125_preprocess_bro_modbus.conf diff --git a/salt/logstash/pipelines/config/1126_preprocess_bro_sip.conf b/salt/logstash/pipelines/config/so/1126_preprocess_bro_sip.conf similarity index 100% rename from salt/logstash/pipelines/config/1126_preprocess_bro_sip.conf rename to salt/logstash/pipelines/config/so/1126_preprocess_bro_sip.conf diff --git a/salt/logstash/pipelines/config/1127_preprocess_bro_radius.conf b/salt/logstash/pipelines/config/so/1127_preprocess_bro_radius.conf similarity index 100% rename from salt/logstash/pipelines/config/1127_preprocess_bro_radius.conf rename to salt/logstash/pipelines/config/so/1127_preprocess_bro_radius.conf diff --git a/salt/logstash/pipelines/config/1128_preprocess_bro_pe.conf b/salt/logstash/pipelines/config/so/1128_preprocess_bro_pe.conf similarity index 100% rename from salt/logstash/pipelines/config/1128_preprocess_bro_pe.conf rename to salt/logstash/pipelines/config/so/1128_preprocess_bro_pe.conf diff --git a/salt/logstash/pipelines/config/1129_preprocess_bro_rfb.conf b/salt/logstash/pipelines/config/so/1129_preprocess_bro_rfb.conf similarity index 100% rename from salt/logstash/pipelines/config/1129_preprocess_bro_rfb.conf rename to salt/logstash/pipelines/config/so/1129_preprocess_bro_rfb.conf diff --git a/salt/logstash/pipelines/config/1130_preprocess_bro_dnp3.conf b/salt/logstash/pipelines/config/so/1130_preprocess_bro_dnp3.conf similarity index 100% rename from salt/logstash/pipelines/config/1130_preprocess_bro_dnp3.conf rename to salt/logstash/pipelines/config/so/1130_preprocess_bro_dnp3.conf diff --git a/salt/logstash/pipelines/config/1131_preprocess_bro_smb_files.conf b/salt/logstash/pipelines/config/so/1131_preprocess_bro_smb_files.conf similarity index 100% rename from salt/logstash/pipelines/config/1131_preprocess_bro_smb_files.conf rename to salt/logstash/pipelines/config/so/1131_preprocess_bro_smb_files.conf diff --git a/salt/logstash/pipelines/config/1132_preprocess_bro_smb_mapping.conf b/salt/logstash/pipelines/config/so/1132_preprocess_bro_smb_mapping.conf similarity index 100% rename from salt/logstash/pipelines/config/1132_preprocess_bro_smb_mapping.conf rename to salt/logstash/pipelines/config/so/1132_preprocess_bro_smb_mapping.conf diff --git a/salt/logstash/pipelines/config/1133_preprocess_bro_ntlm.conf b/salt/logstash/pipelines/config/so/1133_preprocess_bro_ntlm.conf similarity index 100% rename from salt/logstash/pipelines/config/1133_preprocess_bro_ntlm.conf rename to salt/logstash/pipelines/config/so/1133_preprocess_bro_ntlm.conf diff --git a/salt/logstash/pipelines/config/1134_preprocess_bro_dce_rpc.conf b/salt/logstash/pipelines/config/so/1134_preprocess_bro_dce_rpc.conf similarity index 100% rename from salt/logstash/pipelines/config/1134_preprocess_bro_dce_rpc.conf rename to salt/logstash/pipelines/config/so/1134_preprocess_bro_dce_rpc.conf diff --git a/salt/logstash/pipelines/config/2000_network_flow.conf b/salt/logstash/pipelines/config/so/2000_network_flow.conf similarity index 100% rename from salt/logstash/pipelines/config/2000_network_flow.conf rename to salt/logstash/pipelines/config/so/2000_network_flow.conf diff --git a/salt/logstash/pipelines/config/6000_bro.conf b/salt/logstash/pipelines/config/so/6000_bro.conf similarity index 100% rename from salt/logstash/pipelines/config/6000_bro.conf rename to salt/logstash/pipelines/config/so/6000_bro.conf diff --git a/salt/logstash/pipelines/config/6001_bro_import.conf b/salt/logstash/pipelines/config/so/6001_bro_import.conf similarity index 100% rename from salt/logstash/pipelines/config/6001_bro_import.conf rename to salt/logstash/pipelines/config/so/6001_bro_import.conf diff --git a/salt/logstash/pipelines/config/6002_syslog.conf b/salt/logstash/pipelines/config/so/6002_syslog.conf similarity index 100% rename from salt/logstash/pipelines/config/6002_syslog.conf rename to salt/logstash/pipelines/config/so/6002_syslog.conf diff --git a/salt/logstash/pipelines/config/6101_switch_brocade.conf b/salt/logstash/pipelines/config/so/6101_switch_brocade.conf similarity index 100% rename from salt/logstash/pipelines/config/6101_switch_brocade.conf rename to salt/logstash/pipelines/config/so/6101_switch_brocade.conf diff --git a/salt/logstash/pipelines/config/6200_firewall_fortinet.conf b/salt/logstash/pipelines/config/so/6200_firewall_fortinet.conf similarity index 100% rename from salt/logstash/pipelines/config/6200_firewall_fortinet.conf rename to salt/logstash/pipelines/config/so/6200_firewall_fortinet.conf diff --git a/salt/logstash/pipelines/config/6201_firewall_pfsense.conf b/salt/logstash/pipelines/config/so/6201_firewall_pfsense.conf similarity index 100% rename from salt/logstash/pipelines/config/6201_firewall_pfsense.conf rename to salt/logstash/pipelines/config/so/6201_firewall_pfsense.conf diff --git a/salt/logstash/pipelines/config/6300_windows.conf b/salt/logstash/pipelines/config/so/6300_windows.conf similarity index 100% rename from salt/logstash/pipelines/config/6300_windows.conf rename to salt/logstash/pipelines/config/so/6300_windows.conf diff --git a/salt/logstash/pipelines/config/6301_dns_windows.conf b/salt/logstash/pipelines/config/so/6301_dns_windows.conf similarity index 100% rename from salt/logstash/pipelines/config/6301_dns_windows.conf rename to salt/logstash/pipelines/config/so/6301_dns_windows.conf diff --git a/salt/logstash/pipelines/config/6400_suricata.conf b/salt/logstash/pipelines/config/so/6400_suricata.conf similarity index 100% rename from salt/logstash/pipelines/config/6400_suricata.conf rename to salt/logstash/pipelines/config/so/6400_suricata.conf diff --git a/salt/logstash/pipelines/config/6500_ossec.conf b/salt/logstash/pipelines/config/so/6500_ossec.conf similarity index 100% rename from salt/logstash/pipelines/config/6500_ossec.conf rename to salt/logstash/pipelines/config/so/6500_ossec.conf diff --git a/salt/logstash/pipelines/config/6501_ossec_sysmon.conf b/salt/logstash/pipelines/config/so/6501_ossec_sysmon.conf similarity index 100% rename from salt/logstash/pipelines/config/6501_ossec_sysmon.conf rename to salt/logstash/pipelines/config/so/6501_ossec_sysmon.conf diff --git a/salt/logstash/pipelines/config/6502_ossec_autoruns.conf b/salt/logstash/pipelines/config/so/6502_ossec_autoruns.conf similarity index 100% rename from salt/logstash/pipelines/config/6502_ossec_autoruns.conf rename to salt/logstash/pipelines/config/so/6502_ossec_autoruns.conf diff --git a/salt/logstash/pipelines/config/6600_winlogbeat_sysmon.conf b/salt/logstash/pipelines/config/so/6600_winlogbeat_sysmon.conf similarity index 100% rename from salt/logstash/pipelines/config/6600_winlogbeat_sysmon.conf rename to salt/logstash/pipelines/config/so/6600_winlogbeat_sysmon.conf diff --git a/salt/logstash/pipelines/config/6700_winlogbeat.conf b/salt/logstash/pipelines/config/so/6700_winlogbeat.conf similarity index 100% rename from salt/logstash/pipelines/config/6700_winlogbeat.conf rename to salt/logstash/pipelines/config/so/6700_winlogbeat.conf diff --git a/salt/logstash/files/dynamic/7100_osquery_wel.conf b/salt/logstash/pipelines/config/so/7100_osquery_wel.conf similarity index 100% rename from salt/logstash/files/dynamic/7100_osquery_wel.conf rename to salt/logstash/pipelines/config/so/7100_osquery_wel.conf diff --git a/salt/logstash/pipelines/config/7200_strelka.conf b/salt/logstash/pipelines/config/so/7200_strelka.conf similarity index 100% rename from salt/logstash/pipelines/config/7200_strelka.conf rename to salt/logstash/pipelines/config/so/7200_strelka.conf diff --git a/salt/logstash/pipelines/config/8000_postprocess_bro_cleanup.conf b/salt/logstash/pipelines/config/so/8000_postprocess_bro_cleanup.conf similarity index 100% rename from salt/logstash/pipelines/config/8000_postprocess_bro_cleanup.conf rename to salt/logstash/pipelines/config/so/8000_postprocess_bro_cleanup.conf diff --git a/salt/logstash/pipelines/config/8001_postprocess_common_ip_augmentation.conf b/salt/logstash/pipelines/config/so/8001_postprocess_common_ip_augmentation.conf similarity index 100% rename from salt/logstash/pipelines/config/8001_postprocess_common_ip_augmentation.conf rename to salt/logstash/pipelines/config/so/8001_postprocess_common_ip_augmentation.conf diff --git a/salt/logstash/pipelines/config/8006_postprocess_dns.conf b/salt/logstash/pipelines/config/so/8006_postprocess_dns.conf similarity index 100% rename from salt/logstash/pipelines/config/8006_postprocess_dns.conf rename to salt/logstash/pipelines/config/so/8006_postprocess_dns.conf diff --git a/salt/logstash/pipelines/config/8007_postprocess_http.conf b/salt/logstash/pipelines/config/so/8007_postprocess_http.conf similarity index 100% rename from salt/logstash/pipelines/config/8007_postprocess_http.conf rename to salt/logstash/pipelines/config/so/8007_postprocess_http.conf diff --git a/salt/logstash/pipelines/config/8200_postprocess_tagging.conf b/salt/logstash/pipelines/config/so/8200_postprocess_tagging.conf similarity index 100% rename from salt/logstash/pipelines/config/8200_postprocess_tagging.conf rename to salt/logstash/pipelines/config/so/8200_postprocess_tagging.conf diff --git a/salt/logstash/pipelines/config/8998_postprocess_log_elapsed.conf b/salt/logstash/pipelines/config/so/8998_postprocess_log_elapsed.conf similarity index 100% rename from salt/logstash/pipelines/config/8998_postprocess_log_elapsed.conf rename to salt/logstash/pipelines/config/so/8998_postprocess_log_elapsed.conf diff --git a/salt/logstash/pipelines/config/8999_postprocess_rename_type.conf b/salt/logstash/pipelines/config/so/8999_postprocess_rename_type.conf similarity index 100% rename from salt/logstash/pipelines/config/8999_postprocess_rename_type.conf rename to salt/logstash/pipelines/config/so/8999_postprocess_rename_type.conf diff --git a/salt/logstash/pipelines/config/9000_output_bro.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_bro.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/9000_output_bro.conf.jinja rename to salt/logstash/pipelines/config/so/9000_output_bro.conf.jinja diff --git a/salt/logstash/pipelines/config/9001_output_switch.conf.jinja b/salt/logstash/pipelines/config/so/9001_output_switch.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/9001_output_switch.conf.jinja rename to salt/logstash/pipelines/config/so/9001_output_switch.conf.jinja diff --git a/salt/logstash/pipelines/config/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/9002_output_import.conf.jinja rename to salt/logstash/pipelines/config/so/9002_output_import.conf.jinja diff --git a/salt/logstash/pipelines/config/9004_output_flow.conf.jinja b/salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/9004_output_flow.conf.jinja rename to salt/logstash/pipelines/config/so/9004_output_flow.conf.jinja diff --git a/salt/logstash/pipelines/config/9026_output_dhcp.conf.jinja b/salt/logstash/pipelines/config/so/9026_output_dhcp.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/9026_output_dhcp.conf.jinja rename to salt/logstash/pipelines/config/so/9026_output_dhcp.conf.jinja diff --git a/salt/logstash/pipelines/config/9029_output_esxi.conf.jinja b/salt/logstash/pipelines/config/so/9029_output_esxi.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/9029_output_esxi.conf.jinja rename to salt/logstash/pipelines/config/so/9029_output_esxi.conf.jinja diff --git a/salt/logstash/pipelines/config/9030_output_greensql.conf.jinja b/salt/logstash/pipelines/config/so/9030_output_greensql.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/9030_output_greensql.conf.jinja rename to salt/logstash/pipelines/config/so/9030_output_greensql.conf.jinja diff --git a/salt/logstash/pipelines/config/9031_output_iis.conf.jinja b/salt/logstash/pipelines/config/so/9031_output_iis.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/9031_output_iis.conf.jinja rename to salt/logstash/pipelines/config/so/9031_output_iis.conf.jinja diff --git a/salt/logstash/pipelines/config/9032_output_mcafee.conf.jinja b/salt/logstash/pipelines/config/so/9032_output_mcafee.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/9032_output_mcafee.conf.jinja rename to salt/logstash/pipelines/config/so/9032_output_mcafee.conf.jinja diff --git a/salt/logstash/pipelines/config/9033_output_snort.conf.jinja b/salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/9033_output_snort.conf.jinja rename to salt/logstash/pipelines/config/so/9033_output_snort.conf.jinja diff --git a/salt/logstash/pipelines/config/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/9034_output_syslog.conf.jinja rename to salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja diff --git a/salt/logstash/pipelines/config/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/9100_output_osquery.conf.jinja rename to salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja diff --git a/salt/logstash/pipelines/config/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/9200_output_firewall.conf.jinja rename to salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja diff --git a/salt/logstash/pipelines/config/9300_output_windows.conf.jinja b/salt/logstash/pipelines/config/so/9300_output_windows.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/9300_output_windows.conf.jinja rename to salt/logstash/pipelines/config/so/9300_output_windows.conf.jinja diff --git a/salt/logstash/pipelines/config/9301_output_dns_windows.conf.jinja b/salt/logstash/pipelines/config/so/9301_output_dns_windows.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/9301_output_dns_windows.conf.jinja rename to salt/logstash/pipelines/config/so/9301_output_dns_windows.conf.jinja diff --git a/salt/logstash/pipelines/config/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/9400_output_suricata.conf.jinja rename to salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja diff --git a/salt/logstash/pipelines/config/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/9500_output_beats.conf.jinja rename to salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja diff --git a/salt/logstash/pipelines/config/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/9600_output_ossec.conf.jinja rename to salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja diff --git a/salt/logstash/pipelines/config/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/9700_output_strelka.conf.jinja rename to salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja diff --git a/salt/logstash/pipelines/config/9997_output_helix.conf.jinja b/salt/logstash/pipelines/config/so/9997_output_helix.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/9997_output_helix.conf.jinja rename to salt/logstash/pipelines/config/so/9997_output_helix.conf.jinja diff --git a/salt/logstash/pipelines/config/9999_output_redis.conf.jinja b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja similarity index 100% rename from salt/logstash/pipelines/config/9999_output_redis.conf.jinja rename to salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja diff --git a/salt/logstash/files/custom/templates/Drop.Your.Custom.Templates.Here.conf b/salt/logstash/pipelines/templates/custom/Drop.Your.Custom.Templates.Here.conf similarity index 100% rename from salt/logstash/files/custom/templates/Drop.Your.Custom.Templates.Here.conf rename to salt/logstash/pipelines/templates/custom/Drop.Your.Custom.Templates.Here.conf diff --git a/salt/logstash/etc/beats-template.json b/salt/logstash/pipelines/templates/so/beats-template.json similarity index 100% rename from salt/logstash/etc/beats-template.json rename to salt/logstash/pipelines/templates/so/beats-template.json diff --git a/salt/logstash/etc/logstash-ossec-template.json b/salt/logstash/pipelines/templates/so/logstash-ossec-template.json similarity index 100% rename from salt/logstash/etc/logstash-ossec-template.json rename to salt/logstash/pipelines/templates/so/logstash-ossec-template.json diff --git a/salt/logstash/etc/logstash-strelka-template.json b/salt/logstash/pipelines/templates/so/logstash-strelka-template.json similarity index 100% rename from salt/logstash/etc/logstash-strelka-template.json rename to salt/logstash/pipelines/templates/so/logstash-strelka-template.json diff --git a/salt/logstash/etc/logstash-template.json b/salt/logstash/pipelines/templates/so/logstash-template.json similarity index 100% rename from salt/logstash/etc/logstash-template.json rename to salt/logstash/pipelines/templates/so/logstash-template.json From a3e48f031506e44448d2510f98619ddc7876126d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 26 Feb 2020 10:58:39 -0500 Subject: [PATCH 8/9] logstash cleanup and conflict resolution - https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/326 --- salt/logstash/init.sls | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 6a12557ec..b93ef1f43 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -78,14 +78,6 @@ lsetcdir: - group: 939 - makedirs: True -# templates not specific to pipeline -lscusttemplatedir: - file.directory: - - name: /opt/so/conf/logstash/custom/templates - - user: 931 - - group: 939 - - makedirs: True - {% for PL in PIPELINES %} ls_pipeline_{{PL}}: file.directory: @@ -141,21 +133,6 @@ lsetcsync: - template: jinja - exclude_pat: pipelines* -lssync: - file.recurse: - - name: /opt/so/conf/logstash/dynamic - - source: salt://logstash/files/dynamic - - user: 931 - - group: 939 - - template: jinja - -lscustsync: - file.recurse: - - name: /opt/so/conf/logstash/custom - - source: salt://logstash/files/custom - - user: 931 - - group: 939 - # Create the import directory importdir: file.directory: @@ -219,6 +196,5 @@ so-logstash: {%- endif %} - watch: - file: /opt/so/conf/logstash/etc - - file: /opt/so/conf/logstash/custom + - file: /opt/so/conf/logstash/pipelines #- file: /opt/so/conf/logstash/rulesets - - file: /opt/so/conf/logstash/dynamic From 847e6d2d139e2a9ab3367345e43ab74d2dec0350 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 26 Feb 2020 16:38:47 -0500 Subject: [PATCH 9/9] create pipelines directory --- salt/logstash/init.sls | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index b93ef1f43..b86ee0e83 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -78,6 +78,12 @@ lsetcdir: - group: 939 - makedirs: True +lspipelinedir: + file.directory: + - name: /opt/so/conf/logstash/pipelines + - user: 931 + - group: 939 + {% for PL in PIPELINES %} ls_pipeline_{{PL}}: file.directory: