From 2b34bdece97548af2fae4cbd6233f65f8e733c9b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 20 Feb 2020 16:47:40 -0500 Subject: [PATCH] logstash cleanup - https://github.com/Security-Onion-Solutions/securityonion-saltstack/issues/326 --- pillar/logstash/helix.sls | 40 ++- pillar/logstash/master.sls | 4 +- pillar/logstash/search.sls | 53 +++- .../conf/pipelines/eval/0800_input_eval.conf | 204 ------------- .../eval/1000_preprocess_log_elapsed.conf | 13 - .../eval/1001_preprocess_syslogng.conf | 33 -- .../pipelines/eval/1002_preprocess_json.conf | 18 -- .../eval/1004_preprocess_syslog_types.conf | 19 -- .../pipelines/eval/1026_preprocess_dhcp.conf | 140 --------- .../pipelines/eval/1029_preprocess_esxi.conf | 31 -- .../eval/1030_preprocess_greensql.conf | 21 -- .../pipelines/eval/1031_preprocess_iis.conf | 21 -- .../eval/1032_preprocess_mcafee.conf | 26 -- .../pipelines/eval/1033_preprocess_snort.conf | 181 ----------- .../eval/1034_preprocess_syslog.conf | 16 - .../pipelines/eval/2000_network_flow.conf | 59 ---- .../conf/pipelines/eval/6002_syslog.conf | 11 - .../pipelines/eval/6101_switch_brocade.conf | 33 -- .../eval/6200_firewall_fortinet.conf | 281 ------------------ .../pipelines/eval/6201_firewall_pfsense.conf | 56 ---- .../conf/pipelines/eval/6300_windows.conf | 161 ---------- .../conf/pipelines/eval/6301_dns_windows.conf | 49 --- .../conf/pipelines/eval/6400_suricata.conf | 92 ------ .../conf/pipelines/eval/6500_ossec.conf | 160 ---------- .../pipelines/eval/6501_ossec_sysmon.conf | 118 -------- .../pipelines/eval/6502_ossec_autoruns.conf | 43 --- .../eval/6600_winlogbeat_sysmon.conf | 23 -- .../conf/pipelines/eval/6700_winlogbeat.conf | 17 -- .../conf/pipelines/eval/7100_osquery_wel.conf | 23 -- .../conf/pipelines/eval/7200_strelka.conf | 8 - ...01_postprocess_common_ip_augmentation.conf | 58 ---- .../pipelines/eval/8007_postprocess_http.conf | 27 -- .../eval/8200_postprocess_tagging.conf | 63 ---- .../eval/8998_postprocess_log_elapsed.conf | 19 -- .../eval/8999_postprocess_rename_type.conf | 8 - .../eval/templates/9000_output_bro.conf | 32 -- .../eval/templates/9001_output_switch.conf | 27 -- .../eval/templates/9002_output_import.conf | 27 -- .../eval/templates/9004_output_flow.conf | 27 -- .../eval/templates/9026_output_dhcp.conf | 26 -- .../eval/templates/9029_output_esxi.conf | 25 -- .../eval/templates/9030_output_greensql.conf | 25 -- .../eval/templates/9031_output_iis.conf | 26 -- .../eval/templates/9032_output_mcafee.conf | 26 -- .../eval/templates/9033_output_snort.conf | 29 -- .../eval/templates/9034_output_syslog.conf | 28 -- .../eval/templates/9100_output_osquery.conf | 32 -- .../eval/templates/9200_output_firewall.conf | 29 -- .../eval/templates/9300_output_windows.conf | 27 -- .../templates/9301_output_dns_windows.conf | 27 -- .../eval/templates/9400_output_suricata.conf | 28 -- .../eval/templates/9500_output_beats.conf | 25 -- .../eval/templates/9600_output_ossec.conf | 29 -- .../helix/1033_preprocess_snort.conf | 181 ----------- ...01_postprocess_common_ip_augmentation.conf | 58 ---- .../pipelines/master/0010_input_hhbeats.conf | 40 --- .../search/1000_preprocess_log_elapsed.conf | 13 - .../search/1001_preprocess_syslogng.conf | 33 -- .../search/1002_preprocess_json.conf | 18 -- .../search/1004_preprocess_syslog_types.conf | 19 -- .../search/1026_preprocess_dhcp.conf | 140 --------- .../search/1029_preprocess_esxi.conf | 31 -- .../search/1030_preprocess_greensql.conf | 21 -- .../pipelines/search/1031_preprocess_iis.conf | 21 -- .../search/1032_preprocess_mcafee.conf | 26 -- .../search/1033_preprocess_snort.conf | 181 ----------- .../search/1034_preprocess_syslog.conf | 16 - .../pipelines/search/2000_network_flow.conf | 59 ---- .../conf/pipelines/search/6002_syslog.conf | 11 - .../pipelines/search/6101_switch_brocade.conf | 33 -- .../search/6200_firewall_fortinet.conf | 281 ------------------ .../search/6201_firewall_pfsense.conf | 56 ---- .../conf/pipelines/search/6300_windows.conf | 161 ---------- .../pipelines/search/6301_dns_windows.conf | 49 --- .../conf/pipelines/search/6400_suricata.conf | 92 ------ .../conf/pipelines/search/6500_ossec.conf | 160 ---------- .../pipelines/search/6501_ossec_sysmon.conf | 118 -------- .../pipelines/search/6502_ossec_autoruns.conf | 43 --- .../search/6600_winlogbeat_sysmon.conf | 23 -- .../pipelines/search/6700_winlogbeat.conf | 17 -- .../pipelines/search/7100_osquery_wel.conf | 23 -- .../conf/pipelines/search/7200_strelka.conf | 8 - ...01_postprocess_common_ip_augmentation.conf | 58 ---- .../search/8007_postprocess_http.conf | 27 -- .../search/8200_postprocess_tagging.conf | 63 ---- .../search/8998_postprocess_log_elapsed.conf | 19 -- .../search/8999_postprocess_rename_type.conf | 8 - .../search/templates/9000_output_bro.conf | 31 -- .../search/templates/9001_output_switch.conf | 27 -- .../search/templates/9002_output_import.conf | 27 -- .../search/templates/9004_output_flow.conf | 27 -- .../search/templates/9026_output_dhcp.conf | 26 -- .../search/templates/9029_output_esxi.conf | 25 -- .../templates/9030_output_greensql.conf | 25 -- .../search/templates/9031_output_iis.conf | 26 -- .../search/templates/9032_output_mcafee.conf | 26 -- .../search/templates/9033_output_snort.conf | 29 -- .../search/templates/9034_output_syslog.conf | 28 -- .../search/templates/9100_output_osquery.conf | 19 -- .../templates/9200_output_firewall.conf | 29 -- .../search/templates/9300_output_windows.conf | 27 -- .../templates/9301_output_dns_windows.conf | 27 -- .../templates/9400_output_suricata.conf | 27 -- .../search/templates/9500_output_beats.conf | 25 -- .../search/templates/9600_output_ossec.conf | 29 -- .../search/templates/9700_output_strelka.conf | 30 -- .../config}/0010_input_hhbeats.conf | 0 .../config/0900_input_redis.conf.jinja} | 0 .../config}/1100_preprocess_bro_conn.conf | 0 .../config}/1101_preprocess_bro_dhcp.conf | 0 .../config}/1102_preprocess_bro_dns.conf | 0 .../config}/1103_preprocess_bro_dpd.conf | 0 .../config}/1104_preprocess_bro_files.conf | 0 .../config}/1105_preprocess_bro_ftp.conf | 0 .../config}/1106_preprocess_bro_http.conf | 0 .../config}/1107_preprocess_bro_irc.conf | 0 .../config}/1108_preprocess_bro_kerberos.conf | 0 .../config}/1109_preprocess_bro_notice.conf | 0 .../config}/1110_preprocess_bro_rdp.conf | 0 .../1111_preprocess_bro_signatures.conf | 0 .../config}/1112_preprocess_bro_smtp.conf | 0 .../config}/1113_preprocess_bro_snmp.conf | 0 .../config}/1114_preprocess_bro_software.conf | 0 .../config}/1115_preprocess_bro_ssh.conf | 0 .../config}/1116_preprocess_bro_ssl.conf | 0 .../config}/1117_preprocess_bro_syslog.conf | 0 .../config}/1118_preprocess_bro_tunnel.conf | 0 .../config}/1119_preprocess_bro_weird.conf | 0 .../config}/1121_preprocess_bro_mysql.conf | 0 .../config}/1122_preprocess_bro_socks.conf | 0 .../config}/1123_preprocess_bro_x509.conf | 0 .../config}/1124_preprocess_bro_intel.conf | 0 .../config}/1125_preprocess_bro_modbus.conf | 0 .../config}/1126_preprocess_bro_sip.conf | 0 .../config}/1127_preprocess_bro_radius.conf | 0 .../config}/1128_preprocess_bro_pe.conf | 0 .../config}/1129_preprocess_bro_rfb.conf | 0 .../config}/1130_preprocess_bro_dnp3.conf | 0 .../1131_preprocess_bro_smb_files.conf | 0 .../1132_preprocess_bro_smb_mapping.conf | 0 .../config}/1133_preprocess_bro_ntlm.conf | 0 .../config}/1134_preprocess_bro_dce_rpc.conf | 0 .../config}/6000_bro.conf | 0 .../config}/6001_bro_import.conf | 0 .../config}/8000_postprocess_bro_cleanup.conf | 0 .../config}/8006_postprocess_dns.conf | 0 .../config/9000_output_bro.conf.jinja | 3 +- .../config/9100_output_osquery.conf.jinja | 21 +- .../config/9400_output_suricata.conf.jinja | 3 +- .../config/9700_ouptut_strelka.conf.jinja | 30 -- .../config/9700_output_strelka.conf.jinja} | 0 .../config/9997_output_helix.conf.jinja} | 0 .../config/9999_output_redis.conf.jinja} | 0 153 files changed, 100 insertions(+), 5198 deletions(-) delete mode 100644 salt/logstash/conf/pipelines/eval/0800_input_eval.conf delete mode 100644 salt/logstash/conf/pipelines/eval/1000_preprocess_log_elapsed.conf delete mode 100644 salt/logstash/conf/pipelines/eval/1001_preprocess_syslogng.conf delete mode 100644 salt/logstash/conf/pipelines/eval/1002_preprocess_json.conf delete mode 100644 salt/logstash/conf/pipelines/eval/1004_preprocess_syslog_types.conf delete mode 100644 salt/logstash/conf/pipelines/eval/1026_preprocess_dhcp.conf delete mode 100644 salt/logstash/conf/pipelines/eval/1029_preprocess_esxi.conf delete mode 100644 salt/logstash/conf/pipelines/eval/1030_preprocess_greensql.conf delete mode 100644 salt/logstash/conf/pipelines/eval/1031_preprocess_iis.conf delete mode 100644 salt/logstash/conf/pipelines/eval/1032_preprocess_mcafee.conf delete mode 100644 salt/logstash/conf/pipelines/eval/1033_preprocess_snort.conf delete mode 100644 salt/logstash/conf/pipelines/eval/1034_preprocess_syslog.conf delete mode 100644 salt/logstash/conf/pipelines/eval/2000_network_flow.conf delete mode 100644 salt/logstash/conf/pipelines/eval/6002_syslog.conf delete mode 100644 salt/logstash/conf/pipelines/eval/6101_switch_brocade.conf delete mode 100644 salt/logstash/conf/pipelines/eval/6200_firewall_fortinet.conf delete mode 100644 salt/logstash/conf/pipelines/eval/6201_firewall_pfsense.conf delete mode 100644 salt/logstash/conf/pipelines/eval/6300_windows.conf delete mode 100644 salt/logstash/conf/pipelines/eval/6301_dns_windows.conf delete mode 100644 salt/logstash/conf/pipelines/eval/6400_suricata.conf delete mode 100644 salt/logstash/conf/pipelines/eval/6500_ossec.conf delete mode 100644 salt/logstash/conf/pipelines/eval/6501_ossec_sysmon.conf delete mode 100644 salt/logstash/conf/pipelines/eval/6502_ossec_autoruns.conf delete mode 100644 salt/logstash/conf/pipelines/eval/6600_winlogbeat_sysmon.conf delete mode 100644 salt/logstash/conf/pipelines/eval/6700_winlogbeat.conf delete mode 100644 salt/logstash/conf/pipelines/eval/7100_osquery_wel.conf delete mode 100644 salt/logstash/conf/pipelines/eval/7200_strelka.conf delete mode 100644 salt/logstash/conf/pipelines/eval/8001_postprocess_common_ip_augmentation.conf delete mode 100644 salt/logstash/conf/pipelines/eval/8007_postprocess_http.conf delete mode 100644 salt/logstash/conf/pipelines/eval/8200_postprocess_tagging.conf delete mode 100644 salt/logstash/conf/pipelines/eval/8998_postprocess_log_elapsed.conf delete mode 100644 salt/logstash/conf/pipelines/eval/8999_postprocess_rename_type.conf delete mode 100644 salt/logstash/conf/pipelines/eval/templates/9000_output_bro.conf delete mode 100644 salt/logstash/conf/pipelines/eval/templates/9001_output_switch.conf delete mode 100644 salt/logstash/conf/pipelines/eval/templates/9002_output_import.conf delete mode 100644 salt/logstash/conf/pipelines/eval/templates/9004_output_flow.conf delete mode 100644 salt/logstash/conf/pipelines/eval/templates/9026_output_dhcp.conf delete mode 100644 salt/logstash/conf/pipelines/eval/templates/9029_output_esxi.conf delete mode 100644 salt/logstash/conf/pipelines/eval/templates/9030_output_greensql.conf delete mode 100644 salt/logstash/conf/pipelines/eval/templates/9031_output_iis.conf delete mode 100644 salt/logstash/conf/pipelines/eval/templates/9032_output_mcafee.conf delete mode 100644 salt/logstash/conf/pipelines/eval/templates/9033_output_snort.conf delete mode 100644 salt/logstash/conf/pipelines/eval/templates/9034_output_syslog.conf delete mode 100644 salt/logstash/conf/pipelines/eval/templates/9100_output_osquery.conf delete mode 100644 salt/logstash/conf/pipelines/eval/templates/9200_output_firewall.conf delete mode 100644 salt/logstash/conf/pipelines/eval/templates/9300_output_windows.conf delete mode 100644 salt/logstash/conf/pipelines/eval/templates/9301_output_dns_windows.conf delete mode 100644 salt/logstash/conf/pipelines/eval/templates/9400_output_suricata.conf delete mode 100644 salt/logstash/conf/pipelines/eval/templates/9500_output_beats.conf delete mode 100644 salt/logstash/conf/pipelines/eval/templates/9600_output_ossec.conf delete mode 100644 salt/logstash/conf/pipelines/helix/1033_preprocess_snort.conf delete mode 100644 salt/logstash/conf/pipelines/helix/8001_postprocess_common_ip_augmentation.conf delete mode 100644 salt/logstash/conf/pipelines/master/0010_input_hhbeats.conf delete mode 100644 salt/logstash/conf/pipelines/search/1000_preprocess_log_elapsed.conf delete mode 100644 salt/logstash/conf/pipelines/search/1001_preprocess_syslogng.conf delete mode 100644 salt/logstash/conf/pipelines/search/1002_preprocess_json.conf delete mode 100644 salt/logstash/conf/pipelines/search/1004_preprocess_syslog_types.conf delete mode 100644 salt/logstash/conf/pipelines/search/1026_preprocess_dhcp.conf delete mode 100644 salt/logstash/conf/pipelines/search/1029_preprocess_esxi.conf delete mode 100644 salt/logstash/conf/pipelines/search/1030_preprocess_greensql.conf delete mode 100644 salt/logstash/conf/pipelines/search/1031_preprocess_iis.conf delete mode 100644 salt/logstash/conf/pipelines/search/1032_preprocess_mcafee.conf delete mode 100644 salt/logstash/conf/pipelines/search/1033_preprocess_snort.conf delete mode 100644 salt/logstash/conf/pipelines/search/1034_preprocess_syslog.conf delete mode 100644 salt/logstash/conf/pipelines/search/2000_network_flow.conf delete mode 100644 salt/logstash/conf/pipelines/search/6002_syslog.conf delete mode 100644 salt/logstash/conf/pipelines/search/6101_switch_brocade.conf delete mode 100644 salt/logstash/conf/pipelines/search/6200_firewall_fortinet.conf delete mode 100644 salt/logstash/conf/pipelines/search/6201_firewall_pfsense.conf delete mode 100644 salt/logstash/conf/pipelines/search/6300_windows.conf delete mode 100644 salt/logstash/conf/pipelines/search/6301_dns_windows.conf delete mode 100644 salt/logstash/conf/pipelines/search/6400_suricata.conf delete mode 100644 salt/logstash/conf/pipelines/search/6500_ossec.conf delete mode 100644 salt/logstash/conf/pipelines/search/6501_ossec_sysmon.conf delete mode 100644 salt/logstash/conf/pipelines/search/6502_ossec_autoruns.conf delete mode 100644 salt/logstash/conf/pipelines/search/6600_winlogbeat_sysmon.conf delete mode 100644 salt/logstash/conf/pipelines/search/6700_winlogbeat.conf delete mode 100644 salt/logstash/conf/pipelines/search/7100_osquery_wel.conf delete mode 100644 salt/logstash/conf/pipelines/search/7200_strelka.conf delete mode 100644 salt/logstash/conf/pipelines/search/8001_postprocess_common_ip_augmentation.conf delete mode 100644 salt/logstash/conf/pipelines/search/8007_postprocess_http.conf delete mode 100644 salt/logstash/conf/pipelines/search/8200_postprocess_tagging.conf delete mode 100644 salt/logstash/conf/pipelines/search/8998_postprocess_log_elapsed.conf delete mode 100644 salt/logstash/conf/pipelines/search/8999_postprocess_rename_type.conf delete mode 100644 salt/logstash/conf/pipelines/search/templates/9000_output_bro.conf delete mode 100644 salt/logstash/conf/pipelines/search/templates/9001_output_switch.conf delete mode 100644 salt/logstash/conf/pipelines/search/templates/9002_output_import.conf delete mode 100644 salt/logstash/conf/pipelines/search/templates/9004_output_flow.conf delete mode 100644 salt/logstash/conf/pipelines/search/templates/9026_output_dhcp.conf delete mode 100644 salt/logstash/conf/pipelines/search/templates/9029_output_esxi.conf delete mode 100644 salt/logstash/conf/pipelines/search/templates/9030_output_greensql.conf delete mode 100644 salt/logstash/conf/pipelines/search/templates/9031_output_iis.conf delete mode 100644 salt/logstash/conf/pipelines/search/templates/9032_output_mcafee.conf delete mode 100644 salt/logstash/conf/pipelines/search/templates/9033_output_snort.conf delete mode 100644 salt/logstash/conf/pipelines/search/templates/9034_output_syslog.conf delete mode 100644 salt/logstash/conf/pipelines/search/templates/9100_output_osquery.conf delete mode 100644 salt/logstash/conf/pipelines/search/templates/9200_output_firewall.conf delete mode 100644 salt/logstash/conf/pipelines/search/templates/9300_output_windows.conf delete mode 100644 salt/logstash/conf/pipelines/search/templates/9301_output_dns_windows.conf delete mode 100644 salt/logstash/conf/pipelines/search/templates/9400_output_suricata.conf delete mode 100644 salt/logstash/conf/pipelines/search/templates/9500_output_beats.conf delete mode 100644 salt/logstash/conf/pipelines/search/templates/9600_output_ossec.conf delete mode 100644 salt/logstash/conf/pipelines/search/templates/9700_output_strelka.conf rename salt/logstash/{conf/pipelines/helix => pipelines/config}/0010_input_hhbeats.conf (100%) rename salt/logstash/{conf/pipelines/search/templates/0900_input_redis.conf => pipelines/config/0900_input_redis.conf.jinja} (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1100_preprocess_bro_conn.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1101_preprocess_bro_dhcp.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1102_preprocess_bro_dns.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1103_preprocess_bro_dpd.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1104_preprocess_bro_files.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1105_preprocess_bro_ftp.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1106_preprocess_bro_http.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1107_preprocess_bro_irc.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1108_preprocess_bro_kerberos.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1109_preprocess_bro_notice.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1110_preprocess_bro_rdp.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1111_preprocess_bro_signatures.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1112_preprocess_bro_smtp.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1113_preprocess_bro_snmp.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1114_preprocess_bro_software.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1115_preprocess_bro_ssh.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1116_preprocess_bro_ssl.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1117_preprocess_bro_syslog.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1118_preprocess_bro_tunnel.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1119_preprocess_bro_weird.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1121_preprocess_bro_mysql.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1122_preprocess_bro_socks.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1123_preprocess_bro_x509.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1124_preprocess_bro_intel.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1125_preprocess_bro_modbus.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1126_preprocess_bro_sip.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1127_preprocess_bro_radius.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1128_preprocess_bro_pe.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1129_preprocess_bro_rfb.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1130_preprocess_bro_dnp3.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1131_preprocess_bro_smb_files.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1132_preprocess_bro_smb_mapping.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1133_preprocess_bro_ntlm.conf (100%) rename salt/logstash/{conf/pipelines/helix => pipelines/config}/1134_preprocess_bro_dce_rpc.conf (100%) rename salt/logstash/{conf/pipelines => pipelines/config}/6000_bro.conf (100%) rename salt/logstash/{conf/pipelines => pipelines/config}/6001_bro_import.conf (100%) rename salt/logstash/{conf/pipelines => pipelines/config}/8000_postprocess_bro_cleanup.conf (100%) rename salt/logstash/{conf/pipelines => pipelines/config}/8006_postprocess_dns.conf (100%) delete mode 100644 salt/logstash/pipelines/config/9700_ouptut_strelka.conf.jinja rename salt/logstash/{conf/pipelines/eval/templates/9700_ouptut_strelka.conf => pipelines/config/9700_output_strelka.conf.jinja} (100%) rename salt/logstash/{conf/pipelines/helix/templates/9997_output_helix.conf => pipelines/config/9997_output_helix.conf.jinja} (100%) rename salt/logstash/{conf/pipelines/master/templates/9999_output_redis.conf => pipelines/config/9999_output_redis.conf.jinja} (100%) diff --git a/pillar/logstash/helix.sls b/pillar/logstash/helix.sls index e396a7aad..cdde880e1 100644 --- a/pillar/logstash/helix.sls +++ b/pillar/logstash/helix.sls @@ -1,4 +1,42 @@ logstash: pipelines: helix: - config: "/usr/share/logstash/pipelines/helix/*.conf" + config: + - 0010_input_hhbeats.conf + - 1033_preprocess_snort.conf + - 1100_preprocess_bro_conn.conf + - 1101_preprocess_bro_dhcp.conf + - 1102_preprocess_bro_dns.conf + - 1103_preprocess_bro_dpd.conf + - 1104_preprocess_bro_files.conf + - 1105_preprocess_bro_ftp.conf + - 1106_preprocess_bro_http.conf + - 1107_preprocess_bro_irc.conf + - 1108_preprocess_bro_kerberos.conf + - 1109_preprocess_bro_notice.conf + - 1110_preprocess_bro_rdp.conf + - 1111_preprocess_bro_signatures.conf + - 1112_preprocess_bro_smtp.conf + - 1113_preprocess_bro_snmp.conf + - 1114_preprocess_bro_software.conf + - 1115_preprocess_bro_ssh.conf + - 1116_preprocess_bro_ssl.conf + - 1117_preprocess_bro_syslog.conf + - 1118_preprocess_bro_tunnel.conf + - 1119_preprocess_bro_weird.conf + - 1121_preprocess_bro_mysql.conf + - 1122_preprocess_bro_socks.conf + - 1123_preprocess_bro_x509.conf + - 1124_preprocess_bro_intel.conf + - 1125_preprocess_bro_modbus.conf + - 1126_preprocess_bro_sip.conf + - 1127_preprocess_bro_radius.conf + - 1128_preprocess_bro_pe.conf + - 1129_preprocess_bro_rfb.conf + - 1130_preprocess_bro_dnp3.conf + - 1131_preprocess_bro_smb_files.conf + - 1132_preprocess_bro_smb_mapping.conf + - 1133_preprocess_bro_ntlm.conf + - 1134_preprocess_bro_dce_rpc.conf + - 8001_postprocess_common_ip_augmentation.conf + - 9997_output_helix.conf.jinja diff --git a/pillar/logstash/master.sls b/pillar/logstash/master.sls index 3be98f6b9..f3ad90d7e 100644 --- a/pillar/logstash/master.sls +++ b/pillar/logstash/master.sls @@ -1,4 +1,6 @@ logstash: pipelines: master: - config: "/usr/share/logstash/pipelines/master/*.conf" + config: + - 0010_input_hhbeats.conf + - 9999_output_redis.conf.jinja diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 0eca8571f..3db36320f 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -1,4 +1,55 @@ logstash: pipelines: search: - config: "/usr/share/logstash/pipelines/search/*.conf" + config: + - 1000_preprocess_log_elapsed.conf + - 1001_preprocess_syslogng.conf + - 1002_preprocess_json.conf + - 1004_preprocess_syslog_types.conf + - 1026_preprocess_dhcp.conf + - 1029_preprocess_esxi.conf + - 1030_preprocess_greensql.conf + - 1031_preprocess_iis.conf + - 1032_preprocess_mcafee.conf + - 1033_preprocess_snort.conf + - 1034_preprocess_syslog.conf + - 2000_network_flow.conf + - 6002_syslog.conf + - 6101_switch_brocade.conf + - 6200_firewall_fortinet.conf + - 6201_firewall_pfsense.conf + - 6300_windows.conf + - 6301_dns_windows.conf + - 6400_suricata.conf + - 6500_ossec.conf + - 6501_ossec_sysmon.conf + - 6502_ossec_autoruns.conf + - 6600_winlogbeat_sysmon.conf + - 6700_winlogbeat.conf + - 7100_osquery_wel.conf + - 7200_strelka.conf + - 8001_postprocess_common_ip_augmentation.conf + - 8007_postprocess_http.conf + - 8200_postprocess_tagging.conf + - 8998_postprocess_log_elapsed.conf + - 8999_postprocess_rename_type.conf + - 0900_input_redis.conf.jinja + - 9000_output_bro.conf.jinja + - 9001_output_switch.conf.jinja + - 9002_output_import.conf.jinja + - 9004_output_flow.conf.jinja + - 9026_output_dhcp.conf.jinja + - 9029_output_esxi.conf.jinja + - 9030_output_greensql.conf.jinja + - 9031_output_iis.conf.jinja + - 9032_output_mcafee.conf.jinja + - 9033_output_snort.conf.jinja + - 9034_output_syslog.conf.jinja + - 9100_output_osquery.conf.jinja + - 9200_output_firewall.conf.jinja + - 9300_output_windows.conf.jinja + - 9301_output_dns_windows.conf.jinja + - 9400_output_suricata.conf.jinja + - 9500_output_beats.conf.jinja + - 9600_output_ossec.conf.jinja + - 9700_output_strelka.conf.jinja diff --git a/salt/logstash/conf/pipelines/eval/0800_input_eval.conf b/salt/logstash/conf/pipelines/eval/0800_input_eval.conf deleted file mode 100644 index b499c3b0f..000000000 --- a/salt/logstash/conf/pipelines/eval/0800_input_eval.conf +++ /dev/null @@ -1,204 +0,0 @@ -# Updated by: Mike Reeves -# Last Update: 11/1/2018 - -input { - file { - path => "/suricata/eve.json" - type => "ids" - add_field => { "engine" => "suricata" } - } - file { - path => "/nsm/zeek/logs/current/conn*.log" - type => "bro_conn" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/dce_rpc*.log" - type => "bro_dce_rpc" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/dhcp*.log" - type => "bro_dhcp" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/dnp3*.log" - type => "bro_dnp3" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/dns*.log" - type => "bro_dns" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/dpd*.log" - type => "bro_dpd" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/files*.log" - type => "bro_files" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/ftp*.log" - type => "bro_ftp" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/http*.log" - type => "bro_http" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/intel*.log" - type => "bro_intel" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/irc*.log" - type => "bro_irc" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/kerberos*.log" - type => "bro_kerberos" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/modbus*.log" - type => "bro_modbus" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/mysql*.log" - type => "bro_mysql" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/notice*.log" - type => "bro_notice" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/ntlm*.log" - type => "bro_ntlm" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/pe*.log" - type => "bro_pe" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/radius*.log" - type => "bro_radius" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/rdp*.log" - type => "bro_rdp" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/rfb*.log" - type => "bro_rfb" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/signatures*.log" - type => "bro_signatures" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/sip*.log" - type => "bro_sip" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/smb_files*.log" - type => "bro_smb_files" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/smb_mapping*.log" - type => "bro_smb_mapping" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/smtp*.log" - type => "bro_smtp" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/snmp*.log" - type => "bro_snmp" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/socks*.log" - type => "bro_socks" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/software*.log" - type => "bro_software" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/ssh*.log" - type => "bro_ssh" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/ssl*.log" - type => "bro_ssl" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/syslog*.log" - type => "bro_syslog" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/tunnel*.log" - type => "bro_tunnels" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/weird*.log" - type => "bro_weird" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/x509*.log" - type => "bro_x509" - tags => ["bro"] - } - file { - path => "/wazuh/alerts/alerts.json" - type => "ossec" - } - file { - path => "/wazuh/archives/archives.json" - type => "ossec_archive" - } - file { - path => "/osquery/logs/result.log" - type => "osquery" - tags => ["osquery"] - } - file { - path => "/strelka/strelka.log" - type => "strelka" - } -} -filter { - if "import" in [tags] { - mutate { - #add_tag => [ "conf_file_0007"] - } - } -} diff --git a/salt/logstash/conf/pipelines/eval/1000_preprocess_log_elapsed.conf b/salt/logstash/conf/pipelines/eval/1000_preprocess_log_elapsed.conf deleted file mode 100644 index d098eb11a..000000000 --- a/salt/logstash/conf/pipelines/eval/1000_preprocess_log_elapsed.conf +++ /dev/null @@ -1,13 +0,0 @@ -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - ruby { - code => "event.set('task_start', Time.now.to_f)" - } - mutate { - #add_tag => [ "conf_file_1000"] - } -} diff --git a/salt/logstash/conf/pipelines/eval/1001_preprocess_syslogng.conf b/salt/logstash/conf/pipelines/eval/1001_preprocess_syslogng.conf deleted file mode 100644 index 84bce8802..000000000 --- a/salt/logstash/conf/pipelines/eval/1001_preprocess_syslogng.conf +++ /dev/null @@ -1,33 +0,0 @@ -# Updated by: Doug Burks and Wes Lambert -# Last Update: 10/30/2018 - -filter { - if "syslogng" in [tags] { - mutate { - rename => { "MESSAGE" => "message" } - rename => { "PROGRAM" => "type" } - rename => { "FACILITY" => "syslog-facility" } - rename => { "FILE_NAME" => "syslog-file_name" } - rename => { "HOST" => "syslog-host" } - rename => { "HOST_FROM" => "syslog-host_from" } - rename => { "LEGACY_MSGHDR" => "syslog-legacy_msghdr" } - rename => { "PID" => "syslog-pid" } - rename => { "PRIORITY" => "syslog-priority" } - rename => { "SOURCEIP" => "syslog-sourceip" } - rename => { "TAGS" => "syslog-tags" } - lowercase => [ "syslog-host_from" ] - remove_field => [ "ISODATE" ] - remove_field => [ "SEQNUM" ] - #add_tag => [ "conf_file_1001"] - } - if "bro_" in [type] { - mutate { - add_tag => [ "bro" ] - } - } else if [type] !~ /ossec.*|snort/ and "firewall" not in [tags] { - mutate { - add_tag => [ "syslog" ] - } - } - } -} diff --git a/salt/logstash/conf/pipelines/eval/1002_preprocess_json.conf b/salt/logstash/conf/pipelines/eval/1002_preprocess_json.conf deleted file mode 100644 index ea7c677da..000000000 --- a/salt/logstash/conf/pipelines/eval/1002_preprocess_json.conf +++ /dev/null @@ -1,18 +0,0 @@ -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if "json" in [tags]{ - json { - source => "message" - } - mutate { - remove_tag => [ "json" ] - } - mutate { - #add_tag => [ "conf_file_1002"] - } - } -} diff --git a/salt/logstash/conf/pipelines/eval/1004_preprocess_syslog_types.conf b/salt/logstash/conf/pipelines/eval/1004_preprocess_syslog_types.conf deleted file mode 100644 index 243abcc15..000000000 --- a/salt/logstash/conf/pipelines/eval/1004_preprocess_syslog_types.conf +++ /dev/null @@ -1,19 +0,0 @@ -filter { - if "syslog" in [tags] { - if [host] == "172.16.1.1" { - mutate { - add_field => { "type" => "fortinet" } - add_tag => [ "firewall" ] - } - } - if [host] == "10.0.0.101" { - mutate { - add_field => { "type" => "brocade" } - add_tag => [ "switch" ] - } - } - mutate { - #add_tag => [ "conf_file_1004"] - } - } -} diff --git a/salt/logstash/conf/pipelines/eval/1026_preprocess_dhcp.conf b/salt/logstash/conf/pipelines/eval/1026_preprocess_dhcp.conf deleted file mode 100644 index 2f893cf7a..000000000 --- a/salt/logstash/conf/pipelines/eval/1026_preprocess_dhcp.conf +++ /dev/null @@ -1,140 +0,0 @@ -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolutions.com -# Last Update: 12/9/2016 -# This conf file is based on accepting logs for DHCP. It is currently based on Windows DHCP only. -filter { - if [type] == "dhcp" { - mutate { - add_field => { "Hostname" => "%{host}" } - } - mutate { - strip => "message" - } - # This is the initial parsing of the log - grok { - # Server 2008+ - match => { "message" => "%{DATA:id},%{DATE_US:date},(?