diff --git a/pillar/logstash/helix.sls b/pillar/logstash/helix.sls index e396a7aad..cdde880e1 100644 --- a/pillar/logstash/helix.sls +++ b/pillar/logstash/helix.sls @@ -1,4 +1,42 @@ logstash: pipelines: helix: - config: "/usr/share/logstash/pipelines/helix/*.conf" + config: + - 0010_input_hhbeats.conf + - 1033_preprocess_snort.conf + - 1100_preprocess_bro_conn.conf + - 1101_preprocess_bro_dhcp.conf + - 1102_preprocess_bro_dns.conf + - 1103_preprocess_bro_dpd.conf + - 1104_preprocess_bro_files.conf + - 1105_preprocess_bro_ftp.conf + - 1106_preprocess_bro_http.conf + - 1107_preprocess_bro_irc.conf + - 1108_preprocess_bro_kerberos.conf + - 1109_preprocess_bro_notice.conf + - 1110_preprocess_bro_rdp.conf + - 1111_preprocess_bro_signatures.conf + - 1112_preprocess_bro_smtp.conf + - 1113_preprocess_bro_snmp.conf + - 1114_preprocess_bro_software.conf + - 1115_preprocess_bro_ssh.conf + - 1116_preprocess_bro_ssl.conf + - 1117_preprocess_bro_syslog.conf + - 1118_preprocess_bro_tunnel.conf + - 1119_preprocess_bro_weird.conf + - 1121_preprocess_bro_mysql.conf + - 1122_preprocess_bro_socks.conf + - 1123_preprocess_bro_x509.conf + - 1124_preprocess_bro_intel.conf + - 1125_preprocess_bro_modbus.conf + - 1126_preprocess_bro_sip.conf + - 1127_preprocess_bro_radius.conf + - 1128_preprocess_bro_pe.conf + - 1129_preprocess_bro_rfb.conf + - 1130_preprocess_bro_dnp3.conf + - 1131_preprocess_bro_smb_files.conf + - 1132_preprocess_bro_smb_mapping.conf + - 1133_preprocess_bro_ntlm.conf + - 1134_preprocess_bro_dce_rpc.conf + - 8001_postprocess_common_ip_augmentation.conf + - 9997_output_helix.conf.jinja diff --git a/pillar/logstash/master.sls b/pillar/logstash/master.sls index 3be98f6b9..f3ad90d7e 100644 --- a/pillar/logstash/master.sls +++ b/pillar/logstash/master.sls @@ -1,4 +1,6 @@ logstash: pipelines: master: - config: "/usr/share/logstash/pipelines/master/*.conf" + config: + - 0010_input_hhbeats.conf + - 9999_output_redis.conf.jinja diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 0eca8571f..3db36320f 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -1,4 +1,55 @@ logstash: pipelines: search: - config: "/usr/share/logstash/pipelines/search/*.conf" + config: + - 1000_preprocess_log_elapsed.conf + - 1001_preprocess_syslogng.conf + - 1002_preprocess_json.conf + - 1004_preprocess_syslog_types.conf + - 1026_preprocess_dhcp.conf + - 1029_preprocess_esxi.conf + - 1030_preprocess_greensql.conf + - 1031_preprocess_iis.conf + - 1032_preprocess_mcafee.conf + - 1033_preprocess_snort.conf + - 1034_preprocess_syslog.conf + - 2000_network_flow.conf + - 6002_syslog.conf + - 6101_switch_brocade.conf + - 6200_firewall_fortinet.conf + - 6201_firewall_pfsense.conf + - 6300_windows.conf + - 6301_dns_windows.conf + - 6400_suricata.conf + - 6500_ossec.conf + - 6501_ossec_sysmon.conf + - 6502_ossec_autoruns.conf + - 6600_winlogbeat_sysmon.conf + - 6700_winlogbeat.conf + - 7100_osquery_wel.conf + - 7200_strelka.conf + - 8001_postprocess_common_ip_augmentation.conf + - 8007_postprocess_http.conf + - 8200_postprocess_tagging.conf + - 8998_postprocess_log_elapsed.conf + - 8999_postprocess_rename_type.conf + - 0900_input_redis.conf.jinja + - 9000_output_bro.conf.jinja + - 9001_output_switch.conf.jinja + - 9002_output_import.conf.jinja + - 9004_output_flow.conf.jinja + - 9026_output_dhcp.conf.jinja + - 9029_output_esxi.conf.jinja + - 9030_output_greensql.conf.jinja + - 9031_output_iis.conf.jinja + - 9032_output_mcafee.conf.jinja + - 9033_output_snort.conf.jinja + - 9034_output_syslog.conf.jinja + - 9100_output_osquery.conf.jinja + - 9200_output_firewall.conf.jinja + - 9300_output_windows.conf.jinja + - 9301_output_dns_windows.conf.jinja + - 9400_output_suricata.conf.jinja + - 9500_output_beats.conf.jinja + - 9600_output_ossec.conf.jinja + - 9700_output_strelka.conf.jinja diff --git a/salt/logstash/conf/pipelines/eval/0800_input_eval.conf b/salt/logstash/conf/pipelines/eval/0800_input_eval.conf deleted file mode 100644 index b499c3b0f..000000000 --- a/salt/logstash/conf/pipelines/eval/0800_input_eval.conf +++ /dev/null @@ -1,204 +0,0 @@ -# Updated by: Mike Reeves -# Last Update: 11/1/2018 - -input { - file { - path => "/suricata/eve.json" - type => "ids" - add_field => { "engine" => "suricata" } - } - file { - path => "/nsm/zeek/logs/current/conn*.log" - type => "bro_conn" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/dce_rpc*.log" - type => "bro_dce_rpc" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/dhcp*.log" - type => "bro_dhcp" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/dnp3*.log" - type => "bro_dnp3" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/dns*.log" - type => "bro_dns" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/dpd*.log" - type => "bro_dpd" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/files*.log" - type => "bro_files" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/ftp*.log" - type => "bro_ftp" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/http*.log" - type => "bro_http" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/intel*.log" - type => "bro_intel" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/irc*.log" - type => "bro_irc" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/kerberos*.log" - type => "bro_kerberos" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/modbus*.log" - type => "bro_modbus" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/mysql*.log" - type => "bro_mysql" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/notice*.log" - type => "bro_notice" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/ntlm*.log" - type => "bro_ntlm" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/pe*.log" - type => "bro_pe" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/radius*.log" - type => "bro_radius" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/rdp*.log" - type => "bro_rdp" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/rfb*.log" - type => "bro_rfb" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/signatures*.log" - type => "bro_signatures" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/sip*.log" - type => "bro_sip" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/smb_files*.log" - type => "bro_smb_files" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/smb_mapping*.log" - type => "bro_smb_mapping" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/smtp*.log" - type => "bro_smtp" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/snmp*.log" - type => "bro_snmp" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/socks*.log" - type => "bro_socks" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/software*.log" - type => "bro_software" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/ssh*.log" - type => "bro_ssh" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/ssl*.log" - type => "bro_ssl" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/syslog*.log" - type => "bro_syslog" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/tunnel*.log" - type => "bro_tunnels" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/weird*.log" - type => "bro_weird" - tags => ["bro"] - } - file { - path => "/nsm/zeek/logs/current/x509*.log" - type => "bro_x509" - tags => ["bro"] - } - file { - path => "/wazuh/alerts/alerts.json" - type => "ossec" - } - file { - path => "/wazuh/archives/archives.json" - type => "ossec_archive" - } - file { - path => "/osquery/logs/result.log" - type => "osquery" - tags => ["osquery"] - } - file { - path => "/strelka/strelka.log" - type => "strelka" - } -} -filter { - if "import" in [tags] { - mutate { - #add_tag => [ "conf_file_0007"] - } - } -} diff --git a/salt/logstash/conf/pipelines/eval/1000_preprocess_log_elapsed.conf b/salt/logstash/conf/pipelines/eval/1000_preprocess_log_elapsed.conf deleted file mode 100644 index d098eb11a..000000000 --- a/salt/logstash/conf/pipelines/eval/1000_preprocess_log_elapsed.conf +++ /dev/null @@ -1,13 +0,0 @@ -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - ruby { - code => "event.set('task_start', Time.now.to_f)" - } - mutate { - #add_tag => [ "conf_file_1000"] - } -} diff --git a/salt/logstash/conf/pipelines/eval/1001_preprocess_syslogng.conf b/salt/logstash/conf/pipelines/eval/1001_preprocess_syslogng.conf deleted file mode 100644 index 84bce8802..000000000 --- a/salt/logstash/conf/pipelines/eval/1001_preprocess_syslogng.conf +++ /dev/null @@ -1,33 +0,0 @@ -# Updated by: Doug Burks and Wes Lambert -# Last Update: 10/30/2018 - -filter { - if "syslogng" in [tags] { - mutate { - rename => { "MESSAGE" => "message" } - rename => { "PROGRAM" => "type" } - rename => { "FACILITY" => "syslog-facility" } - rename => { "FILE_NAME" => "syslog-file_name" } - rename => { "HOST" => "syslog-host" } - rename => { "HOST_FROM" => "syslog-host_from" } - rename => { "LEGACY_MSGHDR" => "syslog-legacy_msghdr" } - rename => { "PID" => "syslog-pid" } - rename => { "PRIORITY" => "syslog-priority" } - rename => { "SOURCEIP" => "syslog-sourceip" } - rename => { "TAGS" => "syslog-tags" } - lowercase => [ "syslog-host_from" ] - remove_field => [ "ISODATE" ] - remove_field => [ "SEQNUM" ] - #add_tag => [ "conf_file_1001"] - } - if "bro_" in [type] { - mutate { - add_tag => [ "bro" ] - } - } else if [type] !~ /ossec.*|snort/ and "firewall" not in [tags] { - mutate { - add_tag => [ "syslog" ] - } - } - } -} diff --git a/salt/logstash/conf/pipelines/eval/1002_preprocess_json.conf b/salt/logstash/conf/pipelines/eval/1002_preprocess_json.conf deleted file mode 100644 index ea7c677da..000000000 --- a/salt/logstash/conf/pipelines/eval/1002_preprocess_json.conf +++ /dev/null @@ -1,18 +0,0 @@ -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolution.com -# Last Update: 12/9/2016 - -filter { - if "json" in [tags]{ - json { - source => "message" - } - mutate { - remove_tag => [ "json" ] - } - mutate { - #add_tag => [ "conf_file_1002"] - } - } -} diff --git a/salt/logstash/conf/pipelines/eval/1004_preprocess_syslog_types.conf b/salt/logstash/conf/pipelines/eval/1004_preprocess_syslog_types.conf deleted file mode 100644 index 243abcc15..000000000 --- a/salt/logstash/conf/pipelines/eval/1004_preprocess_syslog_types.conf +++ /dev/null @@ -1,19 +0,0 @@ -filter { - if "syslog" in [tags] { - if [host] == "172.16.1.1" { - mutate { - add_field => { "type" => "fortinet" } - add_tag => [ "firewall" ] - } - } - if [host] == "10.0.0.101" { - mutate { - add_field => { "type" => "brocade" } - add_tag => [ "switch" ] - } - } - mutate { - #add_tag => [ "conf_file_1004"] - } - } -} diff --git a/salt/logstash/conf/pipelines/eval/1026_preprocess_dhcp.conf b/salt/logstash/conf/pipelines/eval/1026_preprocess_dhcp.conf deleted file mode 100644 index 2f893cf7a..000000000 --- a/salt/logstash/conf/pipelines/eval/1026_preprocess_dhcp.conf +++ /dev/null @@ -1,140 +0,0 @@ -# Author: Justin Henderson -# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics -# Email: justin@hasecuritysolutions.com -# Last Update: 12/9/2016 -# This conf file is based on accepting logs for DHCP. It is currently based on Windows DHCP only. -filter { - if [type] == "dhcp" { - mutate { - add_field => { "Hostname" => "%{host}" } - } - mutate { - strip => "message" - } - # This is the initial parsing of the log - grok { - # Server 2008+ - match => { "message" => "%{DATA:id},%{DATE_US:date},(?