From d94065fa00d95b244796f607c5d6b653befb0bf2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 30 Jan 2020 17:09:06 -0500 Subject: [PATCH] evalfix --- salt/filebeat/init.sls | 12 +- .../pipelines/eval/0010_input_hhbeats.conf | 40 +++ .../eval/1000_preprocess_log_elapsed.conf | 13 + .../eval/1001_preprocess_syslogng.conf | 33 ++ .../pipelines/eval/1002_preprocess_json.conf | 18 ++ .../eval/1004_preprocess_syslog_types.conf | 19 ++ .../pipelines/eval/1026_preprocess_dhcp.conf | 140 +++++++++ .../pipelines/eval/1029_preprocess_esxi.conf | 31 ++ .../eval/1030_preprocess_greensql.conf | 21 ++ .../pipelines/eval/1031_preprocess_iis.conf | 21 ++ .../eval/1032_preprocess_mcafee.conf | 26 ++ .../pipelines/eval/1033_preprocess_snort.conf | 181 +++++++++++ .../eval/1034_preprocess_syslog.conf | 16 + .../pipelines/eval/2000_network_flow.conf | 59 ++++ .../conf/pipelines/eval/6002_syslog.conf | 11 + .../pipelines/eval/6101_switch_brocade.conf | 33 ++ .../eval/6200_firewall_fortinet.conf | 281 ++++++++++++++++++ .../pipelines/eval/6201_firewall_pfsense.conf | 56 ++++ .../conf/pipelines/eval/6300_windows.conf | 161 ++++++++++ .../conf/pipelines/eval/6301_dns_windows.conf | 49 +++ .../conf/pipelines/eval/6400_suricata.conf | 92 ++++++ .../conf/pipelines/eval/6500_ossec.conf | 160 ++++++++++ .../pipelines/eval/6501_ossec_sysmon.conf | 118 ++++++++ .../pipelines/eval/6502_ossec_autoruns.conf | 43 +++ .../eval/6600_winlogbeat_sysmon.conf | 23 ++ .../conf/pipelines/eval/6700_winlogbeat.conf | 17 ++ .../conf/pipelines/eval/7100_osquery_wel.conf | 23 ++ ...01_postprocess_common_ip_augmentation.conf | 58 ++++ .../pipelines/eval/8007_postprocess_http.conf | 27 ++ .../eval/8200_postprocess_tagging.conf | 63 ++++ .../eval/8998_postprocess_log_elapsed.conf | 19 ++ .../eval/8999_postprocess_rename_type.conf | 8 + .../eval/templates/0800_input_eval.conf | 203 +++++++++++++ .../eval/templates/9000_output_bro.conf | 31 ++ .../eval/templates/9001_output_switch.conf | 27 ++ .../eval/templates/9002_output_import.conf | 27 ++ .../eval/templates/9004_output_flow.conf | 27 ++ .../eval/templates/9026_output_dhcp.conf | 26 ++ .../eval/templates/9029_output_esxi.conf | 25 ++ .../eval/templates/9030_output_greensql.conf | 25 ++ .../eval/templates/9031_output_iis.conf | 26 ++ .../eval/templates/9032_output_mcafee.conf | 26 ++ .../eval/templates/9033_output_snort.conf | 29 ++ .../eval/templates/9034_output_syslog.conf | 28 ++ .../eval/templates/9100_output_osquery.conf | 19 ++ .../eval/templates/9200_output_firewall.conf | 29 ++ .../eval/templates/9300_output_windows.conf | 27 ++ .../templates/9301_output_dns_windows.conf | 27 ++ .../eval/templates/9400_output_suricata.conf | 27 ++ .../eval/templates/9500_output_beats.conf | 25 ++ .../eval/templates/9600_output_ossec.conf | 29 ++ .../eval/templates/9999_output_redis.conf | 26 ++ salt/logstash/etc/logstash.yml | 6 +- salt/logstash/init.sls | 4 + salt/top.sls | 1 - 55 files changed, 2582 insertions(+), 10 deletions(-) create mode 100644 salt/logstash/conf/pipelines/eval/0010_input_hhbeats.conf create mode 100644 salt/logstash/conf/pipelines/eval/1000_preprocess_log_elapsed.conf create mode 100644 salt/logstash/conf/pipelines/eval/1001_preprocess_syslogng.conf create mode 100644 salt/logstash/conf/pipelines/eval/1002_preprocess_json.conf create mode 100644 salt/logstash/conf/pipelines/eval/1004_preprocess_syslog_types.conf create mode 100644 salt/logstash/conf/pipelines/eval/1026_preprocess_dhcp.conf create mode 100644 salt/logstash/conf/pipelines/eval/1029_preprocess_esxi.conf create mode 100644 salt/logstash/conf/pipelines/eval/1030_preprocess_greensql.conf create mode 100644 salt/logstash/conf/pipelines/eval/1031_preprocess_iis.conf create mode 100644 salt/logstash/conf/pipelines/eval/1032_preprocess_mcafee.conf create mode 100644 salt/logstash/conf/pipelines/eval/1033_preprocess_snort.conf create mode 100644 salt/logstash/conf/pipelines/eval/1034_preprocess_syslog.conf create mode 100644 salt/logstash/conf/pipelines/eval/2000_network_flow.conf create mode 100644 salt/logstash/conf/pipelines/eval/6002_syslog.conf create mode 100644 salt/logstash/conf/pipelines/eval/6101_switch_brocade.conf create mode 100644 salt/logstash/conf/pipelines/eval/6200_firewall_fortinet.conf create mode 100644 salt/logstash/conf/pipelines/eval/6201_firewall_pfsense.conf create mode 100644 salt/logstash/conf/pipelines/eval/6300_windows.conf create mode 100644 salt/logstash/conf/pipelines/eval/6301_dns_windows.conf create mode 100644 salt/logstash/conf/pipelines/eval/6400_suricata.conf create mode 100644 salt/logstash/conf/pipelines/eval/6500_ossec.conf create mode 100644 salt/logstash/conf/pipelines/eval/6501_ossec_sysmon.conf create mode 100644 salt/logstash/conf/pipelines/eval/6502_ossec_autoruns.conf create mode 100644 salt/logstash/conf/pipelines/eval/6600_winlogbeat_sysmon.conf create mode 100644 salt/logstash/conf/pipelines/eval/6700_winlogbeat.conf create mode 100644 salt/logstash/conf/pipelines/eval/7100_osquery_wel.conf create mode 100644 salt/logstash/conf/pipelines/eval/8001_postprocess_common_ip_augmentation.conf create mode 100644 salt/logstash/conf/pipelines/eval/8007_postprocess_http.conf create mode 100644 salt/logstash/conf/pipelines/eval/8200_postprocess_tagging.conf create mode 100644 salt/logstash/conf/pipelines/eval/8998_postprocess_log_elapsed.conf create mode 100644 salt/logstash/conf/pipelines/eval/8999_postprocess_rename_type.conf create mode 100644 salt/logstash/conf/pipelines/eval/templates/0800_input_eval.conf create mode 100644 salt/logstash/conf/pipelines/eval/templates/9000_output_bro.conf create mode 100644 salt/logstash/conf/pipelines/eval/templates/9001_output_switch.conf create mode 100644 salt/logstash/conf/pipelines/eval/templates/9002_output_import.conf create mode 100644 salt/logstash/conf/pipelines/eval/templates/9004_output_flow.conf create mode 100644 salt/logstash/conf/pipelines/eval/templates/9026_output_dhcp.conf create mode 100644 salt/logstash/conf/pipelines/eval/templates/9029_output_esxi.conf create mode 100644 salt/logstash/conf/pipelines/eval/templates/9030_output_greensql.conf create mode 100644 salt/logstash/conf/pipelines/eval/templates/9031_output_iis.conf create mode 100644 salt/logstash/conf/pipelines/eval/templates/9032_output_mcafee.conf create mode 100644 salt/logstash/conf/pipelines/eval/templates/9033_output_snort.conf create mode 100644 salt/logstash/conf/pipelines/eval/templates/9034_output_syslog.conf create mode 100644 salt/logstash/conf/pipelines/eval/templates/9100_output_osquery.conf create mode 100644 salt/logstash/conf/pipelines/eval/templates/9200_output_firewall.conf create mode 100644 salt/logstash/conf/pipelines/eval/templates/9300_output_windows.conf create mode 100644 salt/logstash/conf/pipelines/eval/templates/9301_output_dns_windows.conf create mode 100644 salt/logstash/conf/pipelines/eval/templates/9400_output_suricata.conf create mode 100644 salt/logstash/conf/pipelines/eval/templates/9500_output_beats.conf create mode 100644 salt/logstash/conf/pipelines/eval/templates/9600_output_ossec.conf create mode 100644 salt/logstash/conf/pipelines/eval/templates/9999_output_redis.conf diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 8528ecc38..7dd59ef01 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -1,4 +1,4 @@ - # Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC +# Copyright 2014,2015,2016,2017,2018 Security Onion Solutions, LLC # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -14,11 +14,11 @@ {% set VERSION = salt['pillar.get']('static:soversion', 'HH1.1.4') %} {% set MASTER = salt['grains.get']('master') %} {%- set MASTERIP = salt['pillar.get']('static:masterip', '') %} -{% set FEATURES = salt['pillar.get']('elastic:features', False) %} -{% if FEATURES %} - {% set FEATURES = "-features" %} -{% else %} - {% set FEATURES = '' %} +{% set FEATURES = salt['pillar.get']('elastic:features', False) %} +{% if FEATURES %} + {% set FEATURES = "-features" %} +{% else %} + {% set FEATURES = '' %} {% endif %} # Filebeat Setup filebeatetcdir: diff --git a/salt/logstash/conf/pipelines/eval/0010_input_hhbeats.conf b/salt/logstash/conf/pipelines/eval/0010_input_hhbeats.conf new file mode 100644 index 000000000..6b7667f5c --- /dev/null +++ b/salt/logstash/conf/pipelines/eval/0010_input_hhbeats.conf @@ -0,0 +1,40 @@ +input { + beats { + port => "5644" + ssl => true + ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"] + ssl_certificate => "/usr/share/logstash/filebeat.crt" + ssl_key => "/usr/share/logstash/filebeat.key" + tags => [ "beat" ] + } +} +filter { + if [type] == "ids" or [type] =~ "bro" { + mutate { + rename => { "host" => "beat_host" } + remove_tag => ["beat"] + add_field => { "sensor_name" => "%{[beat][name]}" } + add_field => { "syslog-host_from" => "%{[beat][name]}" } + remove_field => [ "beat", "prospector", "input", "offset" ] + } + } + if [type] =~ "ossec" { + mutate { + rename => { "host" => "beat_host" } + remove_tag => ["beat"] + add_field => { "syslog-host_from" => "%{[beat][name]}" } + remove_field => [ "beat", "prospector", "input", "offset" ] + } + } + if [type] == "osquery" { + mutate { + rename => { "host" => "beat_host" } + remove_tag => ["beat"] + add_tag => ["osquery"] + } + json { + source => "message" + target => "osquery" + } + } +} diff --git a/salt/logstash/conf/pipelines/eval/1000_preprocess_log_elapsed.conf b/salt/logstash/conf/pipelines/eval/1000_preprocess_log_elapsed.conf new file mode 100644 index 000000000..d098eb11a --- /dev/null +++ b/salt/logstash/conf/pipelines/eval/1000_preprocess_log_elapsed.conf @@ -0,0 +1,13 @@ +# Author: Justin Henderson +# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics +# Email: justin@hasecuritysolution.com +# Last Update: 12/9/2016 + +filter { + ruby { + code => "event.set('task_start', Time.now.to_f)" + } + mutate { + #add_tag => [ "conf_file_1000"] + } +} diff --git a/salt/logstash/conf/pipelines/eval/1001_preprocess_syslogng.conf b/salt/logstash/conf/pipelines/eval/1001_preprocess_syslogng.conf new file mode 100644 index 000000000..84bce8802 --- /dev/null +++ b/salt/logstash/conf/pipelines/eval/1001_preprocess_syslogng.conf @@ -0,0 +1,33 @@ +# Updated by: Doug Burks and Wes Lambert +# Last Update: 10/30/2018 + +filter { + if "syslogng" in [tags] { + mutate { + rename => { "MESSAGE" => "message" } + rename => { "PROGRAM" => "type" } + rename => { "FACILITY" => "syslog-facility" } + rename => { "FILE_NAME" => "syslog-file_name" } + rename => { "HOST" => "syslog-host" } + rename => { "HOST_FROM" => "syslog-host_from" } + rename => { "LEGACY_MSGHDR" => "syslog-legacy_msghdr" } + rename => { "PID" => "syslog-pid" } + rename => { "PRIORITY" => "syslog-priority" } + rename => { "SOURCEIP" => "syslog-sourceip" } + rename => { "TAGS" => "syslog-tags" } + lowercase => [ "syslog-host_from" ] + remove_field => [ "ISODATE" ] + remove_field => [ "SEQNUM" ] + #add_tag => [ "conf_file_1001"] + } + if "bro_" in [type] { + mutate { + add_tag => [ "bro" ] + } + } else if [type] !~ /ossec.*|snort/ and "firewall" not in [tags] { + mutate { + add_tag => [ "syslog" ] + } + } + } +} diff --git a/salt/logstash/conf/pipelines/eval/1002_preprocess_json.conf b/salt/logstash/conf/pipelines/eval/1002_preprocess_json.conf new file mode 100644 index 000000000..ea7c677da --- /dev/null +++ b/salt/logstash/conf/pipelines/eval/1002_preprocess_json.conf @@ -0,0 +1,18 @@ +# Author: Justin Henderson +# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics +# Email: justin@hasecuritysolution.com +# Last Update: 12/9/2016 + +filter { + if "json" in [tags]{ + json { + source => "message" + } + mutate { + remove_tag => [ "json" ] + } + mutate { + #add_tag => [ "conf_file_1002"] + } + } +} diff --git a/salt/logstash/conf/pipelines/eval/1004_preprocess_syslog_types.conf b/salt/logstash/conf/pipelines/eval/1004_preprocess_syslog_types.conf new file mode 100644 index 000000000..243abcc15 --- /dev/null +++ b/salt/logstash/conf/pipelines/eval/1004_preprocess_syslog_types.conf @@ -0,0 +1,19 @@ +filter { + if "syslog" in [tags] { + if [host] == "172.16.1.1" { + mutate { + add_field => { "type" => "fortinet" } + add_tag => [ "firewall" ] + } + } + if [host] == "10.0.0.101" { + mutate { + add_field => { "type" => "brocade" } + add_tag => [ "switch" ] + } + } + mutate { + #add_tag => [ "conf_file_1004"] + } + } +} diff --git a/salt/logstash/conf/pipelines/eval/1026_preprocess_dhcp.conf b/salt/logstash/conf/pipelines/eval/1026_preprocess_dhcp.conf new file mode 100644 index 000000000..2f893cf7a --- /dev/null +++ b/salt/logstash/conf/pipelines/eval/1026_preprocess_dhcp.conf @@ -0,0 +1,140 @@ +# Author: Justin Henderson +# SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics +# Email: justin@hasecuritysolutions.com +# Last Update: 12/9/2016 +# This conf file is based on accepting logs for DHCP. It is currently based on Windows DHCP only. +filter { + if [type] == "dhcp" { + mutate { + add_field => { "Hostname" => "%{host}" } + } + mutate { + strip => "message" + } + # This is the initial parsing of the log + grok { + # Server 2008+ + match => { "message" => "%{DATA:id},%{DATE_US:date},(?