From b0293c24b57d9dcf50aa449c9b5043d1d9514fa3 Mon Sep 17 00:00:00 2001
From: Mike Reeves <luke@geekempire.com>
Date: Tue, 16 Oct 2018 14:53:48 -0400
Subject: [PATCH] Logstash Module - Disable freq and modify some configs

---
 salt/filebeat/etc/filebeat.yml                |   4 +-
 salt/logstash/conf/conf.enabled.txt.so-master |   2 +-
 salt/logstash/conf/conf.enabled.txt.so-node   |  10 +-
 salt/logstash/conf/conf.enabled.txt.storage   | 102 ------
 .../files/dynamic/0006_input_beats.conf       | 304 +-----------------
 5 files changed, 11 insertions(+), 411 deletions(-)
 delete mode 100644 salt/logstash/conf/conf.enabled.txt.storage

diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml
index a45768ae3..05197a29c 100644
--- a/salt/filebeat/etc/filebeat.yml
+++ b/salt/filebeat/etc/filebeat.yml
@@ -72,7 +72,7 @@ output.logstash:
 
 
 # Elasticsearch template settings
-setup.template.settings:
+#setup.template.settings:
 
   # A dictionary of settings to place into the settings.index dictionary
   # of the Elasticsearch template. For more details, please check
@@ -91,7 +91,7 @@ setup.template.settings:
 
 # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.
 # This requires a Kibana endpoint configuration.
-setup.kibana:
+#setup.kibana:
 
   # Kibana Host
   # Scheme and port can be left out and will be set to the default (http and 5601)
diff --git a/salt/logstash/conf/conf.enabled.txt.so-master b/salt/logstash/conf/conf.enabled.txt.so-master
index a4da29cad..c33e46abe 100644
--- a/salt/logstash/conf/conf.enabled.txt.so-master
+++ b/salt/logstash/conf/conf.enabled.txt.so-master
@@ -12,6 +12,6 @@
 /usr/share/logstash/pipeline.so/0002_input_windows_json.conf
 /usr/share/logstash/pipeline.so/0003_input_syslog.conf
 /usr/share/logstash/pipeline.so/0005_input_suricata.conf
-/usr/share/logstash/pipeline.so/0006_input_beats.conf
+/usr/share/logstash/pipeline.dynamic/0006_input_beats.conf
 /usr/share/logstash/pipeline.so/0007_input_import.conf
 /usr/share/logstash/pipeline.dynamic/9999_output_redis.conf
diff --git a/salt/logstash/conf/conf.enabled.txt.so-node b/salt/logstash/conf/conf.enabled.txt.so-node
index 42e68c933..218f5b04d 100644
--- a/salt/logstash/conf/conf.enabled.txt.so-node
+++ b/salt/logstash/conf/conf.enabled.txt.so-node
@@ -75,12 +75,12 @@
 /usr/share/logstash/pipeline.so/8006_postprocess_dns.conf
 /usr/share/logstash/pipeline.so/8007_postprocess_dns_top1m_tagging.conf
 /usr/share/logstash/pipeline.so/8007_postprocess_http.conf
-/usr/share/logstash/pipeline.so/8008_postprocess_dns_whois_age.conf
+#/usr/share/logstash/pipeline.so/8008_postprocess_dns_whois_age.conf
 /usr/share/logstash/pipeline.so/8200_postprocess_tagging.conf
-/usr/share/logstash/pipeline.so/8502_postprocess_freq_analysis_bro_dns.conf
-/usr/share/logstash/pipeline.so/8503_postprocess_freq_analysis_bro_http.conf
-/usr/share/logstash/pipeline.so/8504_postprocess_freq_analysis_bro_ssl.conf
-/usr/share/logstash/pipeline.so/8505_postprocess_freq_analysis_bro_x509.conf
+#/usr/share/logstash/pipeline.so/8502_postprocess_freq_analysis_bro_dns.conf
+#/usr/share/logstash/pipeline.so/8503_postprocess_freq_analysis_bro_http.conf
+#/usr/share/logstash/pipeline.so/8504_postprocess_freq_analysis_bro_ssl.conf
+#/usr/share/logstash/pipeline.so/8505_postprocess_freq_analysis_bro_x509.conf
 /usr/share/logstash/pipeline.so/8998_postprocess_log_elapsed.conf
 /usr/share/logstash/pipeline.so/8999_postprocess_rename_type.conf
 /usr/share/logstash/pipeline.dynamic/9000_output_bro.conf
diff --git a/salt/logstash/conf/conf.enabled.txt.storage b/salt/logstash/conf/conf.enabled.txt.storage
deleted file mode 100644
index 9ad77c23f..000000000
--- a/salt/logstash/conf/conf.enabled.txt.storage
+++ /dev/null
@@ -1,102 +0,0 @@
-# This is where can specify which LogStash configs get loaded.
-#
-# The custom folder on the master gets automatically synced to each logstash
-# node.
-#
-# To enable a custom configuration see the following example and uncomment:
-# /usr/share/logstash/pipeline.custom/1234_input_custom.conf
-##
-# All of the defaults are loaded.
-/usr/share/logstash/pipeline.dynamic/0900_input_redis.conf
-/usr/share/logstash/pipeline.so/1000_preprocess_log_elapsed.conf
-/usr/share/logstash/pipeline.so/1001_preprocess_syslogng.conf
-/usr/share/logstash/pipeline.so/1002_preprocess_json.conf
-/usr/share/logstash/pipeline.so/1003_preprocess_bro.conf
-/usr/share/logstash/pipeline.so/1004_preprocess_syslog_types.conf
-/usr/share/logstash/pipeline.so/1026_preprocess_dhcp.conf
-/usr/share/logstash/pipeline.so/1029_preprocess_esxi.conf
-/usr/share/logstash/pipeline.so/1030_preprocess_greensql.conf
-/usr/share/logstash/pipeline.so/1031_preprocess_iis.conf
-/usr/share/logstash/pipeline.so/1032_preprocess_mcafee.conf
-/usr/share/logstash/pipeline.so/1033_preprocess_snort.conf
-/usr/share/logstash/pipeline.so/1034_preprocess_syslog.conf
-/usr/share/logstash/pipeline.so/1100_preprocess_bro_conn.conf
-/usr/share/logstash/pipeline.so/1101_preprocess_bro_dhcp.conf
-/usr/share/logstash/pipeline.so/1102_preprocess_bro_dns.conf
-/usr/share/logstash/pipeline.so/1103_preprocess_bro_dpd.conf
-/usr/share/logstash/pipeline.so/1104_preprocess_bro_files.conf
-/usr/share/logstash/pipeline.so/1105_preprocess_bro_ftp.conf
-/usr/share/logstash/pipeline.so/1106_preprocess_bro_http.conf
-/usr/share/logstash/pipeline.so/1107_preprocess_bro_irc.conf
-/usr/share/logstash/pipeline.so/1108_preprocess_bro_kerberos.conf
-/usr/share/logstash/pipeline.so/1109_preprocess_bro_notice.conf
-/usr/share/logstash/pipeline.so/1110_preprocess_bro_rdp.conf
-/usr/share/logstash/pipeline.so/1111_preprocess_bro_signatures.conf
-/usr/share/logstash/pipeline.so/1112_preprocess_bro_smtp.conf
-/usr/share/logstash/pipeline.so/1113_preprocess_bro_snmp.conf
-/usr/share/logstash/pipeline.so/1114_preprocess_bro_software.conf
-/usr/share/logstash/pipeline.so/1115_preprocess_bro_ssh.conf
-/usr/share/logstash/pipeline.so/1116_preprocess_bro_ssl.conf
-/usr/share/logstash/pipeline.so/1117_preprocess_bro_syslog.conf
-/usr/share/logstash/pipeline.so/1118_preprocess_bro_tunnel.conf
-/usr/share/logstash/pipeline.so/1119_preprocess_bro_weird.conf
-/usr/share/logstash/pipeline.so/1121_preprocess_bro_mysql.conf
-/usr/share/logstash/pipeline.so/1122_preprocess_bro_socks.conf
-/usr/share/logstash/pipeline.so/1123_preprocess_bro_x509.conf
-/usr/share/logstash/pipeline.so/1124_preprocess_bro_intel.conf
-/usr/share/logstash/pipeline.so/1125_preprocess_bro_modbus.conf
-/usr/share/logstash/pipeline.so/1126_preprocess_bro_sip.conf
-/usr/share/logstash/pipeline.so/1127_preprocess_bro_radius.conf
-/usr/share/logstash/pipeline.so/1128_preprocess_bro_pe.conf
-/usr/share/logstash/pipeline.so/1129_preprocess_bro_rfb.conf
-/usr/share/logstash/pipeline.so/1130_preprocess_bro_dnp3.conf
-/usr/share/logstash/pipeline.so/1131_preprocess_bro_smb_files.conf
-/usr/share/logstash/pipeline.so/1132_preprocess_bro_smb_mapping.conf
-/usr/share/logstash/pipeline.so/1133_preprocess_bro_ntlm.conf
-/usr/share/logstash/pipeline.so/1134_preprocess_bro_dce_rpc.conf
-/usr/share/logstash/pipeline.so/1998_test_data.conf
-/usr/share/logstash/pipeline.so/2000_network_flow.conf
-/usr/share/logstash/pipeline.so/6000_bro.conf
-/usr/share/logstash/pipeline.so/6001_bro_import.conf
-/usr/share/logstash/pipeline.so/6002_syslog.conf
-/usr/share/logstash/pipeline.so/6101_switch_brocade.conf
-/usr/share/logstash/pipeline.so/6200_firewall_fortinet.conf
-/usr/share/logstash/pipeline.so/6201_firewall_pfsense.conf
-/usr/share/logstash/pipeline.so/6300_windows.conf
-/usr/share/logstash/pipeline.so/6301_dns_windows.conf
-/usr/share/logstash/pipeline.so/6400_suricata.conf
-/usr/share/logstash/pipeline.so/6500_ossec.conf
-/usr/share/logstash/pipeline.so/6501_ossec_sysmon.conf
-/usr/share/logstash/pipeline.so/6502_ossec_autoruns.conf
-/usr/share/logstash/pipeline.so/6600_winlogbeat_sysmon.conf
-/usr/share/logstash/pipeline.so/8000_postprocess_bro_cleanup.conf
-/usr/share/logstash/pipeline.so/8001_postprocess_common_ip_augmentation.conf
-/usr/share/logstash/pipeline.so/8006_postprocess_dns.conf
-/usr/share/logstash/pipeline.so/8007_postprocess_dns_top1m_tagging.conf
-/usr/share/logstash/pipeline.so/8007_postprocess_http.conf
-/usr/share/logstash/pipeline.so/8008_postprocess_dns_whois_age.conf
-/usr/share/logstash/pipeline.so/8200_postprocess_tagging.conf
-/usr/share/logstash/pipeline.so/8502_postprocess_freq_analysis_bro_dns.conf
-/usr/share/logstash/pipeline.so/8503_postprocess_freq_analysis_bro_http.conf
-/usr/share/logstash/pipeline.so/8504_postprocess_freq_analysis_bro_ssl.conf
-/usr/share/logstash/pipeline.so/8505_postprocess_freq_analysis_bro_x509.conf
-/usr/share/logstash/pipeline.so/8998_postprocess_log_elapsed.conf
-/usr/share/logstash/pipeline.so/8999_postprocess_rename_type.conf
-/usr/share/logstash/pipeline.dynamic/9000_output_bro.conf
-/usr/share/logstash/pipeline.dynamic/9001_output_switch.conf
-/usr/share/logstash/pipeline.dynamic/9002_output_import.conf
-/usr/share/logstash/pipeline.dynamic/9004_output_flow.conf
-/usr/share/logstash/pipeline.dynamic/9026_output_dhcp.conf
-/usr/share/logstash/pipeline.dynamic/9029_output_esxi.conf
-/usr/share/logstash/pipeline.dynamic/9030_output_greensql.conf
-/usr/share/logstash/pipeline.dynamic/9031_output_iis.conf
-/usr/share/logstash/pipeline.dynamic/9032_output_mcafee.conf
-/usr/share/logstash/pipeline.dynamic/9033_output_snort.conf
-/usr/share/logstash/pipeline.dynamic/9034_output_syslog.conf
-/usr/share/logstash/pipeline.dynamic/9200_output_firewall.conf
-/usr/share/logstash/pipeline.dynamic/9300_output_windows.conf
-/usr/share/logstash/pipeline.dynamic/9301_output_dns_windows.conf
-/usr/share/logstash/pipeline.dynamic/9400_output_suricata.conf
-/usr/share/logstash/pipeline.dynamic/9500_output_beats.conf
-/usr/share/logstash/pipeline.dynamic/9600_output_ossec.conf
-/usr/share/logstash/pipeline.dynamic/9998_output_test_data.conf
diff --git a/salt/logstash/files/dynamic/0006_input_beats.conf b/salt/logstash/files/dynamic/0006_input_beats.conf
index 4fece4143..bac23e150 100644
--- a/salt/logstash/files/dynamic/0006_input_beats.conf
+++ b/salt/logstash/files/dynamic/0006_input_beats.conf
@@ -11,313 +11,15 @@ input {
 filter {
   if "ids" in [tags] {
     mutate {
-      replace => ["type" => "snort"]
-      add_tag => ["snort"]
       remove_tag => ["beat"]
+      rename => { "host" => "beat_host" }
     }
   }
 
-  if "bro_conn" in [tags] {
+  if "bro" in [tags] {
     mutate {
-      replace => ["type" => "bro_conn"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_dhcp" in [tags] {
-    mutate {
-      replace => ["type" => "bro_dhcp"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_dns" in [tags] {
-    mutate {
-      replace => ["type" => "bro_dns"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_dpd" in [tags] {
-    mutate {
-      replace => ["type" => "bro_dpd"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_files" in [tags] {
-    mutate {
-      replace => ["type" => "bro_files"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_ftp" in [tags] {
-    mutate {
-      replace => ["type" => "bro_ftp"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_http" in [tags] {
-    mutate {
-      replace => ["type" => "bro_http"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_irc" in [tags] {
-    mutate {
-      replace => ["type" => "bro_irc"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_kerberos" in [tags] {
-    mutate {
-      replace => ["type" => "bro_kerberos"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_notice" in [tags] {
-    mutate {
-      replace => ["type" => "bro_notice"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_rdp" in [tags] {
-    mutate {
-      replace => ["type" => "bro_rdp"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_signatures" in [tags] {
-    mutate {
-      replace => ["type" => "bro_signatures"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_smtp" in [tags] {
-    mutate {
-      replace => ["type" => "bro_smtp"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_snmp" in [tags] {
-    mutate {
-      replace => ["type" => "bro_snmp"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_software" in [tags] {
-    mutate {
-      replace => ["type" => "bro_software"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_ssh" in [tags] {
-    mutate {
-      replace => ["type" => "bro_ssh"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_ssl" in [tags] {
-    mutate {
-      replace => ["type" => "bro_ssl"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_syslog" in [tags] {
-    mutate {
-      replace => ["type" => "bro_syslog"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_tunnel" in [tags] {
-    mutate {
-      replace => ["type" => "bro_tunnel"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_weird" in [tags] {
-    mutate {
-      replace => ["type" => "bro_weird"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_mysql" in [tags] {
-    mutate {
-      replace => ["type" => "bro_mysql"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_socks" in [tags] {
-    mutate {
-      replace => ["type" => "bro_socks"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_x509" in [tags] {
-    mutate {
-      replace => ["type" => "bro_x509"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_intel" in [tags] {
-    mutate {
-      replace => ["type" => "bro_intel"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_modbus" in [tags] {
-    mutate {
-      replace => ["type" => "bro_modbus"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_sip" in [tags] {
-    mutate {
-      replace => ["type" => "bro_sip"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_radius" in [tags] {
-    mutate {
-      replace => ["type" => "bro_radius"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_pe" in [tags] {
-    mutate {
-      replace => ["type" => "bro_pe"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_rfb" in [tags] {
-    mutate {
-      replace => ["type" => "bro_rfb"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_dnp3" in [tags] {
-    mutate {
-      replace => ["type" => "bro_dnp3"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_smb_files" in [tags] {
-    mutate {
-      replace => ["type" => "bro_smb_files"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_smb_mapping" in [tags] {
-    mutate {
-      replace => ["type" => "bro_smb_mapping"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_ntlm" in [tags] {
-    mutate {
-      replace => ["type" => "bro_ntlm"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_dce_rpc" in [tags] {
-    mutate {
-      replace => ["type" => "bro_dce_rpc"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_telnet" in [tags] {
-    mutate {
-      replace => ["type" => "bro_telnet"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_openvpn" in [tags] {
-    mutate {
-      replace => ["type" => "bro_openvpn"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_mqtt" in [tags] {
-    mutate {
-      replace => ["type" => "bro_mqtt"]
-      add_tag => ["bro"]
-      remove_tag => ["beat"]
-    }
-  }
-
-  if "bro_dhcpv6" in [tags] {
-    mutate {
-      replace => ["type" => "bro_dhcpv6"]
-      add_tag => ["bro"]
       remove_tag => ["beat"]
+      rename => { "host" => "beat_host" }
     }
   }
 }