Merge branch 'dev' of https://github.com/Security-Onion-Solutions/securityonion-saltstack into dev

pillar/logstash/eval.sls

@@ -2,54 +2,59 @@ logstash:
   pipelines:
     eval:
       config:
-        - 0800_input_eval.conf
+        - so/0800_input_eval.conf
-        - 1000_preprocess_log_elapsed.conf
+        - so/1000_preprocess_log_elapsed.conf
-        - 1001_preprocess_syslogng.conf
+        - so/1001_preprocess_syslogng.conf
-        - 1002_preprocess_json.conf
+        - so/1002_preprocess_json.conf
-        - 1004_preprocess_syslog_types.conf
+        - so/1004_preprocess_syslog_types.conf
-        - 1026_preprocess_dhcp.conf
+        - so/1026_preprocess_dhcp.conf
-        - 1029_preprocess_esxi.conf
+        - so/1029_preprocess_esxi.conf
-        - 1030_preprocess_greensql.conf
+        - so/1030_preprocess_greensql.conf
-        - 1031_preprocess_iis.conf
+        - so/1031_preprocess_iis.conf
-        - 1032_preprocess_mcafee.conf
+        - so/1032_preprocess_mcafee.conf
-        - 1033_preprocess_snort.conf
+        - so/1033_preprocess_snort.conf
-        - 1034_preprocess_syslog.conf
+        - so/1034_preprocess_syslog.conf
-        - 2000_network_flow.conf
+        - so/2000_network_flow.conf
-        - 6002_syslog.conf
+        - so/6002_syslog.conf
-        - 6101_switch_brocade.conf
+        - so/6101_switch_brocade.conf
-        - 6200_firewall_fortinet.conf
+        - so/6200_firewall_fortinet.conf
-        - 6201_firewall_pfsense.conf
+        - so/6201_firewall_pfsense.conf
-        - 6300_windows.conf
+        - so/6300_windows.conf
-        - 6301_dns_windows.conf
+        - so/6301_dns_windows.conf
-        - 6400_suricata.conf
+        - so/6400_suricata.conf
-        - 6500_ossec.conf
+        - so/6500_ossec.conf
-        - 6501_ossec_sysmon.conf
+        - so/6501_ossec_sysmon.conf
-        - 6502_ossec_autoruns.conf
+        - so/6502_ossec_autoruns.conf
-        - 6600_winlogbeat_sysmon.conf
+        - so/6600_winlogbeat_sysmon.conf
-        - 6700_winlogbeat.conf
+        - so/6700_winlogbeat.conf
-        - 7100_osquery_wel.conf
+        - so/7100_osquery_wel.conf
-        - 7200_strelka.conf
+        - so/7200_strelka.conf
-        - 8001_postprocess_common_ip_augmentation.conf
+        - so/8001_postprocess_common_ip_augmentation.conf
-        - 8007_postprocess_http.conf
+        - so/8007_postprocess_http.conf
-        - 8200_postprocess_tagging.conf
+        - so/8200_postprocess_tagging.conf
-        - 8998_postprocess_log_elapsed.conf
+        - so/8998_postprocess_log_elapsed.conf
-        - 8999_postprocess_rename_type.conf
+        - so/8999_postprocess_rename_type.conf
-        - 9000_output_bro.conf.jinja
+        - so/9000_output_bro.conf.jinja
-        - 9001_output_switch.conf.jinja
+        - so/9001_output_switch.conf.jinja
-        - 9002_output_import.conf.jinja
+        - so/9002_output_import.conf.jinja
-        - 9004_output_flow.conf.jinja
+        - so/9004_output_flow.conf.jinja
-        - 9026_output_dhcp.conf.jinja
+        - so/9026_output_dhcp.conf.jinja
-        - 9029_output_esxi.conf.jinja
+        - so/9029_output_esxi.conf.jinja
-        - 9030_output_greensql.conf.jinja
+        - so/9030_output_greensql.conf.jinja
-        - 9031_output_iis.conf.jinja
+        - so/9031_output_iis.conf.jinja
-        - 9032_output_mcafee.conf.jinja
+        - so/9032_output_mcafee.conf.jinja
-        - 9033_output_snort.conf.jinja
+        - so/9033_output_snort.conf.jinja
-        - 9034_output_syslog.conf.jinja
+        - so/9034_output_syslog.conf.jinja
-        - 9100_output_osquery.conf.jinja
+        - so/9100_output_osquery.conf.jinja
-        - 9200_output_firewall.conf.jinja
+        - so/9200_output_firewall.conf.jinja
-        - 9300_output_windows.conf.jinja
+        - so/9300_output_windows.conf.jinja
-        - 9301_output_dns_windows.conf.jinja
+        - so/9301_output_dns_windows.conf.jinja
-        - 9400_output_suricata.conf.jinja
+        - so/9400_output_suricata.conf.jinja
-        - 9500_output_beats.conf.jinja
+        - so/9500_output_beats.conf.jinja
-        - 9600_output_ossec.conf.jinja
+        - so/9600_output_ossec.conf.jinja
-        - 9700_ouptut_strelka.conf.jinja
+        - so/9700_output_strelka.conf.jinja
+  templates:
+    - so/beats-template.json
+    - so/logstash-ossec-template.json
+    - so/logstash-strelka-template.json
+    - so/logstash-template.json

pillar/logstash/helix.sls

@@ -2,41 +2,41 @@ logstash:
   pipelines:
     helix:
       config:
-        - 0010_input_hhbeats.conf
+        - so/0010_input_hhbeats.conf
-        - 1033_preprocess_snort.conf
+        - so/1033_preprocess_snort.conf
-        - 1100_preprocess_bro_conn.conf
+        - so/1100_preprocess_bro_conn.conf
-        - 1101_preprocess_bro_dhcp.conf
+        - so/1101_preprocess_bro_dhcp.conf
-        - 1102_preprocess_bro_dns.conf
+        - so/1102_preprocess_bro_dns.conf
-        - 1103_preprocess_bro_dpd.conf
+        - so/1103_preprocess_bro_dpd.conf
-        - 1104_preprocess_bro_files.conf
+        - so/1104_preprocess_bro_files.conf
-        - 1105_preprocess_bro_ftp.conf
+        - so/1105_preprocess_bro_ftp.conf
-        - 1106_preprocess_bro_http.conf
+        - so/1106_preprocess_bro_http.conf
-        - 1107_preprocess_bro_irc.conf
+        - so/1107_preprocess_bro_irc.conf
-        - 1108_preprocess_bro_kerberos.conf
+        - so/1108_preprocess_bro_kerberos.conf
-        - 1109_preprocess_bro_notice.conf
+        - so/1109_preprocess_bro_notice.conf
-        - 1110_preprocess_bro_rdp.conf
+        - so/1110_preprocess_bro_rdp.conf
-        - 1111_preprocess_bro_signatures.conf
+        - so/1111_preprocess_bro_signatures.conf
-        - 1112_preprocess_bro_smtp.conf
+        - so/1112_preprocess_bro_smtp.conf
-        - 1113_preprocess_bro_snmp.conf
+        - so/1113_preprocess_bro_snmp.conf
-        - 1114_preprocess_bro_software.conf
+        - so/1114_preprocess_bro_software.conf
-        - 1115_preprocess_bro_ssh.conf
+        - so/1115_preprocess_bro_ssh.conf
-        - 1116_preprocess_bro_ssl.conf
+        - so/1116_preprocess_bro_ssl.conf
-        - 1117_preprocess_bro_syslog.conf
+        - so/1117_preprocess_bro_syslog.conf
-        - 1118_preprocess_bro_tunnel.conf
+        - so/1118_preprocess_bro_tunnel.conf
-        - 1119_preprocess_bro_weird.conf
+        - so/1119_preprocess_bro_weird.conf
-        - 1121_preprocess_bro_mysql.conf
+        - so/1121_preprocess_bro_mysql.conf
-        - 1122_preprocess_bro_socks.conf
+        - so/1122_preprocess_bro_socks.conf
-        - 1123_preprocess_bro_x509.conf
+        - so/1123_preprocess_bro_x509.conf
-        - 1124_preprocess_bro_intel.conf
+        - so/1124_preprocess_bro_intel.conf
-        - 1125_preprocess_bro_modbus.conf
+        - so/1125_preprocess_bro_modbus.conf
-        - 1126_preprocess_bro_sip.conf
+        - so/1126_preprocess_bro_sip.conf
-        - 1127_preprocess_bro_radius.conf
+        - so/1127_preprocess_bro_radius.conf
-        - 1128_preprocess_bro_pe.conf
+        - so/1128_preprocess_bro_pe.conf
-        - 1129_preprocess_bro_rfb.conf
+        - so/1129_preprocess_bro_rfb.conf
-        - 1130_preprocess_bro_dnp3.conf
+        - so/1130_preprocess_bro_dnp3.conf
-        - 1131_preprocess_bro_smb_files.conf
+        - so/1131_preprocess_bro_smb_files.conf
-        - 1132_preprocess_bro_smb_mapping.conf
+        - so/1132_preprocess_bro_smb_mapping.conf
-        - 1133_preprocess_bro_ntlm.conf
+        - so/1133_preprocess_bro_ntlm.conf
-        - 1134_preprocess_bro_dce_rpc.conf
+        - so/1134_preprocess_bro_dce_rpc.conf
-        - 8001_postprocess_common_ip_augmentation.conf
+        - so/8001_postprocess_common_ip_augmentation.conf
-        - 9997_output_helix.conf.jinja
+        - so/9997_output_helix.conf.jinja

pillar/logstash/init.sls

@@ -0,0 +1,11 @@
+logstash:
+  docker_options:
+    port_bindings:
+      - 0.0.0.0:514:514
+      - 0.0.0.0:5044:5044
+      - 0.0.0.0:5644:5644
+      - 0.0.0.0:6050:6050
+      - 0.0.0.0:6051:6051
+      - 0.0.0.0:6052:6052
+      - 0.0.0.0:6053:6053
+      - 0.0.0.0:9600:9600

pillar/logstash/master.sls

@@ -2,5 +2,5 @@ logstash:
   pipelines:
     master:
       config:
-        - 0010_input_hhbeats.conf
+        - so/0010_input_hhbeats.conf
-        - 9999_output_redis.conf.jinja
+        - so/9999_output_redis.conf.jinja

pillar/logstash/search.sls

@@ -2,54 +2,59 @@ logstash:
   pipelines:
     search:
       config:
-        - 1000_preprocess_log_elapsed.conf
+        - so/1000_preprocess_log_elapsed.conf
-        - 1001_preprocess_syslogng.conf
+        - so/1001_preprocess_syslogng.conf
-        - 1002_preprocess_json.conf
+        - so/1002_preprocess_json.conf
-        - 1004_preprocess_syslog_types.conf
+        - so/1004_preprocess_syslog_types.conf
-        - 1026_preprocess_dhcp.conf
+        - so/1026_preprocess_dhcp.conf
-        - 1029_preprocess_esxi.conf
+        - so/1029_preprocess_esxi.conf
-        - 1030_preprocess_greensql.conf
+        - so/1030_preprocess_greensql.conf
-        - 1031_preprocess_iis.conf
+        - so/1031_preprocess_iis.conf
-        - 1032_preprocess_mcafee.conf
+        - so/1032_preprocess_mcafee.conf
-        - 1033_preprocess_snort.conf
+        - so/1033_preprocess_snort.conf
-        - 1034_preprocess_syslog.conf
+        - so/1034_preprocess_syslog.conf
-        - 2000_network_flow.conf
+        - so/2000_network_flow.conf
-        - 6002_syslog.conf
+        - so/6002_syslog.conf
-        - 6101_switch_brocade.conf
+        - so/6101_switch_brocade.conf
-        - 6200_firewall_fortinet.conf
+        - so/6200_firewall_fortinet.conf
-        - 6201_firewall_pfsense.conf
+        - so/6201_firewall_pfsense.conf
-        - 6300_windows.conf
+        - so/6300_windows.conf
-        - 6301_dns_windows.conf
+        - so/6301_dns_windows.conf
-        - 6400_suricata.conf
+        - so/6400_suricata.conf
-        - 6500_ossec.conf
+        - so/6500_ossec.conf
-        - 6501_ossec_sysmon.conf
+        - so/6501_ossec_sysmon.conf
-        - 6502_ossec_autoruns.conf
+        - so/6502_ossec_autoruns.conf
-        - 6600_winlogbeat_sysmon.conf
+        - so/6600_winlogbeat_sysmon.conf
-        - 6700_winlogbeat.conf
+        - so/6700_winlogbeat.conf
-        - 7100_osquery_wel.conf
+        - so/7100_osquery_wel.conf
-        - 7200_strelka.conf
+        - so/7200_strelka.conf
-        - 8001_postprocess_common_ip_augmentation.conf
+        - so/8001_postprocess_common_ip_augmentation.conf
-        - 8007_postprocess_http.conf
+        - so/8007_postprocess_http.conf
-        - 8200_postprocess_tagging.conf
+        - so/8200_postprocess_tagging.conf
-        - 8998_postprocess_log_elapsed.conf
+        - so/8998_postprocess_log_elapsed.conf
-        - 8999_postprocess_rename_type.conf
+        - so/8999_postprocess_rename_type.conf
-        - 0900_input_redis.conf.jinja
+        - so/0900_input_redis.conf.jinja
-        - 9000_output_bro.conf.jinja
+        - so/9000_output_bro.conf.jinja
-        - 9001_output_switch.conf.jinja
+        - so/9001_output_switch.conf.jinja
-        - 9002_output_import.conf.jinja
+        - so/9002_output_import.conf.jinja
-        - 9004_output_flow.conf.jinja
+        - so/9004_output_flow.conf.jinja
-        - 9026_output_dhcp.conf.jinja
+        - so/9026_output_dhcp.conf.jinja
-        - 9029_output_esxi.conf.jinja
+        - so/9029_output_esxi.conf.jinja
-        - 9030_output_greensql.conf.jinja
+        - so/9030_output_greensql.conf.jinja
-        - 9031_output_iis.conf.jinja
+        - so/9031_output_iis.conf.jinja
-        - 9032_output_mcafee.conf.jinja
+        - so/9032_output_mcafee.conf.jinja
-        - 9033_output_snort.conf.jinja
+        - so/9033_output_snort.conf.jinja
-        - 9034_output_syslog.conf.jinja
+        - so/9034_output_syslog.conf.jinja
-        - 9100_output_osquery.conf.jinja
+        - so/9100_output_osquery.conf.jinja
-        - 9200_output_firewall.conf.jinja
+        - so/9200_output_firewall.conf.jinja
-        - 9300_output_windows.conf.jinja
+        - so/9300_output_windows.conf.jinja
-        - 9301_output_dns_windows.conf.jinja
+        - so/9301_output_dns_windows.conf.jinja
-        - 9400_output_suricata.conf.jinja
+        - so/9400_output_suricata.conf.jinja
-        - 9500_output_beats.conf.jinja
+        - so/9500_output_beats.conf.jinja
-        - 9600_output_ossec.conf.jinja
+        - so/9600_output_ossec.conf.jinja
-        - 9700_output_strelka.conf.jinja
+        - so/9700_output_strelka.conf.jinja
+  templates:
+    - so/beats-template.json
+    - so/logstash-ossec-template.json
+    - so/logstash-strelka-template.json
+    - so/logstash-template.json

pillar/top.sls

@@ -5,6 +5,7 @@ base:
 
   'G@role:so-mastersearch or G@role:so-heavynode':
     - match: compound
+    - logstash
     - logstash.master
     - logstash.search
 
@@ -23,6 +24,7 @@ base:
     - minions.{{ grains.id }}
 
   'G@role:so-master':
+    - logstash
     - logstash.master
 
   'G@role:so-eval':
@@ -31,6 +33,7 @@ base:
     - data.*
     - brologs
     - auth
+    - logstash
     - logstash.eval
     - minions.{{ grains.id }}
 
@@ -50,6 +53,6 @@ base:
     - firewall.*
     - fireeye
     - brologs
+    - logstash
     - logstash.helix
-    - static
     - minions.{{ grains.id }}

salt/logstash/defaults.yml

@@ -1,6 +0,0 @@
-logstash:
-  pipelines:
-    master: 
-      config: "/usr/share/logstash/pipelines/master/*.conf"
-    search: 
-      config: "/usr/share/logstash/pipelines/search/*.conf"

salt/logstash/files/custom/parsers/Drop.Your.Custom.Parsers.Here.conf

@@ -1,2 +0,0 @@
-#
-#

salt/logstash/files/dynamic/0006_input_beats.conf

@@ -1,23 +0,0 @@
-input {
-  beats {
-    port => "5044"
-    ssl => false
-    ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"]
-    ssl_certificate => "/usr/share/logstash/filebeat.crt"
-    ssl_key => "/usr/share/logstash/filebeat.key"
-    tags => [ "beat" ]
-  }
-}
-filter {
-    if [type] == "osquery" {
-    mutate {
-      rename => { "host" => "beat_host" }
-      remove_tag => ["beat"]
-      add_tag => ["osquery"]
-        	}
-   json {
-    source => "message"
-    target => "osquery"
-        }
-  }
-}

salt/logstash/files/dynamic/0008_input_eval.conf

@@ -1,203 +0,0 @@
-# Updated by: Mike Reeves
-# Last Update: 11/1/2018
-
-input {
-  file {
-	  path => "/suricata/eve.json"
-		type => "ids"
-                add_field => { "engine" => "suricata" }
-	}
-	file {
-		path => "/nsm/bro/logs/current/conn*.log"
-		type => "bro_conn"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/dce_rpc*.log"
-		type => "bro_dce_rpc"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/dhcp*.log"
-		type => "bro_dhcp"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/dnp3*.log"
-		type => "bro_dnp3"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/dns*.log"
-		type => "bro_dns"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/dpd*.log"
-		type => "bro_dpd"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/files*.log"
-		type => "bro_files"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/ftp*.log"
-		type => "bro_ftp"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/http*.log"
-		type => "bro_http"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/intel*.log"
-		type => "bro_intel"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/irc*.log"
-		type => "bro_irc"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/kerberos*.log"
-		type => "bro_kerberos"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/modbus*.log"
-		type => "bro_modbus"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/mysql*.log"
-		type => "bro_mysql"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/notice*.log"
-		type => "bro_notice"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/ntlm*.log"
-		type => "bro_ntlm"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/pe*.log"
-		type => "bro_pe"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/radius*.log"
-		type => "bro_radius"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/rdp*.log"
-		type => "bro_rdp"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/rfb*.log"
-		type => "bro_rfb"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/signatures*.log"
-		type => "bro_signatures"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/sip*.log"
-		type => "bro_sip"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/smb_files*.log"
-		type => "bro_smb_files"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/smb_mapping*.log"
-		type => "bro_smb_mapping"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/smtp*.log"
-		type => "bro_smtp"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/snmp*.log"
-		type => "bro_snmp"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/socks*.log"
-		type => "bro_socks"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/software*.log"
-		type => "bro_software"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/ssh*.log"
-		type => "bro_ssh"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/ssl*.log"
-		type => "bro_ssl"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/syslog*.log"
-		type => "bro_syslog"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/tunnel*.log"
-		type => "bro_tunnels"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/weird*.log"
-		type => "bro_weird"
-	    	tags => ["bro"]
-	}
-	file {
-		path => "/nsm/bro/logs/current/x509*.log"
-		type => "bro_x509"
-	    	tags => ["bro"]
-	}
-  file {
-    path => "/wazuh/alerts/alerts.json"
-    type => "ossec"
-  }
-  file {
-    path => "/wazuh/archives/archive.json"
-    type => "ossec_archive"
-  }
-  file {
-    path => "/osquery/logs/result.log"
-    type => "osquery"
-  }
-  file {
-    path => "/strelka/strelka.log"
-    type => "strelka"
-  }
-}
-filter {
-  if "import" in [tags] {
-	mutate {
-	  #add_tag => [ "conf_file_0007"]
-	}
-  }
-}

salt/logstash/files/dynamic/9000_output_bro.conf

@@ -1,31 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('node:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-
-filter {
-  if "bro" in [tags] and "test_data" not in [tags] and "import" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9000"]
-	}
-  }
-}
-output {
-  if "bro" in [tags] and "test_data" not in [tags] and "import" not in [tags] {
-#    stdout { codec => rubydebug }
-    elasticsearch {
-      pipeline => "%{event_type}"
-      hosts => "{{ ES }}"
-      index => "logstash-bro-%{+YYYY.MM.dd}"
-      template_name => "logstash"
-      template => "/logstash-template.json"
-      template_overwrite => true
-    }
-  }
-}

salt/logstash/files/dynamic/9001_output_switch.conf

@@ -1,27 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('node:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if "switch" in [tags] and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9001"]
-	}
-  }
-}
-output {
-  if "switch" in [tags] and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      index => "logstash-switch-%{+YYYY.MM.dd}"
-      template => "/logstash-template.json"
-    }
-  }
-}

salt/logstash/files/dynamic/9002_output_import.conf

@@ -1,27 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('node:mainip', '') -%}
-{%- endif %}
-# Updated by: Doug Burks
-# Last Update: 5/16/2017
-
-filter {
-  if "import" in [tags] and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9002"]
-	}
-  }
-}
-output {
-  if "import" in [tags] and "test_data" not in [tags] {
-#    stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      index => "logstash-import-%{+YYYY.MM.dd}"
-      template_name => "logstash-*"
-      template => "/logstash-template.json"
-      template_overwrite => true
-    }
-  }
-}

salt/logstash/files/dynamic/9004_output_flow.conf

@@ -1,27 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('node:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "sflow" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9004"]
-	}
-  }
-}
-output {
-  if [event_type] == "sflow" and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      index => "logstash-flow-%{+YYYY.MM.dd}"
-      template => "/logstash-template.json"
-    }
-  }
-}

salt/logstash/files/dynamic/9026_output_dhcp.conf

@@ -1,26 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('node:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "dhcp" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9026"]
-	}
-  }
-}
-output {
-  if [event_type] == "dhcp" and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      template => "/logstash-template.json"
-    }
-  }
-}

salt/logstash/files/dynamic/9029_output_esxi.conf

@@ -1,25 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('node:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "esxi" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9029"]
-	}
-  }
-}
-output {
-  if [event_type] == "esxi" and "test_data" not in [tags] {
-    elasticsearch {
-      hosts => "{{ ES }}"
-      template => "/logstash-template.json"
-    }
-  }
-}

salt/logstash/files/dynamic/9030_output_greensql.conf

@@ -1,25 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('node:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "greensql" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9030"]
-	}
-  }
-}
-output {
-  if [event_type] == "greensql" and "test_data" not in [tags] {
-    elasticsearch {
-      hosts => "{{ ES }}"
-      template => "/logstash-template.json"
-    }
-  }
-}

salt/logstash/files/dynamic/9031_output_iis.conf

@@ -1,26 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('node:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "iis" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9031"]
-	}
-  }
-}
-output {
-  if [event_type] == "iis" and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      template => "/logstash-template.json"
-    }
-  }
-}

salt/logstash/files/dynamic/9032_output_mcafee.conf

@@ -1,26 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('node:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "mcafee" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9032"]
-	}
-  }
-}
-output {
-  if [event_type] == "mcafee" and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      template => "/logstash-template.json"
-    }
-  }
-}

salt/logstash/files/dynamic/9033_output_snort.conf

@@ -1,29 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('node:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "ids" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9033"]
-	}
-  }
-}
-output {
-    if [event_type] == "ids" and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      index => "logstash-ids-%{+YYYY.MM.dd}"
-      template_name => "logstash"
-      template => "/logstash-template.json"
-      template_overwrite => true
-    }
-  }
-}

salt/logstash/files/dynamic/9034_output_syslog.conf

@@ -1,28 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('node:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Updated by: Doug Burks
-# Last Update: 5/15/2017
-
-filter {
-  if "syslog" in [tags] and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9034"]
-	}
-  }
-}
-output {
-  if "syslog" in [tags] and "test_data" not in [tags] {
-    elasticsearch {
-      hosts => "{{ ES }}"
-      index => "logstash-syslog-%{+YYYY.MM.dd}"
-      template_name => "logstash"
-      template => "/logstash-template.json"
-      template_overwrite => true
-    }
-  }
-}

salt/logstash/files/dynamic/9100_output_osquery.conf

@@ -1,19 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('node:mainip', '') -%}
-{%- endif %}
-# Author: Josh Brower
-# Last Update: 12/29/2018
-# Output to ES for osquery tagged logs
-
-
-output {
-  if "osquery" in [tags] {
-    elasticsearch {
-      hosts => "{{ ES }}"
-      index => "logstash-osquery-%{+YYYY.MM.dd}"
-      template => "/logstash-template.json"
-    }
-  }
-}

salt/logstash/files/dynamic/9200_output_firewall.conf

@@ -1,29 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('node:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if "firewall" in [tags] and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9200"]
-	}
-  }
-}
-output {
-  if "firewall" in [tags] and "test_data" not in [tags] {
-#    stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      index => "logstash-firewall-%{+YYYY.MM.dd}"
-      template_name => "logstash"
-      template => "/logstash-template.json"
-      template_overwrite => true
-    }
-  }
-}

salt/logstash/files/dynamic/9300_output_windows.conf

@@ -1,27 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('node:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "windows" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9300"]
-	}
-  }
-}
-output {
-  if [event_type] == "windows" and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      index => "logstash-windows-%{+YYYY.MM.dd}"
-      template => "/logstash-template.json"
-    }
-  }
-}

salt/logstash/files/dynamic/9301_output_dns_windows.conf

@@ -1,27 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('node:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "dns" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9301"]
-	}
-  }
-}
-output {
-  if [event_type] == "dns" and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      index => "logstash-%{+YYYY.MM.dd}"
-      template => "/logstash-template.json"
-    }
-  }
-}

salt/logstash/files/dynamic/9400_output_suricata.conf

@@ -1,27 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('node:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if [event_type] == "suricata" and "test_data" not in [tags] {
-    mutate {
-	  ##add_tag => [ "conf_file_9400"]
-	}
-  }
-}
-output {
-  if [event_type] == "suricata" and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => "{{ ES }}"
-      index => "logstash-ids-%{+YYYY.MM.dd}"
-      template => "/logstash-template.json"
-    }
-  }
-}

salt/logstash/files/dynamic/9500_output_beats.conf

@@ -1,25 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('node:mainip', '') -%}
-{%- endif %}
-# Author: Wes Lambert
-# Last Update: 09/14/2018
-filter {
-  if "beat" in [tags] {
-    mutate {
-          ##add_tag => [ "conf_file_9500"]
-        }
-  }
-}
-output {
-  if "beat" in [tags] {
-    elasticsearch {
-      hosts => "{{ ES }}"
-      index => "logstash-beats-%{+YYYY.MM.dd}"
-      template_name => "logstash-beats"
-      template => "/beats-template.json"
-      template_overwrite => true
-    }
-  }
-}

salt/logstash/files/dynamic/9600_output_ossec.conf

@@ -1,29 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('node:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Updated by: Doug Burks
-# Last Update: 9/19/2018
-
-filter {
-  if [event_type] =~ "ossec" {
-    mutate {
-          ##add_tag => [ "conf_file_9600"]
-        }
-  }
-}
-
-output {
-  if [event_type] =~ "ossec" or "ossec" in [tags] {
-    elasticsearch {
-      hosts => "{{ ES }}"
-      index => "logstash-ossec-%{+YYYY.MM.dd}"
-      template_name => "logstash-ossec"
-      template => "/logstash-ossec-template.json"
-      template_overwrite => true
-    }
-  }
-}

salt/logstash/files/dynamic/9997_output_helix.conf

@@ -1,142 +0,0 @@
-{% set HELIX_API_KEY = salt['pillar.get']('fireeye:helix:api_key', '') %}
-
-filter {
-  if [type] =~ /^bro_conn|bro_dns|bro_http|bro_files|bro_ssl|bro_dhcp|bro_x509$/ {
-    grok {
-      match => [
-        "source_ip", "^%{IPV4:srcipv4}$",
-        "source_ip",  "(?<srcipv6>^([0-9A-Fa-f]{0,4}:){2,7}([0-9A-Fa-f]{1,4}$|((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.|$)){4})$)"
-        ]
-    }
-    grok {
-      match => [
-        "destination_ip",  "(?<dstipv6>^([0-9A-Fa-f]{0,4}:){2,7}([0-9A-Fa-f]{1,4}$|((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.|$)){4})$)",
-        "destination_ip", "^%{IPV4:dstipv4}$"
-       ]
-    }
-
-    geoip {
-      source => "[source_ip]"
-      target => "source_geo"
-    }
-    geoip {
-      source => "[destination_ip]"
-      target => "destination_geo"
-    }
-    mutate {
-      #rename => { "%{[source_geo][country_code]}" => "srccountrycode" }
-      #rename => { "%{[destination_geo][country_code]}" => "dstcountrycode" }
-      rename => { "[beat_host][name]" => "sensor" }
-      copy => { "sensor" => "rawmsghostname" }
-      rename => { "message" => "rawmsg" }
-      #rename => { "event_type" => "program" }
-      copy => { "type" => "class" }
-      copy => { "class" => "program"}
-      rename => { "source_port" => "srcport" }
-      rename => { "destination_port" => "dstport" }
-      remove_field => ["source_ip", "destination_ip"]
-      remove_field => ["sensorname", "sensor_name", "service", "source", "tags", "syslog-host"]
-      remove_field => ["sensor_name", "source_ips", "ips", "destination_ips", "syslog-priority", "syslog-file_name", "syslog-facility"]
-    }
-    if "bro_conn" in [class] {
-      mutate {
-        #add_field => { "metaclass" => "connection" }
-        rename => { "original_bytes" => "sentbytes" }
-        rename => { "respond_bytes" => "rcvdbytes" }
-        rename => { "connection_state" => "connstate" }
-        rename => { "uid" => "connectionid" }
-        rename => { "respond_packets" => "rcvdpackets" }
-        rename => { "original_packets" => "sentpackets" }
-        rename => { "respond_ip_bytes" => "rcvdipbytes" }
-        rename => { "original_ip_bytes" => "sentipbytes" }
-        rename => { "local_respond" => "local_resp" }
-        rename => { "local_orig" => "localorig" }
-        rename => { "missed_bytes" => "missingbytes" }
-      }
-    }
-    if "bro_dns" in [class] {
-      mutate{
-        #add_field = { "metaclass" => "dns"}
-        rename => { "answers" => "answer" }
-        rename => { "query" => "domain" }
-        rename => { "query_class" => "queryclass" }
-        rename => { "query_class_name" => "queryclassname" }
-        rename => { "query_type" => "querytype" }
-        rename => { "query_type_name" => "querytypename" }
-        rename => { "ra" => "recursionavailable" }
-        rename => { "rd" => "recursiondesired" }
-      }
-    }
-    if "bro_dhcp" in [class] {
-      mutate{
-        #add_field = { "metaclass" => "dhcp"}
-        rename => { "message_types" => "direction" }
-				rename => { "lease_time" => "duration" }
-      }
-    }
-    if "bro_files" in [class] {
-      mutate{
-        #add_field = { "metaclass" => "dns"}
-        rename => { "missing_bytes" => "missingbytes" }
-        rename => { "fuid" => "fileid" }
-        rename => { "uid" => "connectionid" }
-      }
-    }
-    if "bro_http" in [class] {
-      mutate{
-        #add_field = { "metaclass" => "dns"}
-        rename => { "virtual_host" => "hostname" }
-        rename => { "status_code" => "statuscode" }
-        rename => { "status_message" => "statusmsg" }
-        rename => { "resp_mime_types" => "rcvdmimetype" }
-        rename => { "resp_fuids" => "rcvdfileid" }
-        rename => { "response_body_len" => "rcvdbodybytes" }
-        rename => { "request_body_len" => "sentbodybytes" }
-        rename => { "uid" => "connectionid" }
-        rename => { "ts"=> "eventtime" }
-	      rename => { "@timestamp"=> "eventtime" }
-      }
-    }
-    if "bro_ssl" in [class] {
-      mutate{
-        #add_field = { "metaclass" => "dns"}
-        rename => { "status_code" => "statuscode" }
-        rename => { "status_message" => "statusmsg" }
-        rename => { "resp_mime_types" => "rcvdmimetype" }
-        rename => { "resp_fuids" => "rcvdfileid" }
-        rename => { "response_body_len" => "rcvdbodybytes" }
-        rename => { "request_body_len" => "sentbodybytes" }
-      }
-    }
-		if "bro_weird" in [class] {
-      mutate{
-        #add_field = { "metaclass" => "dns"}
-        rename => { "name" => "eventname" }
-      }
-    }
-		if "bro_x509" in [class] {
-      mutate{
-        #add_field = { "metaclass" => "dns"}
-				rename => { "certificate_common_name" => "certname" }
-        rename => { "certificate_subject" => "certsubject" }
-				rename => { "issuer_common_name" => "issuer" }
-				rename => { "certificate_issuer" => "issuersubject" }
-				rename => { "certificate_not_valid_before" => "issuetime" }
-				rename => { "certificate_key_type" => "cert_type" }
-      }
-    }
-  }
-}
-
-output {
-  if [type] =~ /^bro_conn|bro_dns|bro_http|bro_files|bro_ssl|bro_dhcp|bro_x509$/ {
-    http {
-      url => "https://helix-integrations.cloud.aws.apps.fireeye.com/api/upload"
-      http_method => post
-      http_compression => true
-      socket_timeout => 60
-      headers => ["Authorization","{{ HELIX_API_KEY }}"]
-      format => json_batch
-    }
-  }
-}

salt/logstash/files/dynamic/9998_output_test_data.conf

@@ -1,26 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('master:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('node:mainip', '') -%}
-{%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Email: justin@hasecuritysolution.com
-# Last Update: 12/9/2016
-
-filter {
-  if "test_data" in [tags] {
-    mutate {
-	  #add_tag => [ "conf_file_9998"]
-	}
-  }
-}
-output {
-  if "test_data" in [tags] {
-    elasticsearch {
-      hosts => "{{ ES }}"
-      index => "logstash-test-%{+YYYY.MM.dd}"
-      template => "/logstash-template.json"
-    }
-  }
-}

salt/logstash/init.sls

@@ -55,6 +55,8 @@
 {% endif %}
 
 {% set PIPELINES = salt['pillar.get']('logstash:pipelines', {}) %}
+{% set TEMPLATES = salt['pillar.get']('logstash:templates', {}) %}
+{% set DOCKER_OPTIONS = salt['pillar.get']('logstash:docker_options', {}) %}
 
 # Create the logstash group
 logstashgroup:
@@ -69,21 +71,6 @@ logstash:
     - gid: 931
     - home: /opt/so/conf/logstash
 
-# Create a directory for people to drop their own custom parsers into
-lscustdir:
-  file.directory:
-    - name: /opt/so/conf/logstash/custom
-    - user: 931
-    - group: 939
-    - makedirs: True
-
-lsdyndir:
-  file.directory:
-    - name: /opt/so/conf/logstash/dynamic
-    - user: 931
-    - group: 939
-    - makedirs: True
-
 lsetcdir:
   file.directory:
     - name: /opt/so/conf/logstash/etc
@@ -91,19 +78,11 @@ lsetcdir:
     - group: 939
     - makedirs: True
 
-lscustparserdir:
+lspipelinedir:
   file.directory:
-    - name: /opt/so/conf/logstash/custom/parsers
+    - name: /opt/so/conf/logstash/pipelines
     - user: 931
     - group: 939
-    - makedirs: True
-
-lscusttemplatedir:
-  file.directory:
-    - name: /opt/so/conf/logstash/custom/templates
-    - user: 931
-    - group: 939
-    - makedirs: True
 
 {% for PL in PIPELINES %}
 ls_pipeline_{{PL}}:
@@ -113,20 +92,35 @@ ls_pipeline_{{PL}}:
     - group: 939
 
   {% for CONFIGFILE in PIPELINES[PL].config %}
-ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0]}}:
+ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}:
   file.managed:
     - source: salt://logstash/pipelines/config/{{CONFIGFILE}}
     {% if 'jinja' in CONFIGFILE.split('.')[-1] %}
-    - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE | replace(".jinja", "")}}
+    - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE.split('/')[1] | replace(".jinja", "")}}
     - template: jinja
     {% else %}
-    - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE}}
+    - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE.split('/')[1]}}
     {% endif %}
     - user: 931
     - group: 939
   {% endfor %}
 {% endfor %}
 
+#sync templates to /opt/so/conf/logstash/etc/ here
+{% for TEMPLATE in TEMPLATES %}
+ls_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
+  file.managed:
+    - source: salt://logstash/pipelines/templates/{{TEMPLATE}}
+    {% if 'jinja' in TEMPLATE.split('.')[-1] %}
+    - name: /opt/so/conf/logstash/etc/{{TEMPLATE.split('/')[1] | replace(".jinja", "")}}
+    - template: jinja
+    {% else %}
+    - name: /opt/so/conf/logstash/etc/{{TEMPLATE.split('/')[1]}}
+    {% endif %}
+    - user: 931
+    - group: 939
+{% endfor %}
+
 lspipelinesyml:
   file.managed:
     - name: /opt/so/conf/logstash/etc/pipelines.yml
@@ -145,21 +139,6 @@ lsetcsync:
     - template: jinja
     - exclude_pat: pipelines*
 
-lssync:
-  file.recurse:
-    - name: /opt/so/conf/logstash/dynamic
-    - source: salt://logstash/files/dynamic
-    - user: 931
-    - group: 939
-    - template: jinja
-
-lscustsync:
-  file.recurse:
-    - name: /opt/so/conf/logstash/custom
-    - source: salt://logstash/files/custom
-    - user: 931
-    - group: 939
-
 # Create the import directory
 importdir:
   file.directory:
@@ -193,14 +172,9 @@ so-logstash:
     - environment:
       - LS_JAVA_OPTS=-Xms{{ lsheap }} -Xmx{{ lsheap }}
     - port_bindings:
-      - 0.0.0.0:514:514
+{% for BINDING in DOCKER_OPTIONS.port_bindings %}
-      - 0.0.0.0:5044:5044
+      - {{ BINDING }}
-      - 0.0.0.0:5644:5644
+{% endfor %}
-      - 0.0.0.0:6050:6050
-      - 0.0.0.0:6051:6051
-      - 0.0.0.0:6052:6052
-      - 0.0.0.0:6053:6053
-      - 0.0.0.0:9600:9600
     - binds:
       - /opt/so/conf/logstash/etc/log4j2.properties:/usr/share/logstash/config/log4j2.properties:ro
       - /opt/so/conf/logstash/etc/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
@@ -228,6 +202,5 @@ so-logstash:
       {%- endif %}
     - watch:
       - file: /opt/so/conf/logstash/etc
-      - file: /opt/so/conf/logstash/custom
+      - file: /opt/so/conf/logstash/pipelines
       #- file: /opt/so/conf/logstash/rulesets
-      - file: /opt/so/conf/logstash/dynamic

salt/logstash/pipelines/config/7100_osquery_wel.conf

@@ -1,23 +0,0 @@
-# Author: Josh Brower
-# Last Update: 12/28/2018
-# If log is tagged osquery and there is an eventid column - then cleanup and parse out the EventData column
-
-filter {
-  if "osquery" in [tags] and [osquery][columns][eventid] {
-
-        mutate {
-     gsub => ["[osquery][columns][data]", "\\x0A", ""]
-    }
-
-        json {
-      source => "[osquery][columns][data]"
-      target => "[osquery][columns][data]"
-     }
-
-        mutate {
-     merge => { "[osquery][columns]" => "[osquery][columns][data]" }
-     remove_field => ["[osquery][columns][data]"]
-        }
-
-  }
-}