pillarize logstash config,parsers,templates and docker port bindings

2026-05-03 09:58:17 +02:00 · 2020-02-25 17:44:32 -05:00
parent e2ccebd2fa
commit 0c4973ad77
130 changed files with 191 additions and 1081 deletions
@@ -2,54 +2,59 @@ logstash:
  pipelines:
    eval:
      config:
-        - 0800_input_eval.conf
-        - 1000_preprocess_log_elapsed.conf
-        - 1001_preprocess_syslogng.conf
-        - 1002_preprocess_json.conf
-        - 1004_preprocess_syslog_types.conf
-        - 1026_preprocess_dhcp.conf
-        - 1029_preprocess_esxi.conf
-        - 1030_preprocess_greensql.conf
-        - 1031_preprocess_iis.conf
-        - 1032_preprocess_mcafee.conf
-        - 1033_preprocess_snort.conf
-        - 1034_preprocess_syslog.conf
-        - 2000_network_flow.conf
-        - 6002_syslog.conf
-        - 6101_switch_brocade.conf
-        - 6200_firewall_fortinet.conf
-        - 6201_firewall_pfsense.conf
-        - 6300_windows.conf
-        - 6301_dns_windows.conf
-        - 6400_suricata.conf
-        - 6500_ossec.conf
-        - 6501_ossec_sysmon.conf
-        - 6502_ossec_autoruns.conf
-        - 6600_winlogbeat_sysmon.conf
-        - 6700_winlogbeat.conf
-        - 7100_osquery_wel.conf
-        - 7200_strelka.conf
-        - 8001_postprocess_common_ip_augmentation.conf
-        - 8007_postprocess_http.conf
-        - 8200_postprocess_tagging.conf
-        - 8998_postprocess_log_elapsed.conf
-        - 8999_postprocess_rename_type.conf
-        - 9000_output_bro.conf.jinja
-        - 9001_output_switch.conf.jinja
-        - 9002_output_import.conf.jinja
-        - 9004_output_flow.conf.jinja
-        - 9026_output_dhcp.conf.jinja
-        - 9029_output_esxi.conf.jinja
-        - 9030_output_greensql.conf.jinja
-        - 9031_output_iis.conf.jinja
-        - 9032_output_mcafee.conf.jinja
-        - 9033_output_snort.conf.jinja
-        - 9034_output_syslog.conf.jinja
-        - 9100_output_osquery.conf.jinja
-        - 9200_output_firewall.conf.jinja
-        - 9300_output_windows.conf.jinja
-        - 9301_output_dns_windows.conf.jinja
-        - 9400_output_suricata.conf.jinja
-        - 9500_output_beats.conf.jinja
-        - 9600_output_ossec.conf.jinja
-        - 9700_ouptut_strelka.conf.jinja
+        - so/0800_input_eval.conf
+        - so/1000_preprocess_log_elapsed.conf
+        - so/1001_preprocess_syslogng.conf
+        - so/1002_preprocess_json.conf
+        - so/1004_preprocess_syslog_types.conf
+        - so/1026_preprocess_dhcp.conf
+        - so/1029_preprocess_esxi.conf
+        - so/1030_preprocess_greensql.conf
+        - so/1031_preprocess_iis.conf
+        - so/1032_preprocess_mcafee.conf
+        - so/1033_preprocess_snort.conf
+        - so/1034_preprocess_syslog.conf
+        - so/2000_network_flow.conf
+        - so/6002_syslog.conf
+        - so/6101_switch_brocade.conf
+        - so/6200_firewall_fortinet.conf
+        - so/6201_firewall_pfsense.conf
+        - so/6300_windows.conf
+        - so/6301_dns_windows.conf
+        - so/6400_suricata.conf
+        - so/6500_ossec.conf
+        - so/6501_ossec_sysmon.conf
+        - so/6502_ossec_autoruns.conf
+        - so/6600_winlogbeat_sysmon.conf
+        - so/6700_winlogbeat.conf
+        - so/7100_osquery_wel.conf
+        - so/7200_strelka.conf
+        - so/8001_postprocess_common_ip_augmentation.conf
+        - so/8007_postprocess_http.conf
+        - so/8200_postprocess_tagging.conf
+        - so/8998_postprocess_log_elapsed.conf
+        - so/8999_postprocess_rename_type.conf
+        - so/9000_output_bro.conf.jinja
+        - so/9001_output_switch.conf.jinja
+        - so/9002_output_import.conf.jinja
+        - so/9004_output_flow.conf.jinja
+        - so/9026_output_dhcp.conf.jinja
+        - so/9029_output_esxi.conf.jinja
+        - so/9030_output_greensql.conf.jinja
+        - so/9031_output_iis.conf.jinja
+        - so/9032_output_mcafee.conf.jinja
+        - so/9033_output_snort.conf.jinja
+        - so/9034_output_syslog.conf.jinja
+        - so/9100_output_osquery.conf.jinja
+        - so/9200_output_firewall.conf.jinja
+        - so/9300_output_windows.conf.jinja
+        - so/9301_output_dns_windows.conf.jinja
+        - so/9400_output_suricata.conf.jinja
+        - so/9500_output_beats.conf.jinja
+        - so/9600_output_ossec.conf.jinja
+        - so/9700_output_strelka.conf.jinja
+  templates:
+    - so/beats-template.json
+    - so/logstash-ossec-template.json
+    - so/logstash-strelka-template.json
+    - so/logstash-template.json
@@ -2,41 +2,41 @@ logstash:
  pipelines:
    helix:
      config:
-        - 0010_input_hhbeats.conf
-        - 1033_preprocess_snort.conf
-        - 1100_preprocess_bro_conn.conf
-        - 1101_preprocess_bro_dhcp.conf
-        - 1102_preprocess_bro_dns.conf
-        - 1103_preprocess_bro_dpd.conf
-        - 1104_preprocess_bro_files.conf
-        - 1105_preprocess_bro_ftp.conf
-        - 1106_preprocess_bro_http.conf
-        - 1107_preprocess_bro_irc.conf
-        - 1108_preprocess_bro_kerberos.conf
-        - 1109_preprocess_bro_notice.conf
-        - 1110_preprocess_bro_rdp.conf
-        - 1111_preprocess_bro_signatures.conf
-        - 1112_preprocess_bro_smtp.conf
-        - 1113_preprocess_bro_snmp.conf
-        - 1114_preprocess_bro_software.conf
-        - 1115_preprocess_bro_ssh.conf
-        - 1116_preprocess_bro_ssl.conf
-        - 1117_preprocess_bro_syslog.conf
-        - 1118_preprocess_bro_tunnel.conf
-        - 1119_preprocess_bro_weird.conf
-        - 1121_preprocess_bro_mysql.conf
-        - 1122_preprocess_bro_socks.conf
-        - 1123_preprocess_bro_x509.conf
-        - 1124_preprocess_bro_intel.conf
-        - 1125_preprocess_bro_modbus.conf
-        - 1126_preprocess_bro_sip.conf
-        - 1127_preprocess_bro_radius.conf
-        - 1128_preprocess_bro_pe.conf
-        - 1129_preprocess_bro_rfb.conf
-        - 1130_preprocess_bro_dnp3.conf
-        - 1131_preprocess_bro_smb_files.conf
-        - 1132_preprocess_bro_smb_mapping.conf
-        - 1133_preprocess_bro_ntlm.conf
-        - 1134_preprocess_bro_dce_rpc.conf
-        - 8001_postprocess_common_ip_augmentation.conf
-        - 9997_output_helix.conf.jinja
+        - so/0010_input_hhbeats.conf
+        - so/1033_preprocess_snort.conf
+        - so/1100_preprocess_bro_conn.conf
+        - so/1101_preprocess_bro_dhcp.conf
+        - so/1102_preprocess_bro_dns.conf
+        - so/1103_preprocess_bro_dpd.conf
+        - so/1104_preprocess_bro_files.conf
+        - so/1105_preprocess_bro_ftp.conf
+        - so/1106_preprocess_bro_http.conf
+        - so/1107_preprocess_bro_irc.conf
+        - so/1108_preprocess_bro_kerberos.conf
+        - so/1109_preprocess_bro_notice.conf
+        - so/1110_preprocess_bro_rdp.conf
+        - so/1111_preprocess_bro_signatures.conf
+        - so/1112_preprocess_bro_smtp.conf
+        - so/1113_preprocess_bro_snmp.conf
+        - so/1114_preprocess_bro_software.conf
+        - so/1115_preprocess_bro_ssh.conf
+        - so/1116_preprocess_bro_ssl.conf
+        - so/1117_preprocess_bro_syslog.conf
+        - so/1118_preprocess_bro_tunnel.conf
+        - so/1119_preprocess_bro_weird.conf
+        - so/1121_preprocess_bro_mysql.conf
+        - so/1122_preprocess_bro_socks.conf
+        - so/1123_preprocess_bro_x509.conf
+        - so/1124_preprocess_bro_intel.conf
+        - so/1125_preprocess_bro_modbus.conf
+        - so/1126_preprocess_bro_sip.conf
+        - so/1127_preprocess_bro_radius.conf
+        - so/1128_preprocess_bro_pe.conf
+        - so/1129_preprocess_bro_rfb.conf
+        - so/1130_preprocess_bro_dnp3.conf
+        - so/1131_preprocess_bro_smb_files.conf
+        - so/1132_preprocess_bro_smb_mapping.conf
+        - so/1133_preprocess_bro_ntlm.conf
+        - so/1134_preprocess_bro_dce_rpc.conf
+        - so/8001_postprocess_common_ip_augmentation.conf
+        - so/9997_output_helix.conf.jinja
@@ -0,0 +1,11 @@
+logstash:
+  docker_options:
+    port_bindings:
+      - 0.0.0.0:514:514
+      - 0.0.0.0:5044:5044
+      - 0.0.0.0:5644:5644
+      - 0.0.0.0:6050:6050
+      - 0.0.0.0:6051:6051
+      - 0.0.0.0:6052:6052
+      - 0.0.0.0:6053:6053
+      - 0.0.0.0:9600:9600
@@ -2,5 +2,5 @@ logstash:
  pipelines:
    master:
      config:
-        - 0010_input_hhbeats.conf
-        - 9999_output_redis.conf.jinja
+        - so/0010_input_hhbeats.conf
+        - so/9999_output_redis.conf.jinja
@@ -2,54 +2,59 @@ logstash:
  pipelines:
    search:
      config:
-        - 1000_preprocess_log_elapsed.conf
-        - 1001_preprocess_syslogng.conf
-        - 1002_preprocess_json.conf
-        - 1004_preprocess_syslog_types.conf
-        - 1026_preprocess_dhcp.conf
-        - 1029_preprocess_esxi.conf
-        - 1030_preprocess_greensql.conf
-        - 1031_preprocess_iis.conf
-        - 1032_preprocess_mcafee.conf
-        - 1033_preprocess_snort.conf
-        - 1034_preprocess_syslog.conf
-        - 2000_network_flow.conf
-        - 6002_syslog.conf
-        - 6101_switch_brocade.conf
-        - 6200_firewall_fortinet.conf
-        - 6201_firewall_pfsense.conf
-        - 6300_windows.conf
-        - 6301_dns_windows.conf
-        - 6400_suricata.conf
-        - 6500_ossec.conf
-        - 6501_ossec_sysmon.conf
-        - 6502_ossec_autoruns.conf
-        - 6600_winlogbeat_sysmon.conf
-        - 6700_winlogbeat.conf
-        - 7100_osquery_wel.conf
-        - 7200_strelka.conf
-        - 8001_postprocess_common_ip_augmentation.conf
-        - 8007_postprocess_http.conf
-        - 8200_postprocess_tagging.conf
-        - 8998_postprocess_log_elapsed.conf
-        - 8999_postprocess_rename_type.conf
-        - 0900_input_redis.conf.jinja
-        - 9000_output_bro.conf.jinja
-        - 9001_output_switch.conf.jinja
-        - 9002_output_import.conf.jinja
-        - 9004_output_flow.conf.jinja
-        - 9026_output_dhcp.conf.jinja
-        - 9029_output_esxi.conf.jinja
-        - 9030_output_greensql.conf.jinja
-        - 9031_output_iis.conf.jinja
-        - 9032_output_mcafee.conf.jinja
-        - 9033_output_snort.conf.jinja
-        - 9034_output_syslog.conf.jinja
-        - 9100_output_osquery.conf.jinja
-        - 9200_output_firewall.conf.jinja
-        - 9300_output_windows.conf.jinja
-        - 9301_output_dns_windows.conf.jinja
-        - 9400_output_suricata.conf.jinja
-        - 9500_output_beats.conf.jinja
-        - 9600_output_ossec.conf.jinja
-        - 9700_output_strelka.conf.jinja
+        - so/1000_preprocess_log_elapsed.conf
+        - so/1001_preprocess_syslogng.conf
+        - so/1002_preprocess_json.conf
+        - so/1004_preprocess_syslog_types.conf
+        - so/1026_preprocess_dhcp.conf
+        - so/1029_preprocess_esxi.conf
+        - so/1030_preprocess_greensql.conf
+        - so/1031_preprocess_iis.conf
+        - so/1032_preprocess_mcafee.conf
+        - so/1033_preprocess_snort.conf
+        - so/1034_preprocess_syslog.conf
+        - so/2000_network_flow.conf
+        - so/6002_syslog.conf
+        - so/6101_switch_brocade.conf
+        - so/6200_firewall_fortinet.conf
+        - so/6201_firewall_pfsense.conf
+        - so/6300_windows.conf
+        - so/6301_dns_windows.conf
+        - so/6400_suricata.conf
+        - so/6500_ossec.conf
+        - so/6501_ossec_sysmon.conf
+        - so/6502_ossec_autoruns.conf
+        - so/6600_winlogbeat_sysmon.conf
+        - so/6700_winlogbeat.conf
+        - so/7100_osquery_wel.conf
+        - so/7200_strelka.conf
+        - so/8001_postprocess_common_ip_augmentation.conf
+        - so/8007_postprocess_http.conf
+        - so/8200_postprocess_tagging.conf
+        - so/8998_postprocess_log_elapsed.conf
+        - so/8999_postprocess_rename_type.conf
+        - so/0900_input_redis.conf.jinja
+        - so/9000_output_bro.conf.jinja
+        - so/9001_output_switch.conf.jinja
+        - so/9002_output_import.conf.jinja
+        - so/9004_output_flow.conf.jinja
+        - so/9026_output_dhcp.conf.jinja
+        - so/9029_output_esxi.conf.jinja
+        - so/9030_output_greensql.conf.jinja
+        - so/9031_output_iis.conf.jinja
+        - so/9032_output_mcafee.conf.jinja
+        - so/9033_output_snort.conf.jinja
+        - so/9034_output_syslog.conf.jinja
+        - so/9100_output_osquery.conf.jinja
+        - so/9200_output_firewall.conf.jinja
+        - so/9300_output_windows.conf.jinja
+        - so/9301_output_dns_windows.conf.jinja
+        - so/9400_output_suricata.conf.jinja
+        - so/9500_output_beats.conf.jinja
+        - so/9600_output_ossec.conf.jinja
+        - so/9700_output_strelka.conf.jinja
+  templates:
+    - so/beats-template.json
+    - so/logstash-ossec-template.json
+    - so/logstash-strelka-template.json
+    - so/logstash-template.json