Merge pull request #494 from Security-Onion-Solutions/search_ls_pillar

Search ls pillar

pillar/logstash/search.sls

@@ -2,59 +2,15 @@ logstash:
   pipelines:
     search:
       config:
-        - so/1000_preprocess_log_elapsed.conf
-        - so/1001_preprocess_syslogng.conf
-        - so/1002_preprocess_json.conf
-        - so/1004_preprocess_syslog_types.conf
-        - so/1026_preprocess_dhcp.conf
-        - so/1029_preprocess_esxi.conf
-        - so/1030_preprocess_greensql.conf
-        - so/1031_preprocess_iis.conf
-        - so/1032_preprocess_mcafee.conf
-        - so/1033_preprocess_snort.conf
-        - so/1034_preprocess_syslog.conf
-        - so/2000_network_flow.conf
-        - so/6002_syslog.conf
-        - so/6101_switch_brocade.conf
-        - so/6200_firewall_fortinet.conf
-        - so/6201_firewall_pfsense.conf
-        - so/6300_windows.conf
-        - so/6301_dns_windows.conf
-        - so/6400_suricata.conf
-        - so/6500_ossec.conf
-        - so/6501_ossec_sysmon.conf
-        - so/6502_ossec_autoruns.conf
-        - so/6600_winlogbeat_sysmon.conf
-        - so/6700_winlogbeat.conf
-        - so/7100_osquery_wel.conf
-        - so/7200_strelka.conf
-        - so/8001_postprocess_common_ip_augmentation.conf
-        - so/8007_postprocess_http.conf
-        - so/8200_postprocess_tagging.conf
-        - so/8998_postprocess_log_elapsed.conf
-        - so/8999_postprocess_rename_type.conf
-        - so/0900_input_redis.conf.jinja
+	- so/0900_input_redis.conf.jinja
         - so/9000_output_bro.conf.jinja
-        - so/9001_output_switch.conf.jinja
         - so/9002_output_import.conf.jinja
-        - so/9004_output_flow.conf.jinja
-        - so/9026_output_dhcp.conf.jinja
-        - so/9029_output_esxi.conf.jinja
-        - so/9030_output_greensql.conf.jinja
-        - so/9031_output_iis.conf.jinja
-        - so/9032_output_mcafee.conf.jinja
-        - so/9033_output_snort.conf.jinja
-        - so/9034_output_syslog.conf.jinja
         - so/9100_output_osquery.conf.jinja
-        - so/9200_output_firewall.conf.jinja
-        - so/9300_output_windows.conf.jinja
-        - so/9301_output_dns_windows.conf.jinja
         - so/9400_output_suricata.conf.jinja
         - so/9500_output_beats.conf.jinja
         - so/9600_output_ossec.conf.jinja
         - so/9700_output_strelka.conf.jinja
   templates:
-    - so/beats-template.json
-    - so/logstash-ossec-template.json
-    - so/logstash-strelka-template.json
-    - so/logstash-template.json
+    - so/so-beats-template.json
+    - so/so-common-template.json
+    - so/so-zeek-template.json

salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja

@@ -10,17 +10,16 @@
 
 
 filter {
-  if "zeek" in [tags] and "test_data" not in [tags] and "import" not in [tags] {
+  if [module] =~ "zeek" {
     mutate {
 	  ##add_tag => [ "conf_file_9000"]
 	}
   }
 }
 output {
-  if "zeek" in [tags] and "test_data" not in [tags] and "import" not in [tags] {
-#    stdout { codec => rubydebug }
+  if [module] =~ "zeek" {
     elasticsearch {
-      pipeline => "%{event_type}"
+      pipeline => "%{module}.%{dataset}"
       hosts => "{{ ES }}"
       index => "so-zeek-%{+YYYY.MM.dd}"
       template_name => "so-zeek"

salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja

@@ -9,8 +9,9 @@
 
 
 output {
-  if "osquery" in [tags] {
+  if [module] =~ "osquery" {
     elasticsearch {
+      pipeline => "%{module}.%{dataset}" 
       hosts => "{{ ES }}"
       index => "so-osquery-%{+YYYY.MM.dd}"
       template => "/so-common-template.json"

salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja

@@ -9,16 +9,16 @@
 # Last Update: 12/9/2016
 
 filter {
-  if [event_type] == "suricata" and "test_data" not in [tags] {
+  if [module] == "suricata" {
     mutate {
 	  ##add_tag => [ "conf_file_9400"]
 	}
   }
 }
 output {
-  if [event_type] == "suricata" and "test_data" not in [tags] {
-    #stdout { codec => rubydebug }
+  if [module] =~ "suricata" {
     elasticsearch {
+      pipeline => "%{module}.%{dataset}"
       hosts => "{{ ES }}"
       index => "so-ids-%{+YYYY.MM.dd}"
       template => "/so-common-template.json"

salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja

@@ -9,7 +9,7 @@
 # Last Update: 9/19/2018
 
 filter {
-  if [event_type] =~ "ossec" {
+  if [module] =~ "ossec" {
     mutate {
           ##add_tag => [ "conf_file_9600"]
         }
@@ -17,9 +17,9 @@ filter {
 }
 
 output {
-  if [event_type] =~ "ossec" or "ossec" in [tags] {
+  if [module] =~ "ossec" {
     elasticsearch {
-      pipeline => "%{event_type}"
+      pipeline => "%{module}.%{dataset}"
       hosts => "{{ ES }}"
       index => "so-ossec-%{+YYYY.MM.dd}"
       template_name => "so-common"

salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja

@@ -10,7 +10,7 @@
 
 
 filter {
-  if [event_type] =~ "strelka" {
+  if [module] =~ "strelka" {
     mutate {
           ##add_tag => [ "conf_file_9000"]
         }
@@ -19,6 +19,7 @@ filter {
 output {
   if [event_type] =~ "strelka" {
     elasticsearch {
+      pipeline => "%{module}.%{dataset}" 
       hosts => "{{ ES }}"
       index => "so-strelka-%{+YYYY.MM.dd}"
       template_name => "so-common"