Dynamix Pipelines take 1

pillar/logstash/search.sls

@@ -7,7 +7,7 @@ logstash:
         - so/9000_output_zeek.conf.jinja
         - so/9002_output_import.conf.jinja
         - so/9034_output_syslog.conf.jinja
-        - so/9050_output_elasticsearch.jinja
+        - so/9050_output_filebeatmodules.conf.jinja
         - so/9100_output_osquery.conf.jinja  
         - so/9400_output_suricata.conf.jinja
         - so/9500_output_beats.conf.jinja

salt/logstash/pipelines/config/so/0009_input_beats.conf

@@ -3,4 +3,9 @@ input {
     port => "5044"
     tags => [ "beat-ext" ]
   }
+}
+filter {
+  mutate {
+    rename => {“@metadata” => “metadata”}
+  }
 }

salt/logstash/pipelines/config/so/9050_output_elasticsearch_log.jinja

@@ -1,20 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
-output {
- if [event][module] == 'elasticsearch' {
-   elasticsearch {
-     id => "elastic_logs"
-     pipeline => "filebeat-%{[agent][version]}-elasticsearch-%{[fileset][name]}-pipeline"
-     hosts => "{{ ES }}"
-     index => "so-grid-%{+YYYY.MM.dd}"
-     template_name => "so-common"
-     template => "/templates/so-common-template.json"
-     template_overwrite => true
-     ssl => true
-     ssl_certificate_verification => false
-   }
- }
-}

salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja

@@ -4,12 +4,12 @@
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
 output {
- if [event][module] == 'redis' {
+ if [metadata][pipeline] {
    elasticsearch {
-     id => "redis_logs"
+     id => "filebeat_modules_metadata_pipeline"
-     pipeline => "filebeat-%{[agent][version]}-redis-%{[fileset][name]}-pipeline"
+     pipeline => "%{[metadata][pipeline]}"
      hosts => "{{ ES }}"
-     index => "so-grid-%{+YYYY.MM.dd}"
+     index => "so-%{[event][module]}-%{+YYYY.MM.dd}"
      template_name => "so-common"
      template => "/templates/so-common-template.json"
      template_overwrite => true
@@ -17,4 +17,4 @@ output {
      ssl_certificate_verification => false
    }
  }
-}
+}

salt/logstash/pipelines/config/so/9050_output_kibana_log.jinja

@@ -1,20 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
-output {
- if [event][module] == 'kibana' {
-   elasticsearch {
-     id => "kibana_logs"
-     pipeline => "filebeat-%{[agent][version]}-kibana-%{[fileset][name]}-pipeline"
-     hosts => "{{ ES }}"
-     index => "so-grid-%{+YYYY.MM.dd}"
-     template_name => "so-common"
-     template => "/templates/so-common-template.json"
-     template_overwrite => true
-     ssl => true
-     ssl_certificate_verification => false
-   }
- }
-}

salt/logstash/pipelines/config/so/9050_output_logstash_log.jinja

@@ -1,20 +0,0 @@
-{%- if grains['role'] == 'so-eval' -%}
-{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
-{%- else %}
-{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
-{%- endif %}
-output {
- if [event][module] == 'logstash' {
-   elasticsearch {
-     id => "logstash_logs"
-     pipeline => "filebeat-%{[agent][version]}-logstash-%{[fileset][name]}-pipeline"
-     hosts => "{{ ES }}"
-     index => "so-grid-%{+YYYY.MM.dd}"
-     template_name => "so-common"
-     template => "/templates/so-common-template.json"
-     template_overwrite => true
-     ssl => true
-     ssl_certificate_verification => false
-   }
- }
-}