diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index 8306ec58b..10fab2ed1 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -7,7 +7,7 @@ logstash: - so/9000_output_zeek.conf.jinja - so/9002_output_import.conf.jinja - so/9034_output_syslog.conf.jinja - - so/9050_output_elasticsearch.jinja + - so/9050_output_filebeatmodules.conf.jinja - so/9100_output_osquery.conf.jinja - so/9400_output_suricata.conf.jinja - so/9500_output_beats.conf.jinja diff --git a/salt/logstash/pipelines/config/so/0009_input_beats.conf b/salt/logstash/pipelines/config/so/0009_input_beats.conf index a5c1d491c..31ba798c9 100644 --- a/salt/logstash/pipelines/config/so/0009_input_beats.conf +++ b/salt/logstash/pipelines/config/so/0009_input_beats.conf @@ -3,4 +3,9 @@ input { port => "5044" tags => [ "beat-ext" ] } +} +filter { + mutate { + rename => {“@metadata” => “metadata”} + } } \ No newline at end of file diff --git a/salt/logstash/pipelines/config/so/9050_output_elasticsearch_log.jinja b/salt/logstash/pipelines/config/so/9050_output_elasticsearch_log.jinja deleted file mode 100644 index eb0d8ef0c..000000000 --- a/salt/logstash/pipelines/config/so/9050_output_elasticsearch_log.jinja +++ /dev/null @@ -1,20 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('manager:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -output { - if [event][module] == 'elasticsearch' { - elasticsearch { - id => "elastic_logs" - pipeline => "filebeat-%{[agent][version]}-elasticsearch-%{[fileset][name]}-pipeline" - hosts => "{{ ES }}" - index => "so-grid-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/templates/so-common-template.json" - template_overwrite => true - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9050_output_redis_log.jinja b/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja similarity index 70% rename from salt/logstash/pipelines/config/so/9050_output_redis_log.jinja rename to salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja index 9cc37de35..20e9f0c0a 100644 --- a/salt/logstash/pipelines/config/so/9050_output_redis_log.jinja +++ b/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja @@ -4,12 +4,12 @@ {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} output { - if [event][module] == 'redis' { + if [metadata][pipeline] { elasticsearch { - id => "redis_logs" - pipeline => "filebeat-%{[agent][version]}-redis-%{[fileset][name]}-pipeline" + id => "filebeat_modules_metadata_pipeline" + pipeline => "%{[metadata][pipeline]}" hosts => "{{ ES }}" - index => "so-grid-%{+YYYY.MM.dd}" + index => "so-%{[event][module]}-%{+YYYY.MM.dd}" template_name => "so-common" template => "/templates/so-common-template.json" template_overwrite => true @@ -17,4 +17,4 @@ output { ssl_certificate_verification => false } } -} +} \ No newline at end of file diff --git a/salt/logstash/pipelines/config/so/9050_output_kibana_log.jinja b/salt/logstash/pipelines/config/so/9050_output_kibana_log.jinja deleted file mode 100644 index bf79e1073..000000000 --- a/salt/logstash/pipelines/config/so/9050_output_kibana_log.jinja +++ /dev/null @@ -1,20 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('manager:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -output { - if [event][module] == 'kibana' { - elasticsearch { - id => "kibana_logs" - pipeline => "filebeat-%{[agent][version]}-kibana-%{[fileset][name]}-pipeline" - hosts => "{{ ES }}" - index => "so-grid-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/templates/so-common-template.json" - template_overwrite => true - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9050_output_logstash_log.jinja b/salt/logstash/pipelines/config/so/9050_output_logstash_log.jinja deleted file mode 100644 index df6fba0e0..000000000 --- a/salt/logstash/pipelines/config/so/9050_output_logstash_log.jinja +++ /dev/null @@ -1,20 +0,0 @@ -{%- if grains['role'] == 'so-eval' -%} -{%- set ES = salt['pillar.get']('manager:mainip', '') -%} -{%- else %} -{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} -{%- endif %} -output { - if [event][module] == 'logstash' { - elasticsearch { - id => "logstash_logs" - pipeline => "filebeat-%{[agent][version]}-logstash-%{[fileset][name]}-pipeline" - hosts => "{{ ES }}" - index => "so-grid-%{+YYYY.MM.dd}" - template_name => "so-common" - template => "/templates/so-common-template.json" - template_overwrite => true - ssl => true - ssl_certificate_verification => false - } - } -}