Add additional config for Filebeat modules

pillar/zeek/init.sls

@@ -52,5 +52,4 @@ zeek:
       - frameworks/signatures/detect-windows-shells
     redef:
       - LogAscii::use_json = T;
-      - LogAscii::json_timestamps = JSON::TS_ISO8601;
       - CaptureLoss::watch_interval = 5 mins;

salt/filebeat/etc/filebeat.yml

@@ -105,84 +105,6 @@ filebeat.inputs:
   fields_under_root: true
 
 {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor',  'so-helix', 'so-heavynode', 'so-import'] %}
-  {%- if ZEEKVER != 'SURICATA' %}
-    {%- for LOGNAME in salt['pillar.get']('zeeklogs:enabled', '')  %}
-- type: log
-  paths:
-    - /nsm/zeek/logs/current/{{ LOGNAME }}.log
-  fields:
-      module: zeek 
-      dataset: {{ LOGNAME }}
-      category: network
-  processors:
-  - drop_fields:
-      fields: ["source", "prospector", "input", "offset", "beat"]
- 
-  fields_under_root: true
-  clean_removed: true
-  close_removed: false
-
-- type: log
-  paths:
-    - /nsm/import/*/zeek/logs/{{ LOGNAME }}.log
-  fields:
-      module: zeek 
-      dataset: {{ LOGNAME }}
-      category: network
-      imported: true
-  processors:
-  - add_tags:
-      tags: ["import"]
-  - dissect:
-      tokenizer: "/nsm/import/%{import.id}/zeek/logs/%{import.file}"
-      field: "log.file.path"
-      target_prefix: ""
-  - drop_fields:
-      fields: ["source", "prospector", "input", "offset", "beat"]
- 
-  fields_under_root: true
-  clean_removed: false
-  close_removed: false
-    {%- endfor %}
-  {%- endif %}
-
-- type: log
-  paths:
-    - /nsm/suricata/eve*.json
-  fields:
-    module: suricata
-    dataset: common
-    category: network
-
-  processors:
-  - drop_fields:
-      fields: ["source", "prospector", "input", "offset", "beat"]
-
-  fields_under_root: true
-  clean_removed: false
-  close_removed: false
-
-- type: log
-  paths:
-    - /nsm/import/*/suricata/eve*.json
-  fields:
-    module: suricata
-    dataset: common
-    category: network
-    imported: true 
-  processors:
-  - add_tags:
-      tags: ["import"]
-  - dissect:
-      tokenizer: "/nsm/import/%{import.id}/suricata/%{import.file}"
-      field: "log.file.path"
-      target_prefix: ""
-  - drop_fields:
-      fields: ["source", "prospector", "input", "offset", "beat"]
-
-  fields_under_root: true
-  clean_removed: false
-  close_removed: false
 
   {%- if STRELKAENABLED == 1 %}
 - type: log

salt/filebeat/init.sls

@@ -26,6 +26,12 @@ filebeatetcdir:
     - user: 939
     - group: 939
     - makedirs: True
+filebeatmoduledir:
+  file.directory:
+    - name: /opt/so/conf/filebeat/modules
+    - user: root
+    - group: root
+    - makedirs: True
 filebeatlogdir:
   file.directory:
     - name: /opt/so/log/filebeat
@@ -55,6 +61,21 @@ filebeatconfsync:
     - defaults:
         INPUTS: {{ salt['pillar.get']('filebeat:config:inputs', {}) }}
         OUTPUT: {{ salt['pillar.get']('filebeat:config:output', {}) }}
+# Filebeat module config file
+filebeatmoduleconfsync:
+  file.managed:
+    - name: /opt/so/conf/filebeat/etc/module-setup.yml
+    - source: salt://filebeat/etc/module-setup.yml
+    - user: root
+    - group: root
+    - template: jinja
+# Sync Filebeat modules
+filebeatmodules:
+  file.recurse:
+    - name: /opt/so/conf/filebeat/modules
+    - source: salt://filebeat/modules
+    - user: root
+    - group: root
 so-filebeat:
   docker_container.running:
     - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }}
@@ -65,8 +86,10 @@ so-filebeat:
       - /nsm:/nsm:ro
       - /opt/so/log/filebeat:/usr/share/filebeat/logs:rw
       - /opt/so/conf/filebeat/etc/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
+      - /opt/so/conf/filebeat/etc/module-setup.yml:/usr/share/filebeat/module-setup.yml:ro
       - /nsm/wazuh/logs/alerts:/wazuh/alerts:ro
       - /nsm/wazuh/logs/archives:/wazuh/archives:ro
+      - /opt/so/conf/filebeat/modules:/usr/share/filebeat/modules.d
       - /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro
       - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro
       - /opt/so/conf/filebeat/registry:/usr/share/filebeat/data/registry:rw

salt/filebeat/modules/suricata.yml.disabled

@@ -1,11 +0,0 @@
-# Module: suricata
-# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-suricata.html
-
-- module: suricata
-  # All logs
-  eve:
-    enabled: true
-
-    # Set custom paths for the log files. If left empty,
-    # Filebeat will choose the paths depending on your OS.
-    #var.paths:

salt/filebeat/modules/zeek.yml.disabled

@@ -1,84 +0,0 @@
-# Module: zeek
-# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-zeek.html
-
-- module: zeek
-  capture_loss:
-    enabled: true
-  connection:
-    enabled: true
-  dce_rpc:
-    enabled: true
-  dhcp:
-    enabled: true
-  dnp3:
-    enabled: true
-  dns:
-    enabled: true
-  dpd:
-    enabled: true
-  files:
-    enabled: true
-  ftp:
-    enabled: true
-  http:
-    enabled: true
-  intel:
-    enabled: true
-  irc:
-    enabled: true
-  kerberos:
-    enabled: true
-  modbus:
-    enabled: true
-  mysql:
-    enabled: true
-  notice:
-    enabled: true
-  ntlm:
-    enabled: true
-  ocsp:
-    enabled: true
-  pe:
-    enabled: true
-  radius:
-    enabled: true
-  rdp:
-    enabled: true
-  rfb:
-    enabled: true
-  signature:
-    enabled: true
-  sip:
-    enabled: true
-  smb_cmd:
-    enabled: true
-  smb_files:
-    enabled: true
-  smb_mapping:
-    enabled: true
-  smtp:
-    enabled: true
-  snmp:
-    enabled: true
-  socks:
-    enabled: true
-  ssh:
-    enabled: true
-  ssl:
-    enabled: true
-  stats:
-    enabled: true
-  syslog:
-    enabled: true
-  traceroute:
-    enabled: true
-  tunnel:
-    enabled: true
-  weird:
-    enabled: true
-  x509:
-    enabled: true
-
-    # Set custom paths for the log files. If left empty,
-    # Filebeat will choose the paths depending on your OS.
-    #var.paths:

salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja

@@ -1,16 +1,17 @@
-{%- if grains['role'] == 'so-eval' -%}
+%- if grains['role'] == 'so-eval' -%}
 {%- set ES = salt['pillar.get']('manager:mainip', '') -%}
 {%- else %}
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
 output {
-  if [module] =~ "zeek" and "import" not in [tags] {
+ if [event][module] == 'zeek' {
    elasticsearch {
-      pipeline => "%{module}.%{dataset}"
+     id => "zeek_logs"
+     pipeline => "filebeat-%{[agent][version]}-zeek-%{[fileset][name]}-pipeline"
      hosts => "{{ ES }}"
-      index => "so-zeek"
+     index => "so-zeek-%{+YYYY.MM.dd}"
-      template_name => "so-zeek"
+     template_name => "so-common"
-      template => "/templates/so-zeek-template.json"
+     template => "/templates/so-common-template.json"
      template_overwrite => true
      ssl => true
      ssl_certificate_verification => false

salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja

@@ -4,9 +4,9 @@
 {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
 {%- endif %}
 output {
-  if [module] =~ "suricata" and "import" not in [tags] {
+  if [event][module] =~ "suricata" and "import" not in [tags] {
     elasticsearch {
-      pipeline => "%{module}.%{dataset}"
+      pipeline => "filebeat-%{[agent][version]}-suricata-%{[fileset][name]}-pipeline"
       hosts => "{{ ES }}"
       index => "so-ids"
       template_name => "so-ids"