From 37929dbd7d09d0da03919541c792eb0cefa25fde Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Thu, 6 May 2021 13:54:28 +0000 Subject: [PATCH] Add additional config for Filebeat modules --- pillar/zeek/init.sls | 3 +- salt/filebeat/etc/filebeat.yml | 78 ----------------- salt/filebeat/init.sls | 23 +++++ salt/filebeat/modules/suricata.yml.disabled | 11 --- salt/filebeat/modules/zeek.yml.disabled | 84 ------------------- .../config/so/9000_output_zeek.conf.jinja | 27 +++--- .../config/so/9400_output_suricata.conf.jinja | 6 +- 7 files changed, 41 insertions(+), 191 deletions(-) delete mode 100644 salt/filebeat/modules/suricata.yml.disabled delete mode 100644 salt/filebeat/modules/zeek.yml.disabled diff --git a/pillar/zeek/init.sls b/pillar/zeek/init.sls index 30a59284a..5eeb273b9 100644 --- a/pillar/zeek/init.sls +++ b/pillar/zeek/init.sls @@ -52,5 +52,4 @@ zeek: - frameworks/signatures/detect-windows-shells redef: - LogAscii::use_json = T; - - LogAscii::json_timestamps = JSON::TS_ISO8601; - - CaptureLoss::watch_interval = 5 mins; \ No newline at end of file + - CaptureLoss::watch_interval = 5 mins; diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index 0f7c9c778..bd72bc583 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -105,84 +105,6 @@ filebeat.inputs: fields_under_root: true {%- if grains['role'] in ['so-eval', 'so-standalone', 'so-sensor', 'so-helix', 'so-heavynode', 'so-import'] %} - {%- if ZEEKVER != 'SURICATA' %} - {%- for LOGNAME in salt['pillar.get']('zeeklogs:enabled', '') %} -- type: log - paths: - - /nsm/zeek/logs/current/{{ LOGNAME }}.log - fields: - module: zeek - dataset: {{ LOGNAME }} - category: network - processors: - - drop_fields: - fields: ["source", "prospector", "input", "offset", "beat"] - - fields_under_root: true - clean_removed: true - close_removed: false - -- type: log - paths: - - /nsm/import/*/zeek/logs/{{ LOGNAME }}.log - fields: - module: zeek - dataset: {{ LOGNAME }} - category: network - imported: true - processors: - - add_tags: - tags: ["import"] - - dissect: - tokenizer: "/nsm/import/%{import.id}/zeek/logs/%{import.file}" - field: "log.file.path" - target_prefix: "" - - drop_fields: - fields: ["source", "prospector", "input", "offset", "beat"] - - fields_under_root: true - clean_removed: false - close_removed: false - {%- endfor %} - {%- endif %} - -- type: log - paths: - - /nsm/suricata/eve*.json - fields: - module: suricata - dataset: common - category: network - - processors: - - drop_fields: - fields: ["source", "prospector", "input", "offset", "beat"] - - fields_under_root: true - clean_removed: false - close_removed: false - -- type: log - paths: - - /nsm/import/*/suricata/eve*.json - fields: - module: suricata - dataset: common - category: network - imported: true - processors: - - add_tags: - tags: ["import"] - - dissect: - tokenizer: "/nsm/import/%{import.id}/suricata/%{import.file}" - field: "log.file.path" - target_prefix: "" - - drop_fields: - fields: ["source", "prospector", "input", "offset", "beat"] - - fields_under_root: true - clean_removed: false - close_removed: false {%- if STRELKAENABLED == 1 %} - type: log diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 64cdc47fc..8ab200276 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -26,6 +26,12 @@ filebeatetcdir: - user: 939 - group: 939 - makedirs: True +filebeatmoduledir: + file.directory: + - name: /opt/so/conf/filebeat/modules + - user: root + - group: root + - makedirs: True filebeatlogdir: file.directory: - name: /opt/so/log/filebeat @@ -55,6 +61,21 @@ filebeatconfsync: - defaults: INPUTS: {{ salt['pillar.get']('filebeat:config:inputs', {}) }} OUTPUT: {{ salt['pillar.get']('filebeat:config:output', {}) }} +# Filebeat module config file +filebeatmoduleconfsync: + file.managed: + - name: /opt/so/conf/filebeat/etc/module-setup.yml + - source: salt://filebeat/etc/module-setup.yml + - user: root + - group: root + - template: jinja +# Sync Filebeat modules +filebeatmodules: + file.recurse: + - name: /opt/so/conf/filebeat/modules + - source: salt://filebeat/modules + - user: root + - group: root so-filebeat: docker_container.running: - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }} @@ -65,8 +86,10 @@ so-filebeat: - /nsm:/nsm:ro - /opt/so/log/filebeat:/usr/share/filebeat/logs:rw - /opt/so/conf/filebeat/etc/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro + - /opt/so/conf/filebeat/etc/module-setup.yml:/usr/share/filebeat/module-setup.yml:ro - /nsm/wazuh/logs/alerts:/wazuh/alerts:ro - /nsm/wazuh/logs/archives:/wazuh/archives:ro + - /opt/so/conf/filebeat/modules:/usr/share/filebeat/modules.d - /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro - /opt/so/conf/filebeat/registry:/usr/share/filebeat/data/registry:rw diff --git a/salt/filebeat/modules/suricata.yml.disabled b/salt/filebeat/modules/suricata.yml.disabled deleted file mode 100644 index 1edd3f832..000000000 --- a/salt/filebeat/modules/suricata.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: suricata -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-suricata.html - -- module: suricata - # All logs - eve: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/filebeat/modules/zeek.yml.disabled b/salt/filebeat/modules/zeek.yml.disabled deleted file mode 100644 index 0667c6e35..000000000 --- a/salt/filebeat/modules/zeek.yml.disabled +++ /dev/null @@ -1,84 +0,0 @@ -# Module: zeek -# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-module-zeek.html - -- module: zeek - capture_loss: - enabled: true - connection: - enabled: true - dce_rpc: - enabled: true - dhcp: - enabled: true - dnp3: - enabled: true - dns: - enabled: true - dpd: - enabled: true - files: - enabled: true - ftp: - enabled: true - http: - enabled: true - intel: - enabled: true - irc: - enabled: true - kerberos: - enabled: true - modbus: - enabled: true - mysql: - enabled: true - notice: - enabled: true - ntlm: - enabled: true - ocsp: - enabled: true - pe: - enabled: true - radius: - enabled: true - rdp: - enabled: true - rfb: - enabled: true - signature: - enabled: true - sip: - enabled: true - smb_cmd: - enabled: true - smb_files: - enabled: true - smb_mapping: - enabled: true - smtp: - enabled: true - snmp: - enabled: true - socks: - enabled: true - ssh: - enabled: true - ssl: - enabled: true - stats: - enabled: true - syslog: - enabled: true - traceroute: - enabled: true - tunnel: - enabled: true - weird: - enabled: true - x509: - enabled: true - - # Set custom paths for the log files. If left empty, - # Filebeat will choose the paths depending on your OS. - #var.paths: diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja index d17dc2b22..da798a79d 100644 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja @@ -1,19 +1,20 @@ -{%- if grains['role'] == 'so-eval' -%} +%- if grains['role'] == 'so-eval' -%} {%- set ES = salt['pillar.get']('manager:mainip', '') -%} {%- else %} {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} output { - if [module] =~ "zeek" and "import" not in [tags] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ ES }}" - index => "so-zeek" - template_name => "so-zeek" - template => "/templates/so-zeek-template.json" - template_overwrite => true - ssl => true - ssl_certificate_verification => false - } - } + if [event][module] == 'zeek' { + elasticsearch { + id => "zeek_logs" + pipeline => "filebeat-%{[agent][version]}-zeek-%{[fileset][name]}-pipeline" + hosts => "{{ ES }}" + index => "so-zeek-%{+YYYY.MM.dd}" + template_name => "so-common" + template => "/templates/so-common-template.json" + template_overwrite => true + ssl => true + ssl_certificate_verification => false + } + } } diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja index 5013bafc1..258781f29 100644 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja @@ -4,12 +4,12 @@ {%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%} {%- endif %} output { - if [module] =~ "suricata" and "import" not in [tags] { + if [event][module] =~ "suricata" and "import" not in [tags] { elasticsearch { - pipeline => "%{module}.%{dataset}" + pipeline => "filebeat-%{[agent][version]}-suricata-%{[fileset][name]}-pipeline" hosts => "{{ ES }}" index => "so-ids" - template_name => "so-ids" + template_name => "so-ids" template => "/templates/so-ids-template.json" ssl => true ssl_certificate_verification => false