Merge pull request #943 from Security-Onion-Solutions/issue/825

Pillarize filebeat inputs and output
2025-12-06 17:22:49 +01:00 · 2020-07-07 15:51:08 -04:00
parent f4f189cc50 fff713db85
commit 07cc89e4d6
2 changed files with 154 additions and 146 deletions
--- a/salt/filebeat/etc/filebeat.yml
+++ b/salt/filebeat/etc/filebeat.yml
@@ -75,177 +75,181 @@ filebeat.modules:
 filebeat.inputs:
 #------------------------------ Log prospector --------------------------------
 {%- if grains['role'] == 'so-sensor' or grains['role'] == "so-eval" or grains['role'] == "so-helix" or grains['role'] == "so-heavynode" or grains['role'] == "so-standalone" %}
+- type: udp
+  enabled: true
+  host: "0.0.0.0:514"
+  fields:
+    module: syslog
+    dataset: syslog
+  pipeline: "syslog"
+  index: "so-syslog-%{+yyyy.MM.dd}"
+  processors:
+  - drop_fields:
+      fields: ["source", "prospector", "input", "offset", "beat"]
+  fields_under_root: true

-  - type: udp
-    enabled: true
-    host: "0.0.0.0:514"
-    fields:
-      module: syslog
-      dataset: syslog
-    pipeline: "syslog"
-    index: "so-syslog-%{+yyyy.MM.dd}"
-    processors:
-    - drop_fields:
-        fields: ["source", "prospector", "input", "offset", "beat"]
-    fields_under_root: true
-
-  - type: tcp
-    enabled: true
-    host: "0.0.0.0:514"
-    fields:
-      module: syslog
-      dataset: syslog
-    pipeline: "syslog"
-    index: "so-syslog-%{+yyyy.MM.dd}"
-    processors:
-    - drop_fields:
-        fields: ["source", "prospector", "input", "offset", "beat"]
-    fields_under_root: true
-{%- if BROVER != 'SURICATA' %}
-{%- for LOGNAME in salt['pillar.get']('brologs:enabled', '')  %}
-  - type: log
-    paths:
-      - /nsm/zeek/logs/current/{{ LOGNAME }}.log
-    fields:
-        module: zeek 
-        dataset: {{ LOGNAME }}
-        category: network
-    processors:
-    - drop_fields:
-        fields: ["source", "prospector", "input", "offset", "beat"]
- 
-    fields_under_root: true
-    clean_removed: false
-    close_removed: false
-
-  - type: log
-    paths:
-      - /nsm/import/*/zeek/logs/{{ LOGNAME }}.log
-    fields:
-        module: zeek 
-        dataset: {{ LOGNAME }}
-        category: network
-        imported: true
-    processors:
-    - dissect:
-        tokenizer: "/nsm/import/%{import.id}/zeek/logs/%{import.file}"
-        field: "log.file.path"
-        target_prefix: ""
-    - drop_fields:
-        fields: ["source", "prospector", "input", "offset", "beat"]
- 
-    fields_under_root: true
-    clean_removed: false
-    close_removed: false
-{%- endfor %}
-{%- endif %}
-
-  - type: log
-    paths:
-      - /nsm/suricata/eve*.json
-    fields:
-      module: suricata
-      dataset: common
+- type: tcp
+  enabled: true
+  host: "0.0.0.0:514"
+  fields:
+    module: syslog
+    dataset: syslog
+  pipeline: "syslog"
+  index: "so-syslog-%{+yyyy.MM.dd}"
+  processors:
+  - drop_fields:
+      fields: ["source", "prospector", "input", "offset", "beat"]
+  fields_under_root: true
+  {%- if BROVER != 'SURICATA' %}
+    {%- for LOGNAME in salt['pillar.get']('brologs:enabled', '')  %}
+- type: log
+  paths:
+    - /nsm/zeek/logs/current/{{ LOGNAME }}.log
+  fields:
+      module: zeek 
+      dataset: {{ LOGNAME }}
      category: network
+  processors:
+  - drop_fields:
+      fields: ["source", "prospector", "input", "offset", "beat"]
 
-    processors:
-    - drop_fields:
-        fields: ["source", "prospector", "input", "offset", "beat"]
+  fields_under_root: true
+  clean_removed: false
+  close_removed: false

-    fields_under_root: true
-    clean_removed: false
-    close_removed: false
-
-  - type: log
-    paths:
-      - /nsm/import/*/suricata/eve*.json
-    fields:
-      module: suricata
-      dataset: common
+- type: log
+  paths:
+    - /nsm/import/*/zeek/logs/{{ LOGNAME }}.log
+  fields:
+      module: zeek 
+      dataset: {{ LOGNAME }}
      category: network
      imported: true
-    processors:
-    - dissect:
-        tokenizer: "/nsm/import/%{import.id}/suricata/%{import.file}"
-        field: "log.file.path"
-        target_prefix: ""
-    - drop_fields:
-        fields: ["source", "prospector", "input", "offset", "beat"]
+  processors:
+  - dissect:
+      tokenizer: "/nsm/import/%{import.id}/zeek/logs/%{import.file}"
+      field: "log.file.path"
+      target_prefix: ""
+  - drop_fields:
+      fields: ["source", "prospector", "input", "offset", "beat"]
 
-    fields_under_root: true
-    clean_removed: false
-    close_removed: false
+  fields_under_root: true
+  clean_removed: false
+  close_removed: false
+    {%- endfor %}
+  {%- endif %}
+
+- type: log
+  paths:
+    - /nsm/suricata/eve*.json
+  fields:
+    module: suricata
+    dataset: common
+    category: network
+
+  processors:
+  - drop_fields:
+      fields: ["source", "prospector", "input", "offset", "beat"]
+
+  fields_under_root: true
+  clean_removed: false
+  close_removed: false
+
+- type: log
+  paths:
+    - /nsm/import/*/suricata/eve*.json
+  fields:
+    module: suricata
+    dataset: common
+    category: network
+    imported: true 
+  processors:
+  - dissect:
+      tokenizer: "/nsm/import/%{import.id}/suricata/%{import.file}"
+      field: "log.file.path"
+      target_prefix: ""
+  - drop_fields:
+      fields: ["source", "prospector", "input", "offset", "beat"]
+
+  fields_under_root: true
+  clean_removed: false
+  close_removed: false

  {%- if STRELKAENABLED == 1 %}
-  - type: log
-    paths:
-      - /nsm/strelka/log/strelka.log
-    fields:
-      module: strelka
-      category: file
-      dataset: file
+- type: log
+  paths:
+    - /nsm/strelka/log/strelka.log
+  fields:
+    module: strelka
+    category: file
+    dataset: file

-    processors:
-    - drop_fields:
-        fields: ["source", "prospector", "input", "offset", "beat"]    
+  processors:
+  - drop_fields:
+      fields: ["source", "prospector", "input", "offset", "beat"]    

-    fields_under_root: true
-    clean_removed: false
-    close_removed: false
+  fields_under_root: true
+  clean_removed: false
+  close_removed: false

  {%- endif %}
 {%- endif %}

 {%- if WAZUHENABLED == 1 %}

-  - type: log
-    paths:
-      - /wazuh/alerts/alerts.json
-    fields:
-      module: ossec
-      dataset: alert
-      category: host
-    processors:
-    - drop_fields:
-        fields: ["source", "prospector", "input", "offset", "beat"]
+- type: log
+  paths:
+    - /wazuh/alerts/alerts.json
+  fields:
+    module: ossec
+    dataset: alert
+    category: host
+  processors:
+  - drop_fields:
+      fields: ["source", "prospector", "input", "offset", "beat"]

-    fields_under_root: true
-    clean_removed: false
-    close_removed: false
-
-# - type: log
-#    paths:
-#      - /wazuh/archives/archives.json
-#    fields:
-#      type: ossec_archive
-#    fields_under_root: true
-#    clean_removed: false
-#    close_removed: false
+  fields_under_root: true
+  clean_removed: false
+  close_removed: false

 {%- endif %}

 {%- if FLEETMASTER or FLEETNODE %}

-  - type: log
-    paths:
-      - /nsm/osquery/fleet/result.log
-    fields:
-      module: osquery
-      dataset: query_result
-      category: host
+- type: log
+  paths:
+    - /nsm/osquery/fleet/result.log
+  fields:
+    module: osquery
+    dataset: query_result
+    category: host

-    processors:
-    - drop_fields:
-        fields: ["source", "prospector", "input", "offset", "beat"]
+  processors:
+  - drop_fields:
+      fields: ["source", "prospector", "input", "offset", "beat"]

-    fields_under_root: true
-    clean_removed: false
-    close_removed: false
+  fields_under_root: true
+  clean_removed: false
+  close_removed: false

 {%- endif %}

+{%- if INPUTS %}
+# USER PILLAR DEFINED INPUTS
+{{ INPUTS | yaml(False) }}
+{%- endif %}
+
+{% if OUTPUT -%}
+# USER PILLAR DEFINED OUTPUT
+{%- set types = OUTPUT.keys() | list %}
+{%- set type = types[0] %}
+output.{{ type }}:
+  {%- for i in OUTPUT[type].items() %}
+  {{ i[0] }}: {{ i[1]}}
+  {%- endfor %}
+{%- else %}
 #----------------------------- Elasticsearch/Logstash output ---------------------------------
-{%- if grains['role'] == "so-eval" %}
+  {%- if grains['role'] == "so-eval" %}
 output.elasticsearch:
  enabled: true
  hosts: ["{{ MASTER }}:9200"]
@@ -269,7 +273,7 @@ output.elasticsearch:
        module: "strelka"
  
 setup.template.enabled: false
-{%- else %}
+  {%- else %}

 output.logstash:
  # Boolean flag to enable or disable the output module.
@@ -320,7 +324,8 @@ setup.template.enabled: false
  # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
  #_source:
    #enabled: false
-{%- endif %}
+  {%- endif %}
+{% endif %}
 #============================== Kibana =====================================

 # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.
--- a/salt/filebeat/init.sls
+++ b/salt/filebeat/init.sls
@@ -46,6 +46,9 @@ filebeatconfsync:
    - user: 0
    - group: 0
    - template: jinja
+    - defaults:
+        INPUTS: {{ salt['pillar.get']('filebeat:config:inputs', {}) }}
+        OUTPUT: {{ salt['pillar.get']('filebeat:config:output', {}) }}
 so-filebeat:
  docker_container.running:
    - image: {{ MASTER }}:5000/soshybridhunter/so-filebeat:{{ VERSION }}{{ FEATURES }}