Merge pull request #10282 from Security-Onion-Solutions/ui/logstash

UI/logstash

pillar/top.sls

@@ -35,6 +35,7 @@ base:
     - manager.adv_manager
     - idstools.soc_idstools
     - idstools.adv_idstools
+    - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
     - soc.soc_soc
@@ -124,9 +125,7 @@ base:
     - minions.adv_{{ grains.id }}
 
   '*_standalone':
-    - logstash
+    - logstash.nodes
-    - logstash.manager
-    - logstash.search
     - logstash.soc_logstash
     - logstash.adv_logstash
     - elasticsearch.index_templates
@@ -175,6 +174,7 @@ base:
 
   '*_heavynode':
     - elasticsearch.auth
+    - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
     - elasticsearch.soc_elasticsearch
@@ -203,6 +203,7 @@ base:
     - minions.adv_{{ grains.id }}
 
   '*_searchnode':
+    - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
     - elasticsearch.soc_elasticsearch
@@ -214,6 +215,7 @@ base:
     - minions.adv_{{ grains.id }}
 
   '*_receiver':
+    - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
@@ -270,6 +272,7 @@ base:
   '*_fleet':
     - backup.soc_backup
     - backup.adv_backup
+    - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
     - minions.{{ grains.id }}

salt/logstash/defaults.yaml

@@ -30,7 +30,7 @@ logstash:
       - so/0011_input_endgame.conf
       - so/0012_input_elastic_agent.conf
       - so/9999_output_redis.conf.jinja
-    searchnode:
+    search:
       - so/0900_input_redis.conf.jinja
       - so/9805_output_elastic_agent.conf.jinja
       - so/9900_output_endgame.conf.jinja
@@ -59,5 +59,5 @@ logstash:
     pipeline_x_workers: 1
     pipeline_x_batch_x_size: 125
     pipeline_x_ecs_compatibility: disabled
-  dmz_nodes: {}
+  dmz_nodes: []
 

salt/logstash/etc/logstash.yml

@@ -1,5 +1 @@
-http.host: 0.0.0.0
+{{ LOGSTASH_MERGED.config | yaml(False) | replace("_x_", ".") }}
-path.logs: /var/log/logstash
-pipeline.workers: {{ pipeline_workers }}
-pipeline.batch.size: {{ pipeline_batch }}
-pipeline.ecs_compatibility: {{ pipeline_ecs_compatibility }}

salt/logstash/etc/pipelines.yml.jinja

@@ -1,4 +1,4 @@
-{%- for pl in pipelines %}
+{%- for assigned_pipeline in ASSIGNED_PIPELINES %}
-- pipeline.id: {{ pl }}
+- pipeline.id: {{ assigned_pipeline }}
-  path.config: "/usr/share/logstash/pipelines/{{ pl }}/"
+  path.config: "/usr/share/logstash/pipelines/{{ assigned_pipeline }}/"
 {% endfor -%}

salt/logstash/init.sls

@@ -6,19 +6,19 @@
 
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls in allowed_states %}
-{% from 'docker/docker.map.jinja' import DOCKER %}
-{% from 'logstash/map.jinja' import REDIS_NODES with context %}
 {% from 'vars/globals.map.jinja' import GLOBALS %}
+{% from 'docker/docker.map.jinja' import DOCKER %}
+{% from 'logstash/map.jinja' import REDIS_NODES %}
+{% from 'logstash/map.jinja' import LOGSTASH_MERGED %}
 
 # Logstash Section - Decide which pillar to use
-{% set lsheap = salt['pillar.get']('logstash:settings:lsheap') %}
+{% set lsheap = LOGSTASH_MERGED.settings.lsheap %}
 {% if GLOBALS.role in ['so-eval','so-managersearch', 'so-manager', 'so-standalone'] %}
     {% set nodetype = GLOBALS.role  %}
 {% endif %}
 
-{% set PIPELINES = salt['pillar.get']('logstash:pipelines', {}) %}
+{% set ASSIGNED_PIPELINES = LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %}
-{% set DOCKER_OPTIONS = salt['pillar.get']('logstash:docker_options', {}) %}
+{% set DOCKER_OPTIONS = LOGSTASH_MERGED.docker_options %}
-{% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
 
 include:
   - ssl
@@ -73,20 +73,22 @@ lspipelinedir:
     - user: 931
     - group: 939
 
-  {% for PL in PIPELINES %}
+{% for assigned_pipeline in ASSIGNED_PIPELINES %}
-    {% for CONFIGFILE in PIPELINES[PL].config %}
+    {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %}
-ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}:
+ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}:
   file.managed:
     - source: salt://logstash/pipelines/config/{{CONFIGFILE}}
       {% if 'jinja' in CONFIGFILE.split('.')[-1] %}
-    - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE.split('/')[1] | replace(".jinja", "")}}
+    - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1] | replace(".jinja", "")}}
     - template: jinja
     - defaults:
         GLOBALS: {{ GLOBALS }}
         ES_USER: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }}"
         ES_PASS: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }}"
+        THREADS: {{ LOGSTASH_MERGED.config.pipeline_x_workers }}
+        BATCH: {{ LOGSTASH_MERGED.config.pipeline_x_batch_x_size }}
       {% else %}
-    - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE.split('/')[1]}}
+    - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1]}}
       {% endif %}
     - user: 931
     - group: 939
@@ -95,28 +97,27 @@ ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}:
     - show_changes: False
     {% endfor %}
 
-ls_pipeline_{{PL}}:
+ls_pipeline_{{assigned_pipeline}}:
   file.directory:
-    - name: /opt/so/conf/logstash/pipelines/{{PL}}
+    - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}
     - user: 931
     - group: 939
     - require:
-    {% for CONFIGFILE in PIPELINES[PL].config %}
+    {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %}
-      - file: ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}
+      - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}
     {% endfor %}
     - clean: True
+{% endfor %}
 
-  {% endfor %}
+# Copy down all the configs
-
 lspipelinesyml:
   file.managed:
     - name: /opt/so/conf/logstash/etc/pipelines.yml
     - source: salt://logstash/etc/pipelines.yml.jinja
     - template: jinja
     - defaults:
-        pipelines: {{ PIPELINES }}
+        ASSIGNED_PIPELINES: {{ ASSIGNED_PIPELINES }}
 
-# Copy down all the configs
 lsetcsync:
   file.recurse:
     - name: /opt/so/conf/logstash/etc
@@ -126,6 +127,8 @@ lsetcsync:
     - template: jinja
     - clean: True
     - exclude_pat: pipelines*
+    - defaults:
+        LOGSTASH_MERGED: {{ LOGSTASH_MERGED }}
 
 # Create the import directory
 importdir:
@@ -202,10 +205,10 @@ so-logstash:
   {%- endif %}
     - watch:
       - file: lsetcsync
-  {% for PL in PIPELINES %}
+  {% for assigned_pipeline in LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %}
-      - file: ls_pipeline_{{PL}}
+      - file: ls_pipeline_{{assigned_pipeline}}
-    {% for CONFIGFILE in PIPELINES[PL].config %}
+    {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %}
-      - file: ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}
+      - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}
     {% endfor %}
   {% endfor %}
     - require:

salt/logstash/map.jinja

@@ -1,4 +1,7 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
+{% import_yaml 'logstash/defaults.yaml' as LOGSTASH_DEFAULTS %}
+{% set LOGSTASH_MERGED = salt['pillar.get']('logstash', LOGSTASH_DEFAULTS.logstash, merge=True) %}
+
 {% set REDIS_NODES = [] %}
 {% set LOGSTASH_NODES = [] %}
 {% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}

salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja

@@ -1,5 +1,3 @@
-{%- set THREADS = salt['pillar.get']('logstash:settings:pipeline_x_workers') %}
-{%- set BATCH = salt['pillar.get']('logstash:settings:pipeline_x_batch_x_size', 125) %}
 {%- from 'logstash/map.jinja' import REDIS_NODES with context %}
 {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %}
 

salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja

@@ -3,7 +3,6 @@
 {%- else %}
   {%- set HOST = GLOBALS.manager %}
 {%- endif %}
-{%- set BATCH = salt['pillar.get']('logstash:settings:pipeline_x_batch_x_size') %}
 {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %}
 
 output {