logstash jinja for ui

salt/logstash/defaults.yaml

@@ -30,7 +30,7 @@ logstash:
       - so/0011_input_endgame.conf
       - so/0012_input_elastic_agent.conf
       - so/9999_output_redis.conf.jinja
-    searchnode:
+    search:
       - so/0900_input_redis.conf.jinja
       - so/9805_output_elastic_agent.conf.jinja
       - so/9900_output_endgame.conf.jinja
@@ -59,5 +59,5 @@ logstash:
     pipeline_x_workers: 1
     pipeline_x_batch_x_size: 125
     pipeline_x_ecs_compatibility: disabled
-  dmz_nodes: {}
+  dmz_nodes: []
 

salt/logstash/etc/logstash.yml

@@ -1,5 +1 @@
-http.host: 0.0.0.0
-path.logs: /var/log/logstash
-pipeline.workers: {{ pipeline_workers }}
-pipeline.batch.size: {{ pipeline_batch }}
-pipeline.ecs_compatibility: {{ pipeline_ecs_compatibility }}
+{{ LOGSTASH_MERGED.config | yaml(False) | replace("_x_", ".") }}

salt/logstash/etc/pipelines.yml.jinja

@@ -1,4 +1,4 @@
-{%- for ap in assigned_pipelines %}
-- pipeline.id: {{ ap }}
-  path.config: "/usr/share/logstash/pipelines/{{ ap }}/"
+{%- for assigned_pipeline in ASSIGNED_PIPELINES %}
+- pipeline.id: {{ assigned_pipeline }}
+  path.config: "/usr/share/logstash/pipelines/{{ assigned_pipeline }}/"
 {% endfor -%}

salt/logstash/init.sls

@@ -68,6 +68,8 @@ ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_")
         GLOBALS: {{ GLOBALS }}
         ES_USER: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }}"
         ES_PASS: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }}"
+        THREADS: {{ LOGSTASH_MERGED.config.pipeline_x_workers }}
+        BATCH: {{ LOGSTASH_MERGED.config.pipeline_x_batch_x_size }}
       {% else %}
     - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1]}}
       {% endif %}
@@ -88,19 +90,17 @@ ls_pipeline_{{assigned_pipeline}}:
       - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}
     {% endfor %}
     - clean: True
-
-  {% endfor %}
 {% endfor %}
 
+# Copy down all the configs
 lspipelinesyml:
   file.managed:
     - name: /opt/so/conf/logstash/etc/pipelines.yml
     - source: salt://logstash/etc/pipelines.yml.jinja
     - template: jinja
     - defaults:
-        assigned_pipelines: {{ ASSIGNED_PIPELINES }}
+        ASSIGNED_PIPELINES: {{ ASSIGNED_PIPELINES }}
 
-# Copy down all the configs
 lsetcsync:
   file.recurse:
     - name: /opt/so/conf/logstash/etc
@@ -110,6 +110,8 @@ lsetcsync:
     - template: jinja
     - clean: True
     - exclude_pat: pipelines*
+    - defaults:
+        LOGSTASH_MERGED: {{ LOGSTASH_MERGED }}
 
 # Create the import directory
 importdir:
@@ -188,7 +190,7 @@ so-logstash:
       - file: lsetcsync
   {% for assigned_pipeline in LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %}
       - file: ls_pipeline_{{assigned_pipeline}}
-    {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[ap] %}
+    {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %}
       - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}
     {% endfor %}
   {% endfor %}

salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja

@@ -1,5 +1,3 @@
-{%- set THREADS = salt['pillar.get']('logstash:settings:pipeline_x_workers') %}
-{%- set BATCH = salt['pillar.get']('logstash:settings:pipeline_x_batch_x_size', 125) %}
 {%- from 'logstash/map.jinja' import REDIS_NODES with context %}
 {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %}
 

salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja

@@ -3,7 +3,6 @@
 {%- else %}
   {%- set HOST = GLOBALS.manager %}
 {%- endif %}
-{%- set BATCH = salt['pillar.get']('logstash:settings:pipeline_x_batch_x_size') %}
 {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %}
 
 output {