diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index 21667ece8..d253a6b51 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -30,7 +30,7 @@ logstash: - so/0011_input_endgame.conf - so/0012_input_elastic_agent.conf - so/9999_output_redis.conf.jinja - searchnode: + search: - so/0900_input_redis.conf.jinja - so/9805_output_elastic_agent.conf.jinja - so/9900_output_endgame.conf.jinja @@ -59,5 +59,5 @@ logstash: pipeline_x_workers: 1 pipeline_x_batch_x_size: 125 pipeline_x_ecs_compatibility: disabled - dmz_nodes: {} + dmz_nodes: [] diff --git a/salt/logstash/etc/logstash.yml b/salt/logstash/etc/logstash.yml index ca953975f..973b2ab10 100644 --- a/salt/logstash/etc/logstash.yml +++ b/salt/logstash/etc/logstash.yml @@ -1,5 +1 @@ -http.host: 0.0.0.0 -path.logs: /var/log/logstash -pipeline.workers: {{ pipeline_workers }} -pipeline.batch.size: {{ pipeline_batch }} -pipeline.ecs_compatibility: {{ pipeline_ecs_compatibility }} +{{ LOGSTASH_MERGED.config | yaml(False) | replace("_x_", ".") }} diff --git a/salt/logstash/etc/pipelines.yml.jinja b/salt/logstash/etc/pipelines.yml.jinja index 07eedce25..427cc9f14 100644 --- a/salt/logstash/etc/pipelines.yml.jinja +++ b/salt/logstash/etc/pipelines.yml.jinja @@ -1,4 +1,4 @@ -{%- for ap in assigned_pipelines %} -- pipeline.id: {{ ap }} - path.config: "/usr/share/logstash/pipelines/{{ ap }}/" +{%- for assigned_pipeline in ASSIGNED_PIPELINES %} +- pipeline.id: {{ assigned_pipeline }} + path.config: "/usr/share/logstash/pipelines/{{ assigned_pipeline }}/" {% endfor -%} diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 54a038668..8d00d059f 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -68,6 +68,8 @@ ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") GLOBALS: {{ GLOBALS }} ES_USER: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }}" ES_PASS: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }}" + THREADS: {{ LOGSTASH_MERGED.config.pipeline_x_workers }} + BATCH: {{ LOGSTASH_MERGED.config.pipeline_x_batch_x_size }} {% else %} - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1]}} {% endif %} @@ -88,19 +90,17 @@ ls_pipeline_{{assigned_pipeline}}: - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} {% endfor %} - clean: True - - {% endfor %} {% endfor %} +# Copy down all the configs lspipelinesyml: file.managed: - name: /opt/so/conf/logstash/etc/pipelines.yml - source: salt://logstash/etc/pipelines.yml.jinja - template: jinja - defaults: - assigned_pipelines: {{ ASSIGNED_PIPELINES }} + ASSIGNED_PIPELINES: {{ ASSIGNED_PIPELINES }} -# Copy down all the configs lsetcsync: file.recurse: - name: /opt/so/conf/logstash/etc @@ -110,6 +110,8 @@ lsetcsync: - template: jinja - clean: True - exclude_pat: pipelines* + - defaults: + LOGSTASH_MERGED: {{ LOGSTASH_MERGED }} # Create the import directory importdir: @@ -188,7 +190,7 @@ so-logstash: - file: lsetcsync {% for assigned_pipeline in LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %} - file: ls_pipeline_{{assigned_pipeline}} - {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[ap] %} + {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} {% endfor %} {% endfor %} diff --git a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja index e0999e490..661bc0b73 100644 --- a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja @@ -1,5 +1,3 @@ -{%- set THREADS = salt['pillar.get']('logstash:settings:pipeline_x_workers') %} -{%- set BATCH = salt['pillar.get']('logstash:settings:pipeline_x_batch_x_size', 125) %} {%- from 'logstash/map.jinja' import REDIS_NODES with context %} {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %} diff --git a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja index 7c4dacf12..0d3b3324b 100644 --- a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set HOST = GLOBALS.manager %} {%- endif %} -{%- set BATCH = salt['pillar.get']('logstash:settings:pipeline_x_batch_x_size') %} {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %} output {