diff --git a/salt/logstash/etc/pipelines.yml.jinja b/salt/logstash/etc/pipelines.yml.jinja index 3ee7a0d3b..07eedce25 100644 --- a/salt/logstash/etc/pipelines.yml.jinja +++ b/salt/logstash/etc/pipelines.yml.jinja @@ -1,4 +1,4 @@ -{%- for pl in pipelines %} -- pipeline.id: {{ pl }} - path.config: "/usr/share/logstash/pipelines/{{ pl }}/" +{%- for ap in assigned_pipelines %} +- pipeline.id: {{ ap }} + path.config: "/usr/share/logstash/pipelines/{{ ap }}/" {% endfor -%} diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index caabd10ea..54a038668 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -6,19 +6,19 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} -{% from 'docker/docker.map.jinja' import DOCKER %} -{% from 'logstash/map.jinja' import REDIS_NODES with context %} {% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'docker/docker.map.jinja' import DOCKER %} +{% from 'logstash/map.jinja' import REDIS_NODES %} +{% from 'logstash/map.jinja' import LOGSTASH_MERGED %} # Logstash Section - Decide which pillar to use -{% set lsheap = salt['pillar.get']('logstash:settings:lsheap') %} +{% set lsheap = LOGSTASH_MERGED.settings.lsheap %} {% if GLOBALS.role in ['so-eval','so-managersearch', 'so-manager', 'so-standalone'] %} {% set nodetype = GLOBALS.role %} {% endif %} -{% set PIPELINES = salt['pillar.get']('logstash:pipelines', {}) %} -{% set DOCKER_OPTIONS = salt['pillar.get']('logstash:docker_options', {}) %} -{% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} +{% set ASSIGNED_PIPELINES = LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %} +{% set DOCKER_OPTIONS = LOGSTASH_MERGED.docker_options %} include: - ssl @@ -56,20 +56,20 @@ lspipelinedir: - user: 931 - group: 939 - {% for PL in PIPELINES %} - {% for CONFIGFILE in PIPELINES[PL].config %} -ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: +{% for assigned_pipeline in ASSIGNED_PIPELINES %} + {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} +ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: file.managed: - source: salt://logstash/pipelines/config/{{CONFIGFILE}} {% if 'jinja' in CONFIGFILE.split('.')[-1] %} - - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE.split('/')[1] | replace(".jinja", "")}} + - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1] | replace(".jinja", "")}} - template: jinja - defaults: GLOBALS: {{ GLOBALS }} ES_USER: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }}" ES_PASS: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }}" {% else %} - - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE.split('/')[1]}} + - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1]}} {% endif %} - user: 931 - group: 939 @@ -78,18 +78,19 @@ ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: - show_changes: False {% endfor %} -ls_pipeline_{{PL}}: +ls_pipeline_{{assigned_pipeline}}: file.directory: - - name: /opt/so/conf/logstash/pipelines/{{PL}} + - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}} - user: 931 - group: 939 - require: - {% for CONFIGFILE in PIPELINES[PL].config %} - - file: ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} + {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} + - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} {% endfor %} - clean: True {% endfor %} +{% endfor %} lspipelinesyml: file.managed: @@ -97,7 +98,7 @@ lspipelinesyml: - source: salt://logstash/etc/pipelines.yml.jinja - template: jinja - defaults: - pipelines: {{ PIPELINES }} + assigned_pipelines: {{ ASSIGNED_PIPELINES }} # Copy down all the configs lsetcsync: @@ -185,10 +186,10 @@ so-logstash: {%- endif %} - watch: - file: lsetcsync - {% for PL in PIPELINES %} - - file: ls_pipeline_{{PL}} - {% for CONFIGFILE in PIPELINES[PL].config %} - - file: ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} + {% for assigned_pipeline in LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %} + - file: ls_pipeline_{{assigned_pipeline}} + {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[ap] %} + - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} {% endfor %} {% endfor %} - require: diff --git a/salt/logstash/map.jinja b/salt/logstash/map.jinja index e23f944a2..c4ad5d96a 100644 --- a/salt/logstash/map.jinja +++ b/salt/logstash/map.jinja @@ -1,4 +1,7 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} +{% import_yaml 'logstash/defaults.yaml' as LOGSTASH_DEFAULTS %} +{% set LOGSTASH_MERGED = salt['pillar.get']('logstash', LOGSTASH_DEFAULTS.logstash, merge=True) %} + {% set REDIS_NODES = [] %} {% set LOGSTASH_NODES = [] %} {% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}