diff --git a/pillar/top.sls b/pillar/top.sls index a0fbcb4c1..259e87c96 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -35,6 +35,7 @@ base: - manager.adv_manager - idstools.soc_idstools - idstools.adv_idstools + - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash - soc.soc_soc @@ -124,9 +125,7 @@ base: - minions.adv_{{ grains.id }} '*_standalone': - - logstash - - logstash.manager - - logstash.search + - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash - elasticsearch.index_templates @@ -175,6 +174,7 @@ base: '*_heavynode': - elasticsearch.auth + - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash - elasticsearch.soc_elasticsearch @@ -203,6 +203,7 @@ base: - minions.adv_{{ grains.id }} '*_searchnode': + - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash - elasticsearch.soc_elasticsearch @@ -214,6 +215,7 @@ base: - minions.adv_{{ grains.id }} '*_receiver': + - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} @@ -270,6 +272,7 @@ base: '*_fleet': - backup.soc_backup - backup.adv_backup + - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash - minions.{{ grains.id }} diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index 21667ece8..d253a6b51 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -30,7 +30,7 @@ logstash: - so/0011_input_endgame.conf - so/0012_input_elastic_agent.conf - so/9999_output_redis.conf.jinja - searchnode: + search: - so/0900_input_redis.conf.jinja - so/9805_output_elastic_agent.conf.jinja - so/9900_output_endgame.conf.jinja @@ -59,5 +59,5 @@ logstash: pipeline_x_workers: 1 pipeline_x_batch_x_size: 125 pipeline_x_ecs_compatibility: disabled - dmz_nodes: {} + dmz_nodes: [] diff --git a/salt/logstash/etc/logstash.yml b/salt/logstash/etc/logstash.yml index ca953975f..973b2ab10 100644 --- a/salt/logstash/etc/logstash.yml +++ b/salt/logstash/etc/logstash.yml @@ -1,5 +1 @@ -http.host: 0.0.0.0 -path.logs: /var/log/logstash -pipeline.workers: {{ pipeline_workers }} -pipeline.batch.size: {{ pipeline_batch }} -pipeline.ecs_compatibility: {{ pipeline_ecs_compatibility }} +{{ LOGSTASH_MERGED.config | yaml(False) | replace("_x_", ".") }} diff --git a/salt/logstash/etc/pipelines.yml.jinja b/salt/logstash/etc/pipelines.yml.jinja index 3ee7a0d3b..427cc9f14 100644 --- a/salt/logstash/etc/pipelines.yml.jinja +++ b/salt/logstash/etc/pipelines.yml.jinja @@ -1,4 +1,4 @@ -{%- for pl in pipelines %} -- pipeline.id: {{ pl }} - path.config: "/usr/share/logstash/pipelines/{{ pl }}/" +{%- for assigned_pipeline in ASSIGNED_PIPELINES %} +- pipeline.id: {{ assigned_pipeline }} + path.config: "/usr/share/logstash/pipelines/{{ assigned_pipeline }}/" {% endfor -%} diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index c80df1f5c..7072ed46d 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -6,19 +6,19 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls in allowed_states %} -{% from 'docker/docker.map.jinja' import DOCKER %} -{% from 'logstash/map.jinja' import REDIS_NODES with context %} {% from 'vars/globals.map.jinja' import GLOBALS %} +{% from 'docker/docker.map.jinja' import DOCKER %} +{% from 'logstash/map.jinja' import REDIS_NODES %} +{% from 'logstash/map.jinja' import LOGSTASH_MERGED %} # Logstash Section - Decide which pillar to use -{% set lsheap = salt['pillar.get']('logstash:settings:lsheap') %} +{% set lsheap = LOGSTASH_MERGED.settings.lsheap %} {% if GLOBALS.role in ['so-eval','so-managersearch', 'so-manager', 'so-standalone'] %} {% set nodetype = GLOBALS.role %} {% endif %} -{% set PIPELINES = salt['pillar.get']('logstash:pipelines', {}) %} -{% set DOCKER_OPTIONS = salt['pillar.get']('logstash:docker_options', {}) %} -{% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} +{% set ASSIGNED_PIPELINES = LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %} +{% set DOCKER_OPTIONS = LOGSTASH_MERGED.docker_options %} include: - ssl @@ -73,20 +73,22 @@ lspipelinedir: - user: 931 - group: 939 - {% for PL in PIPELINES %} - {% for CONFIGFILE in PIPELINES[PL].config %} -ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: +{% for assigned_pipeline in ASSIGNED_PIPELINES %} + {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} +ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: file.managed: - source: salt://logstash/pipelines/config/{{CONFIGFILE}} {% if 'jinja' in CONFIGFILE.split('.')[-1] %} - - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE.split('/')[1] | replace(".jinja", "")}} + - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1] | replace(".jinja", "")}} - template: jinja - defaults: GLOBALS: {{ GLOBALS }} ES_USER: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }}" ES_PASS: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }}" + THREADS: {{ LOGSTASH_MERGED.config.pipeline_x_workers }} + BATCH: {{ LOGSTASH_MERGED.config.pipeline_x_batch_x_size }} {% else %} - - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE.split('/')[1]}} + - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1]}} {% endif %} - user: 931 - group: 939 @@ -95,28 +97,27 @@ ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: - show_changes: False {% endfor %} -ls_pipeline_{{PL}}: +ls_pipeline_{{assigned_pipeline}}: file.directory: - - name: /opt/so/conf/logstash/pipelines/{{PL}} + - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}} - user: 931 - group: 939 - require: - {% for CONFIGFILE in PIPELINES[PL].config %} - - file: ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} + {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} + - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} {% endfor %} - clean: True +{% endfor %} - {% endfor %} - +# Copy down all the configs lspipelinesyml: file.managed: - name: /opt/so/conf/logstash/etc/pipelines.yml - source: salt://logstash/etc/pipelines.yml.jinja - template: jinja - defaults: - pipelines: {{ PIPELINES }} + ASSIGNED_PIPELINES: {{ ASSIGNED_PIPELINES }} -# Copy down all the configs lsetcsync: file.recurse: - name: /opt/so/conf/logstash/etc @@ -126,6 +127,8 @@ lsetcsync: - template: jinja - clean: True - exclude_pat: pipelines* + - defaults: + LOGSTASH_MERGED: {{ LOGSTASH_MERGED }} # Create the import directory importdir: @@ -202,10 +205,10 @@ so-logstash: {%- endif %} - watch: - file: lsetcsync - {% for PL in PIPELINES %} - - file: ls_pipeline_{{PL}} - {% for CONFIGFILE in PIPELINES[PL].config %} - - file: ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} + {% for assigned_pipeline in LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %} + - file: ls_pipeline_{{assigned_pipeline}} + {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} + - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} {% endfor %} {% endfor %} - require: diff --git a/salt/logstash/map.jinja b/salt/logstash/map.jinja index e23f944a2..c4ad5d96a 100644 --- a/salt/logstash/map.jinja +++ b/salt/logstash/map.jinja @@ -1,4 +1,7 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} +{% import_yaml 'logstash/defaults.yaml' as LOGSTASH_DEFAULTS %} +{% set LOGSTASH_MERGED = salt['pillar.get']('logstash', LOGSTASH_DEFAULTS.logstash, merge=True) %} + {% set REDIS_NODES = [] %} {% set LOGSTASH_NODES = [] %} {% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} diff --git a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja index e0999e490..661bc0b73 100644 --- a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja @@ -1,5 +1,3 @@ -{%- set THREADS = salt['pillar.get']('logstash:settings:pipeline_x_workers') %} -{%- set BATCH = salt['pillar.get']('logstash:settings:pipeline_x_batch_x_size', 125) %} {%- from 'logstash/map.jinja' import REDIS_NODES with context %} {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %} diff --git a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja index 7c4dacf12..0d3b3324b 100644 --- a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja @@ -3,7 +3,6 @@ {%- else %} {%- set HOST = GLOBALS.manager %} {%- endif %} -{%- set BATCH = salt['pillar.get']('logstash:settings:pipeline_x_batch_x_size') %} {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %} output {