Merge pull request #10282 from Security-Onion-Solutions/ui/logstash

UI/logstash
2025-12-06 17:22:49 +01:00 · 2023-05-04 13:55:28 -04:00
parent 8055088d25 98705608a6
commit 51869ce5b2
8 changed files with 41 additions and 39 deletions
--- a/pillar/top.sls
+++ b/pillar/top.sls
@@ -35,6 +35,7 @@ base:
    - manager.adv_manager
    - idstools.soc_idstools
    - idstools.adv_idstools
+    - logstash.nodes
    - logstash.soc_logstash
    - logstash.adv_logstash
    - soc.soc_soc
@@ -124,9 +125,7 @@ base:
    - minions.adv_{{ grains.id }}

  '*_standalone':
-    - logstash
-    - logstash.manager
-    - logstash.search
+    - logstash.nodes
    - logstash.soc_logstash
    - logstash.adv_logstash
    - elasticsearch.index_templates
@@ -175,6 +174,7 @@ base:

  '*_heavynode':
    - elasticsearch.auth
+    - logstash.nodes
    - logstash.soc_logstash
    - logstash.adv_logstash
    - elasticsearch.soc_elasticsearch
@@ -203,6 +203,7 @@ base:
    - minions.adv_{{ grains.id }}

  '*_searchnode':
+    - logstash.nodes
    - logstash.soc_logstash
    - logstash.adv_logstash
    - elasticsearch.soc_elasticsearch
@@ -214,6 +215,7 @@ base:
    - minions.adv_{{ grains.id }}

  '*_receiver':
+    - logstash.nodes
    - logstash.soc_logstash
    - logstash.adv_logstash
    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
@@ -270,6 +272,7 @@ base:
  '*_fleet':
    - backup.soc_backup
    - backup.adv_backup
+    - logstash.nodes
    - logstash.soc_logstash
    - logstash.adv_logstash
    - minions.{{ grains.id }}
--- a/salt/logstash/defaults.yaml
+++ b/salt/logstash/defaults.yaml
@@ -30,7 +30,7 @@ logstash:
      - so/0011_input_endgame.conf
      - so/0012_input_elastic_agent.conf
      - so/9999_output_redis.conf.jinja
-    searchnode:
+    search:
      - so/0900_input_redis.conf.jinja
      - so/9805_output_elastic_agent.conf.jinja
      - so/9900_output_endgame.conf.jinja
@@ -59,5 +59,5 @@ logstash:
    pipeline_x_workers: 1
    pipeline_x_batch_x_size: 125
    pipeline_x_ecs_compatibility: disabled
-  dmz_nodes: {}
+  dmz_nodes: []

--- a/salt/logstash/etc/logstash.yml
+++ b/salt/logstash/etc/logstash.yml
@@ -1,5 +1 @@
-http.host: 0.0.0.0
-path.logs: /var/log/logstash
-pipeline.workers: {{ pipeline_workers }}
-pipeline.batch.size: {{ pipeline_batch }}
-pipeline.ecs_compatibility: {{ pipeline_ecs_compatibility }}
+{{ LOGSTASH_MERGED.config | yaml(False) | replace("_x_", ".") }}
--- a/salt/logstash/etc/pipelines.yml.jinja
+++ b/salt/logstash/etc/pipelines.yml.jinja
@@ -1,4 +1,4 @@
-{%- for pl in pipelines %}
- pipeline.id: {{ pl }}
-  path.config: "/usr/share/logstash/pipelines/{{ pl }}/"
+{%- for assigned_pipeline in ASSIGNED_PIPELINES %}
+- pipeline.id: {{ assigned_pipeline }}
+  path.config: "/usr/share/logstash/pipelines/{{ assigned_pipeline }}/"
 {% endfor -%}
--- a/salt/logstash/init.sls
+++ b/salt/logstash/init.sls
@@ -6,19 +6,19 @@

 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls in allowed_states %}
-{% from 'docker/docker.map.jinja' import DOCKER %}
-{% from 'logstash/map.jinja' import REDIS_NODES with context %}
 {% from 'vars/globals.map.jinja' import GLOBALS %}
+{% from 'docker/docker.map.jinja' import DOCKER %}
+{% from 'logstash/map.jinja' import REDIS_NODES %}
+{% from 'logstash/map.jinja' import LOGSTASH_MERGED %}

 # Logstash Section - Decide which pillar to use
-{% set lsheap = salt['pillar.get']('logstash:settings:lsheap') %}
+{% set lsheap = LOGSTASH_MERGED.settings.lsheap %}
 {% if GLOBALS.role in ['so-eval','so-managersearch', 'so-manager', 'so-standalone'] %}
    {% set nodetype = GLOBALS.role  %}
 {% endif %}

-{% set PIPELINES = salt['pillar.get']('logstash:pipelines', {}) %}
-{% set DOCKER_OPTIONS = salt['pillar.get']('logstash:docker_options', {}) %}
-{% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
+{% set ASSIGNED_PIPELINES = LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %}
+{% set DOCKER_OPTIONS = LOGSTASH_MERGED.docker_options %}

 include:
  - ssl
@@ -73,20 +73,22 @@ lspipelinedir:
    - user: 931
    - group: 939

-  {% for PL in PIPELINES %}
-    {% for CONFIGFILE in PIPELINES[PL].config %}
-ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}:
+{% for assigned_pipeline in ASSIGNED_PIPELINES %}
+    {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %}
+ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}:
  file.managed:
    - source: salt://logstash/pipelines/config/{{CONFIGFILE}}
      {% if 'jinja' in CONFIGFILE.split('.')[-1] %}
-    - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE.split('/')[1] | replace(".jinja", "")}}
+    - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1] | replace(".jinja", "")}}
    - template: jinja
    - defaults:
        GLOBALS: {{ GLOBALS }}
        ES_USER: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') }}"
        ES_PASS: "{{ salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') }}"
+        THREADS: {{ LOGSTASH_MERGED.config.pipeline_x_workers }}
+        BATCH: {{ LOGSTASH_MERGED.config.pipeline_x_batch_x_size }}
      {% else %}
-    - name: /opt/so/conf/logstash/pipelines/{{PL}}/{{CONFIGFILE.split('/')[1]}}
+    - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{CONFIGFILE.split('/')[1]}}
      {% endif %}
    - user: 931
    - group: 939
@@ -95,28 +97,27 @@ ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}:
    - show_changes: False
    {% endfor %}

-ls_pipeline_{{PL}}:
+ls_pipeline_{{assigned_pipeline}}:
  file.directory:
-    - name: /opt/so/conf/logstash/pipelines/{{PL}}
+    - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}
    - user: 931
    - group: 939
    - require:
-    {% for CONFIGFILE in PIPELINES[PL].config %}
-      - file: ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}
+    {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %}
+      - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}
    {% endfor %}
    - clean: True
-
 {% endfor %}

+# Copy down all the configs
 lspipelinesyml:
  file.managed:
    - name: /opt/so/conf/logstash/etc/pipelines.yml
    - source: salt://logstash/etc/pipelines.yml.jinja
    - template: jinja
    - defaults:
-        pipelines: {{ PIPELINES }}
+        ASSIGNED_PIPELINES: {{ ASSIGNED_PIPELINES }}

-# Copy down all the configs
 lsetcsync:
  file.recurse:
    - name: /opt/so/conf/logstash/etc
@@ -126,6 +127,8 @@ lsetcsync:
    - template: jinja
    - clean: True
    - exclude_pat: pipelines*
+    - defaults:
+        LOGSTASH_MERGED: {{ LOGSTASH_MERGED }}

 # Create the import directory
 importdir:
@@ -202,10 +205,10 @@ so-logstash:
  {%- endif %}
    - watch:
      - file: lsetcsync
-  {% for PL in PIPELINES %}
-      - file: ls_pipeline_{{PL}}
-    {% for CONFIGFILE in PIPELINES[PL].config %}
-      - file: ls_pipeline_{{PL}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}
+  {% for assigned_pipeline in LOGSTASH_MERGED.assigned_pipelines.roles[GLOBALS.role.split('-')[1]] %}
+      - file: ls_pipeline_{{assigned_pipeline}}
+    {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %}
+      - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}
    {% endfor %}
  {% endfor %}
    - require:
--- a/salt/logstash/map.jinja
+++ b/salt/logstash/map.jinja
@@ -1,4 +1,7 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
+{% import_yaml 'logstash/defaults.yaml' as LOGSTASH_DEFAULTS %}
+{% set LOGSTASH_MERGED = salt['pillar.get']('logstash', LOGSTASH_DEFAULTS.logstash, merge=True) %}
+
 {% set REDIS_NODES = [] %}
 {% set LOGSTASH_NODES = [] %}
 {% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
--- a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja
+++ b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja
@@ -1,5 +1,3 @@
-{%- set THREADS = salt['pillar.get']('logstash:settings:pipeline_x_workers') %}
-{%- set BATCH = salt['pillar.get']('logstash:settings:pipeline_x_batch_x_size', 125) %}
 {%- from 'logstash/map.jinja' import REDIS_NODES with context %}
 {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %}

--- a/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja
+++ b/salt/logstash/pipelines/config/so/9999_output_redis.conf.jinja
@@ -3,7 +3,6 @@
 {%- else %}
  {%- set HOST = GLOBALS.manager %}
 {%- endif %}
-{%- set BATCH = salt['pillar.get']('logstash:settings:pipeline_x_batch_x_size') %}
 {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %}

 output {