diff --git a/salt/elasticsearch/config.sls b/salt/elasticsearch/config.sls index 23e11a710..27a8a0fd6 100644 --- a/salt/elasticsearch/config.sls +++ b/salt/elasticsearch/config.sls @@ -118,6 +118,19 @@ esingestconf: - user: 930 - group: 939 +# Auto-generate Elasticsearch ingest node pipelines from pillar +{% for pipeline, config in ELASTICSEARCHMERGED.pipelines.items() %} +es_ingest_conf_{{pipeline}}: + file.managed: + - name: /opt/so/conf/elasticsearch/ingest/{{ pipeline }} + - source: salt://elasticsearch/base-template.json.jinja + - defaults: + TEMPLATE_CONFIG: {{ config }} + - template: jinja + - onchanges_in: + - file: so-pipelines-reload +{% endfor %} + eslog4jfile: file.managed: - name: /opt/so/conf/elasticsearch/log4j2.properties diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index ce1bfb08d..03cd6d519 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -55,6 +55,87 @@ elasticsearch: key: /usr/share/elasticsearch/config/elasticsearch.key verification_mode: none enabled: false + pipelines: + custom001: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom001 + - pipeline: + name: common + custom002: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom002 + - pipeline: + name: common + custom003: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom003 + - pipeline: + name: common + custom004: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom004 + - pipeline: + name: common + custom005: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom005 + - pipeline: + name: common + custom006: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom006 + - pipeline: + name: common + custom007: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom007 + - pipeline: + name: common + custom008: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom008 + - pipeline: + name: common + custom009: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom009 + - pipeline: + name: common + custom010: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom010 + - pipeline: + name: common index_settings: global_overrides: index_template: diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index c54e07660..9a64190b3 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -45,6 +45,28 @@ elasticsearch: description: Max number of boolean clauses per query. global: True helpLink: elasticsearch.html + pipelines: + custom001: &pipelines + description: + description: Description of the ingest node pipeline + global: True + advanced: True + helpLink: elasticsearch.html + processors: + description: Processors for the ingest node pipeline + global: True + advanced: True + multiline: True + helpLink: elasticsearch.html + custom002: *pipelines + custom003: *pipelines + custom004: *pipelines + custom005: *pipelines + custom006: *pipelines + custom007: *pipelines + custom008: *pipelines + custom009: *pipelines + custom010: *pipelines index_settings: global_overrides: index_template: diff --git a/salt/logstash/config.sls b/salt/logstash/config.sls index 6ff33ff1a..8a59c83b7 100644 --- a/salt/logstash/config.sls +++ b/salt/logstash/config.sls @@ -63,6 +63,20 @@ lspipelinedir: - user: 931 - group: 939 +# Auto-generate Logstash pipeline config +{% for pipeline, config in LOGSTASH_MERGED.pipeline_config.items() %} +{% for assigned_pipeline in ASSIGNED_PIPELINES %} +{% set custom_pipeline = 'custom/' + pipeline + '.conf' %} +{% if custom_pipeline in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} +ls_custom_pipeline_conf_{{assigned_pipeline}}_{{pipeline}}: + file.managed: + - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{ pipeline }}.conf + - contents: LOGSTASH_MERGED.pipeline_config.{{pipeline}} +{% endif %} +{% endfor %} +{% endfor %} + + {% for assigned_pipeline in ASSIGNED_PIPELINES %} {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index e4c18cc64..348acb622 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -42,6 +42,24 @@ logstash: custom2: [] custom3: [] custom4: [] + pipeline_config: + custom001: |- + filter { + if [event][module] =~ "zeek" { + mutate { + add_tag => ["network_stuff"] + } + } + } + custom002: PLACEHOLDER + custom003: PLACEHOLDER + custom004: PLACEHOLDER + custom005: PLACEHOLDER + custom006: PLACEHOLDER + custom007: PLACEHOLDER + custom008: PLACEHOLDER + custom009: PLACEHOLDER + custom010: PLACEHOLDER settings: lsheap: 500m config: diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index bcb99bad5..3172ff7c5 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -31,6 +31,22 @@ logstash: custom2: *defined_pipelines custom3: *defined_pipelines custom4: *defined_pipelines + pipeline_config: + custom001: &pipeline_config + description: Pipeline configuration for Logstash + advanced: True + multiline: True + forcedType: string + helpLink: logstash.html + custom002: *pipeline_config + custom003: *pipeline_config + custom004: *pipeline_config + custom005: *pipeline_config + custom006: *pipeline_config + custom007: *pipeline_config + custom008: *pipeline_config + custom009: *pipeline_config + custom010: *pipeline_config settings: lsheap: description: Heap size to use for logstash