From cd4bd6460aa117ebc798c6f74f5ddfa32b6fc65f Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 Jan 2024 20:16:18 +0000 Subject: [PATCH 1/8] Custom pipelines --- salt/elasticsearch/defaults.yaml | 81 ++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index ce1bfb08d..541f3fae7 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -55,6 +55,87 @@ elasticsearch: key: /usr/share/elasticsearch/config/elasticsearch.key verification_mode: none enabled: false + pipelines: + custom01: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom01 + - pipeline: + name: common + custom02: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom02 + - pipeline: + name: common + custom03: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom03 + - pipeline: + name: common + custom04: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom04 + - pipeline: + name: common + custom05: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom05 + - pipeline: + name: common + custom06: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom06 + - pipeline: + name: common + custom07: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom07 + - pipeline: + name: common + custom08: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom08 + - pipeline: + name: common + custom09: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom09 + - pipeline: + name: common + custom10: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom10 + - pipeline: + name: common index_settings: global_overrides: index_template: From bc75be940243793ee0b33c031efbcc00c2c988e3 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 Jan 2024 20:16:48 +0000 Subject: [PATCH 2/8] Custom pipelines in UI --- salt/elasticsearch/config.sls | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/salt/elasticsearch/config.sls b/salt/elasticsearch/config.sls index 23e11a710..27a8a0fd6 100644 --- a/salt/elasticsearch/config.sls +++ b/salt/elasticsearch/config.sls @@ -118,6 +118,19 @@ esingestconf: - user: 930 - group: 939 +# Auto-generate Elasticsearch ingest node pipelines from pillar +{% for pipeline, config in ELASTICSEARCHMERGED.pipelines.items() %} +es_ingest_conf_{{pipeline}}: + file.managed: + - name: /opt/so/conf/elasticsearch/ingest/{{ pipeline }} + - source: salt://elasticsearch/base-template.json.jinja + - defaults: + TEMPLATE_CONFIG: {{ config }} + - template: jinja + - onchanges_in: + - file: so-pipelines-reload +{% endfor %} + eslog4jfile: file.managed: - name: /opt/so/conf/elasticsearch/log4j2.properties From 1853dc398bc377dc367cd0aaeeaefece7acec3c5 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 Jan 2024 20:17:33 +0000 Subject: [PATCH 3/8] Custom pipeline configuration --- salt/logstash/defaults.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index e4c18cc64..2cafce6fd 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -42,6 +42,24 @@ logstash: custom2: [] custom3: [] custom4: [] + pipeline_config: + custom01: |- + filter { + if [event][module] =~ "zeek" { + mutate { + add_tag => ["network_stuff"] + } + } + } + custom02: PLACEHOLDER + custom03: PLACEHOLDER + custom04: PLACEHOLDER + custom05: PLACEHOLDER + custom06: PLACEHOLDER + custom07: PLACEHOLDER + custom08: PLACEHOLDER + custom09: PLACEHOLDER + custom10: PLACEHOLDER settings: lsheap: 500m config: From 4672a5b8ebe83e4c5abac79abf5e87ab3c1f3d39 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 Jan 2024 20:18:17 +0000 Subject: [PATCH 4/8] Custom pipeline configuration in UI --- salt/logstash/config.sls | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/salt/logstash/config.sls b/salt/logstash/config.sls index 6ff33ff1a..8a59c83b7 100644 --- a/salt/logstash/config.sls +++ b/salt/logstash/config.sls @@ -63,6 +63,20 @@ lspipelinedir: - user: 931 - group: 939 +# Auto-generate Logstash pipeline config +{% for pipeline, config in LOGSTASH_MERGED.pipeline_config.items() %} +{% for assigned_pipeline in ASSIGNED_PIPELINES %} +{% set custom_pipeline = 'custom/' + pipeline + '.conf' %} +{% if custom_pipeline in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} +ls_custom_pipeline_conf_{{assigned_pipeline}}_{{pipeline}}: + file.managed: + - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{ pipeline }}.conf + - contents: LOGSTASH_MERGED.pipeline_config.{{pipeline}} +{% endif %} +{% endfor %} +{% endfor %} + + {% for assigned_pipeline in ASSIGNED_PIPELINES %} {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %} ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}: From bc502cc065c934d1afad9c08c4001a5bcb9dd732 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 Jan 2024 21:46:33 +0000 Subject: [PATCH 5/8] Custom Elasticserach pipeline annotations --- salt/elasticsearch/soc_elasticsearch.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index c54e07660..0f410e716 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -45,6 +45,28 @@ elasticsearch: description: Max number of boolean clauses per query. global: True helpLink: elasticsearch.html + pipelines: + custom01: &pipelines + description: + description: Description of the ingest node pipeline + global: True + advanced: True + helpLink: elasticsearch.html + processors: + description: Processors for the ingest node pipeline + global: True + advanced: True + multiline: True + helpLink: elasticsearch.html + custom02: *pipelines + custom03: *pipelines + custom04: *pipelines + custom05: *pipelines + custom06: *pipelines + custom07: *pipelines + custom08: *pipelines + custom09: *pipelines + custom10: *pipelines index_settings: global_overrides: index_template: From 136097f9816b30b85804a2b43ddf208205c28581 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 Jan 2024 21:47:09 +0000 Subject: [PATCH 6/8] Custom Logstash pipeline annotations --- salt/logstash/soc_logstash.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index bcb99bad5..dc52d63a8 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -31,6 +31,22 @@ logstash: custom2: *defined_pipelines custom3: *defined_pipelines custom4: *defined_pipelines + pipeline_config: + custom01: &pipeline_config + description: Pipeline configuration for Logstash + advanced: True + multiline: True + forcedType: string + helpLink: logstash.html + custom02: *pipeline_config + custom03: *pipeline_config + custom04: *pipeline_config + custom05: *pipeline_config + custom06: *pipeline_config + custom07: *pipeline_config + custom08: *pipeline_config + custom09: *pipeline_config + custom10: *pipeline_config settings: lsheap: description: Heap size to use for logstash From 182667bafb3769b636f715385d24805dda40c2bb Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 1 Feb 2024 13:59:23 +0000 Subject: [PATCH 7/8] Change numbers for Elasticsearch --- salt/elasticsearch/defaults.yaml | 40 +++++++++++------------ salt/elasticsearch/soc_elasticsearch.yaml | 20 ++++++------ 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 541f3fae7..03cd6d519 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -56,84 +56,84 @@ elasticsearch: verification_mode: none enabled: false pipelines: - custom01: + custom001: description: Custom Pipeline processors: - set: field: tags - value: custom01 + value: custom001 - pipeline: name: common - custom02: + custom002: description: Custom Pipeline processors: - set: field: tags - value: custom02 + value: custom002 - pipeline: name: common - custom03: + custom003: description: Custom Pipeline processors: - set: field: tags - value: custom03 + value: custom003 - pipeline: name: common - custom04: + custom004: description: Custom Pipeline processors: - set: field: tags - value: custom04 + value: custom004 - pipeline: name: common - custom05: + custom005: description: Custom Pipeline processors: - set: field: tags - value: custom05 + value: custom005 - pipeline: name: common - custom06: + custom006: description: Custom Pipeline processors: - set: field: tags - value: custom06 + value: custom006 - pipeline: name: common - custom07: + custom007: description: Custom Pipeline processors: - set: field: tags - value: custom07 + value: custom007 - pipeline: name: common - custom08: + custom008: description: Custom Pipeline processors: - set: field: tags - value: custom08 + value: custom008 - pipeline: name: common - custom09: + custom009: description: Custom Pipeline processors: - set: field: tags - value: custom09 + value: custom009 - pipeline: name: common - custom10: + custom010: description: Custom Pipeline processors: - set: field: tags - value: custom10 + value: custom010 - pipeline: name: common index_settings: diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 0f410e716..9a64190b3 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -46,7 +46,7 @@ elasticsearch: global: True helpLink: elasticsearch.html pipelines: - custom01: &pipelines + custom001: &pipelines description: description: Description of the ingest node pipeline global: True @@ -58,15 +58,15 @@ elasticsearch: advanced: True multiline: True helpLink: elasticsearch.html - custom02: *pipelines - custom03: *pipelines - custom04: *pipelines - custom05: *pipelines - custom06: *pipelines - custom07: *pipelines - custom08: *pipelines - custom09: *pipelines - custom10: *pipelines + custom002: *pipelines + custom003: *pipelines + custom004: *pipelines + custom005: *pipelines + custom006: *pipelines + custom007: *pipelines + custom008: *pipelines + custom009: *pipelines + custom010: *pipelines index_settings: global_overrides: index_template: From 1818e134cafe3c29224d66cd623f839d2065e8cd Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 1 Feb 2024 14:01:55 +0000 Subject: [PATCH 8/8] Change numbers for Logstash --- salt/logstash/defaults.yaml | 20 ++++++++++---------- salt/logstash/soc_logstash.yaml | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index 2cafce6fd..348acb622 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -43,7 +43,7 @@ logstash: custom3: [] custom4: [] pipeline_config: - custom01: |- + custom001: |- filter { if [event][module] =~ "zeek" { mutate { @@ -51,15 +51,15 @@ logstash: } } } - custom02: PLACEHOLDER - custom03: PLACEHOLDER - custom04: PLACEHOLDER - custom05: PLACEHOLDER - custom06: PLACEHOLDER - custom07: PLACEHOLDER - custom08: PLACEHOLDER - custom09: PLACEHOLDER - custom10: PLACEHOLDER + custom002: PLACEHOLDER + custom003: PLACEHOLDER + custom004: PLACEHOLDER + custom005: PLACEHOLDER + custom006: PLACEHOLDER + custom007: PLACEHOLDER + custom008: PLACEHOLDER + custom009: PLACEHOLDER + custom010: PLACEHOLDER settings: lsheap: 500m config: diff --git a/salt/logstash/soc_logstash.yaml b/salt/logstash/soc_logstash.yaml index dc52d63a8..3172ff7c5 100644 --- a/salt/logstash/soc_logstash.yaml +++ b/salt/logstash/soc_logstash.yaml @@ -32,21 +32,21 @@ logstash: custom3: *defined_pipelines custom4: *defined_pipelines pipeline_config: - custom01: &pipeline_config + custom001: &pipeline_config description: Pipeline configuration for Logstash advanced: True multiline: True forcedType: string helpLink: logstash.html - custom02: *pipeline_config - custom03: *pipeline_config - custom04: *pipeline_config - custom05: *pipeline_config - custom06: *pipeline_config - custom07: *pipeline_config - custom08: *pipeline_config - custom09: *pipeline_config - custom10: *pipeline_config + custom002: *pipeline_config + custom003: *pipeline_config + custom004: *pipeline_config + custom005: *pipeline_config + custom006: *pipeline_config + custom007: *pipeline_config + custom008: *pipeline_config + custom009: *pipeline_config + custom010: *pipeline_config settings: lsheap: description: Heap size to use for logstash