From 2bb77251b0c0d5e8c4e3c4412a5e42f105242837 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 May 2023 13:38:58 +0000 Subject: [PATCH 1/7] Move Elastic Fleet logging exclusions to the Fleet pillar --- .../integrations-dynamic/grid-nodes/import-zeek-logs.json | 4 ++-- .../files/integrations-dynamic/grid-nodes/zeek-logs.json | 4 ++-- salt/elasticfleet/soc_elasticfleet.yaml | 6 ++++++ salt/zeek/soc_zeek.yaml | 5 ----- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes/import-zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes/import-zeek-logs.json index dc94afbaa..feaebf60b 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes/import-zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes/import-zeek-logs.json @@ -1,4 +1,4 @@ -{% from 'zeek/config.map.jinja' import ZEEKMERGED %} +{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} {%- raw -%} { "package": { @@ -22,7 +22,7 @@ "data_stream.dataset": "import", "tags": [], "processors": "- dissect:\n tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{import.file}\"\n field: \"log.file.path\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"import.file\").slice(0,-4);\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n imported: true\n- add_tags:\n tags: \"ics\"\n when:\n regexp:\n import.file: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"", - "custom": "exclude_files: [\"{%- endraw -%}{{ ZEEKMERGED.logging.excluded | join('|') }}{%- raw -%}.log$\"]\n" + "custom": "exclude_files: [\"{%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%}.log$\"]\n" } } } diff --git a/salt/elasticfleet/files/integrations-dynamic/grid-nodes/zeek-logs.json b/salt/elasticfleet/files/integrations-dynamic/grid-nodes/zeek-logs.json index 5e2ed4f9b..e2dd069ab 100644 --- a/salt/elasticfleet/files/integrations-dynamic/grid-nodes/zeek-logs.json +++ b/salt/elasticfleet/files/integrations-dynamic/grid-nodes/zeek-logs.json @@ -1,4 +1,4 @@ -{% from 'zeek/config.map.jinja' import ZEEKMERGED %} +{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %} {%- raw -%} { "package": { @@ -23,7 +23,7 @@ "data_stream.dataset": "zeek", "tags": [], "processors": "- dissect:\n tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n field: \"log.file.path\"\n trim_chars: \".log\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\");\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n- add_tags:\n tags: \"ics\"\n when:\n regexp:\n pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"", - "custom": "exclude_files: [\"{%- endraw -%}{{ ZEEKMERGED.logging.excluded | join('|') }}{%- raw -%}.log$\"]\n" + "custom": "exclude_files: [\"{%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%}.log$\"]\n" } } } diff --git a/salt/elasticfleet/soc_elasticfleet.yaml b/salt/elasticfleet/soc_elasticfleet.yaml index e8bf03ad1..80b3a22b5 100644 --- a/salt/elasticfleet/soc_elasticfleet.yaml +++ b/salt/elasticfleet/soc_elasticfleet.yaml @@ -3,6 +3,12 @@ elasticfleet: description: You can enable or disable Elastic Fleet. advanced: True helpLink: elastic-fleet.html + logging: + zeek: + excluded: + description: This is a list of Zeek logs that are excluded from being shipped through the data processing pipeline. If you remove a log from this list, Elastic Agent will attempt to process it. If an ingest node pipeline is not available to process the logs, you may experience errors. + forcedType: "[]string" + helpLink: zeek.html config: server: endpoints_enrollment: diff --git a/salt/zeek/soc_zeek.yaml b/salt/zeek/soc_zeek.yaml index 2b8bb3969..b1d0d7f7f 100644 --- a/salt/zeek/soc_zeek.yaml +++ b/salt/zeek/soc_zeek.yaml @@ -2,11 +2,6 @@ zeek: enabled: description: You can enable or disable ZEEK on all sensors or a single sensor. helpLink: zeek.html - logging: - excluded: - description: This is a list of Zeek logs that are excluded from being shipped through the data processing pipeline. If you remove a log from this list, Elastic Agent will attempt to process it. If an ingest node pipeline is not available to process the logs, you may experience errors. - forcedType: "[]string" - helpLink: zeek.html config: local: load: From 20aaa794763c607a696d9c91517a1b690ae75bd2 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 May 2023 13:45:19 +0000 Subject: [PATCH 2/7] Add pillar files for Fleet --- setup/so-functions | 6 ++++++ setup/so-variables | 8 +++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 247cf6c94..6df738608 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -898,6 +898,7 @@ create_local_nids_rules() { } create_manager_pillars() { + elasticfleet_pillar elasticsearch_pillar logstash_pillar manager_pillar @@ -1120,6 +1121,11 @@ docker_seed_registry() { fi } +elasticfleet_pillar() { + touch $adv_elasticfleet_pillar_file + touch $elasticfleet_pillar_file +} + elasticsearch_pillar() { title "Create Advanced File" logCmd "touch $adv_elasticsearch_pillar_file" diff --git a/setup/so-variables b/setup/so-variables index b2e439a5c..2c7cb3dba 100644 --- a/setup/so-variables +++ b/setup/so-variables @@ -82,6 +82,12 @@ export global_pillar_file adv_global_pillar_file="$local_salt_dir/pillar/global/adv_global.sls" export adv_global_pillar_file +elasticfleet_pillar_file="$local_salt_dir/pillar/elasticfleet/soc_elasticfleet.sls" +export elasticfleet_pillar_file + +adv_elasticfleet_pillar_file="$local_salt_dir/pillar/elasticfleet/adv_elasticfleet.sls" +export adv_elasticfleet_pillar_file + elasticsearch_pillar_file="$local_salt_dir/pillar/elasticsearch/soc_elasticsearch.sls" export elasticsearch_pillar_file @@ -212,4 +218,4 @@ patch_pillar_file="$local_salt_dir/pillar/patch/soc_patch.sls" export patch_pillar_file adv_patch_pillar_file="$local_salt_dir/pillar/patch/adv_patch.sls" -export adv_patch_pillar_file \ No newline at end of file +export adv_patch_pillar_file From 9ae26ec8666bb3a8ef1c15dab52305070227c017 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 May 2023 14:21:39 +0000 Subject: [PATCH 3/7] Add Fleet to top file --- pillar/top.sls | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pillar/top.sls b/pillar/top.sls index 7a36dcc53..692d310b2 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -52,6 +52,8 @@ base: - influxdb.adv_influxdb - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch + - elasticfleet.adv_elasticfleet + - elasticfleet.soc_elasticfleet - elastalert.soc_elastalert - elastalert.adv_elastalert - backup.soc_backup @@ -75,6 +77,8 @@ base: - pcap.adv_pcap - suricata.soc_suricata - suricata.adv_suricata + - elasticfleet.adv_elasticfleet + - elasticfleet.soc_elasticfleet - minions.{{ grains.id }} - minions.adv_{{ grains.id }} @@ -91,6 +95,8 @@ base: - kratos.soc_kratos - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch + - elasticfleet.soc_elasticfleet + - elasticfleet.adv_elasticfleet - elastalert.soc_elastalert - elastalert.adv_elastalert - manager.soc_manager @@ -149,6 +155,8 @@ base: - influxdb.adv_influxdb - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch + - elasticfleet.soc_elasticfleet + - elasticfleet.adv_elasticfleet - elastalert.soc_elastalert - elastalert.adv_elastalert - manager.soc_manager @@ -183,6 +191,8 @@ base: - logstash.adv_logstash - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch + - elasticfleet.soc_elasticfleet + - elasticfleet.adv_elasticfleet - curator.soc_curator - curator.adv_curator - redis.soc_redis @@ -215,6 +225,8 @@ base: {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} + - elasticfleet.soc_elasticfleet + - elasticfleet.adv_elasticfleet - redis.soc_redis - redis.adv_redis - minions.{{ grains.id }} @@ -227,6 +239,8 @@ base: {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} + - elasticfleet.soc_elasticfleet + - elasticfleet.adv_elasticfleet - redis.soc_redis - redis.adv_redis - minions.{{ grains.id }} @@ -244,6 +258,8 @@ base: - kratos.soc_kratos - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch + - elasticfleet.soc_elasticfleet + - elasticfleet.adv_elasticfleet - elastalert.soc_elastalert - elastalert.adv_elastalert - manager.soc_manager @@ -283,6 +299,8 @@ base: - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash + - elasticfleet.soc_elasticfleet + - elasticfleet.adv_elasticfleet - minions.{{ grains.id }} - minions.adv_{{ grains.id }} From b3f60128566bab01fc9c0a9f4a9cd5af1045a602 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 May 2023 14:22:47 +0000 Subject: [PATCH 4/7] Change ordering --- pillar/top.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pillar/top.sls b/pillar/top.sls index 692d310b2..51897e8f9 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -52,8 +52,8 @@ base: - influxdb.adv_influxdb - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch - - elasticfleet.adv_elasticfleet - elasticfleet.soc_elasticfleet + - elasticfleet.adv_elasticfleet - elastalert.soc_elastalert - elastalert.adv_elastalert - backup.soc_backup @@ -77,8 +77,8 @@ base: - pcap.adv_pcap - suricata.soc_suricata - suricata.adv_suricata - - elasticfleet.adv_elasticfleet - elasticfleet.soc_elasticfleet + - elasticfleet.adv_elasticfleet - minions.{{ grains.id }} - minions.adv_{{ grains.id }} From ed560f19d3f4ca6abbaa69a4c08addb65db1a1c6 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 May 2023 14:28:43 +0000 Subject: [PATCH 5/7] Remove where not applicable --- pillar/top.sls | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pillar/top.sls b/pillar/top.sls index 51897e8f9..75117e35f 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -77,8 +77,6 @@ base: - pcap.adv_pcap - suricata.soc_suricata - suricata.adv_suricata - - elasticfleet.soc_elasticfleet - - elasticfleet.adv_elasticfleet - minions.{{ grains.id }} - minions.adv_{{ grains.id }} @@ -191,8 +189,6 @@ base: - logstash.adv_logstash - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch - - elasticfleet.soc_elasticfleet - - elasticfleet.adv_elasticfleet - curator.soc_curator - curator.adv_curator - redis.soc_redis @@ -225,8 +221,6 @@ base: {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} - - elasticfleet.soc_elasticfleet - - elasticfleet.adv_elasticfleet - redis.soc_redis - redis.adv_redis - minions.{{ grains.id }} @@ -239,8 +233,6 @@ base: {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} - - elasticfleet.soc_elasticfleet - - elasticfleet.adv_elasticfleet - redis.soc_redis - redis.adv_redis - minions.{{ grains.id }} From 3441c0684e7bc2a1eb9281884f9e1ad8d331c7de Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 May 2023 15:10:35 +0000 Subject: [PATCH 6/7] Create elasticfleet pillar dir --- setup/so-functions | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/so-functions b/setup/so-functions index 6df738608..a9d5b434e 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1122,6 +1122,7 @@ docker_seed_registry() { } elasticfleet_pillar() { + logCmd "mkdir -p $local_salt_dir/pillar/elasticfleet" touch $adv_elasticfleet_pillar_file touch $elasticfleet_pillar_file } From 344e2bf1d027d4e4c064dea9a16b45eb1a7adbe1 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 31 May 2023 15:30:03 +0000 Subject: [PATCH 7/7] Update defaults file --- salt/elasticfleet/defaults.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/salt/elasticfleet/defaults.yaml b/salt/elasticfleet/defaults.yaml index d29e08f9a..4da5123ac 100644 --- a/salt/elasticfleet/defaults.yaml +++ b/salt/elasticfleet/defaults.yaml @@ -6,3 +6,18 @@ elasticfleet: es_token: '' grid_enrollment: '' url: '' + logging: + zeek: + excluded: + - broker + - capture_loss + - ecat_arp_info + - known_hosts + - known_services + - loaded_scripts + - ntp + - packet_filter + - reporter + - stats + - stderr + - stdout