From b8afef1ee4edfb447b2c4cb11c2ce828b2419ad5 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Thu, 19 Dec 2024 14:56:40 -0500 Subject: [PATCH 1/6] cloud installs should use the local docker registry data --- setup/so-functions | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 914e0c2cd..94b6aab21 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -962,7 +962,12 @@ docker_seed_update() { docker_seed_registry() { local VERSION="$SOVERSION" - if ! [ -f /nsm/docker-registry/docker/registry.tar ]; then + if [ -f /nsm/docker-registry/docker/registry.tar ]; then + logCmd "tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker" + logCmd "rm /nsm/docker-registry/docker/registry.tar" + elif [ -d /nsm/docker-registry/docker/registry ] && [ -f /etc/SOCLOUD ]; then + echo "Using existing docker registry content for cloud install" + else if [ "$install_type" == 'IMPORT' ]; then container_list 'so-import' else @@ -972,9 +977,6 @@ docker_seed_registry() { docker_seed_update_percent=25 update_docker_containers 'netinstall' '' 'docker_seed_update' '/dev/stdout' 2>&1 | tee -a "$setup_log" - else - logCmd "tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker" - logCmd "rm /nsm/docker-registry/docker/registry.tar" fi } From 09ef09662046014a2670dc6ff1e9cd11e1655339 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 23 Dec 2024 08:27:45 -0500 Subject: [PATCH 2/6] Update soup --- salt/manager/tools/sbin/soup | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 77227c569..fc0c7aca4 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -404,7 +404,8 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.80 ]] && up_to_2.4.90 [[ "$INSTALLEDVERSION" == 2.4.90 ]] && up_to_2.4.100 [[ "$INSTALLEDVERSION" == 2.4.100 ]] && up_to_2.4.110 - [[ "$INSTALLEDVERSION" == 2.4.110 ]] && up_to_2.4.120 + [[ "$INSTALLEDVERSION" == 2.4.110 ]] && up_to_2.4.111 + [[ "$INSTALLEDVERSION" == 2.4.111 ]] && up_to_2.4.120 true } @@ -519,6 +520,11 @@ post_to_2.4.110() { POSTVERSION=2.4.110 } +post_to_2.4.111() { + echo "Nothing to apply" + POSTVERSION=2.4.111 +} + post_to_2.4.120() { update_elasticsearch_index_settings POSTVERSION=2.4.120 @@ -714,6 +720,12 @@ up_to_2.4.110() { INSTALLEDVERSION=2.4.110 } +up_to_2.4.111() { + echo "Nothing to do for 2.4.111" + + INSTALLEDVERSION=2.4.111 +} + up_to_2.4.120() { add_hydra_pillars From 7237b8971e6264b464c3b4d2773b1f374914bc00 Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Mon, 23 Dec 2024 15:41:13 -0500 Subject: [PATCH 3/6] Refactor pipeline for hash changes --- salt/soc/files/soc/sigma_so_pipeline.yaml | 41 ++++++++++++----------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index df8b2709a..48e9e1215 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -45,24 +45,25 @@ transformations: rule_conditions: - type: logsource category: antivirus - # Drops the Hashes field which is specific to Sysmon logs - # Ingested sysmon logs will have the Hashes field mapped to ECS specific fields - - id: hashes_drop_sysmon-specific-field - type: drop_detection_item + # Transforms the `Hashes` field to ECS fields + # ECS fields are used by the hash fields emitted by Elastic Defend + # If shipped with Elastic Agent, sysmon logs will also have hashes mapped to ECS fields + - id: hashes_break_out_field + type: hashes_fields + valid_hash_algos: ["MD5", "SHA1", "SHA256", "SHA512", "IMPHASH"] + field_prefix: "file" + drop_algo_prefix: False field_name_conditions: - type: include_fields fields: - - winlog.event_data.Hashes - rule_conditions: - - type: logsource - product: windows + - winlog.event_data.Hashes - id: hashes_process-creation type: field_name_mapping mapping: - winlog.event_data.sha256: process.hash.sha256 - winlog.event_data.sha1: process.hash.sha1 - winlog.event_data.md5: process.hash.md5 - winlog.event_data.Imphash: process.pe.imphash + fileSHA256: process.hash.sha256 + fileSHA1: process.hash.sha1 + fileMD5: process.hash.md5 + fileIMPHASH: process.pe.imphash rule_conditions: - type: logsource product: windows @@ -70,10 +71,10 @@ transformations: - id: hashes_image-load type: field_name_mapping mapping: - winlog.event_data.sha256: dll.hash.sha256 - winlog.event_data.sha1: dll.hash.sha1 - winlog.event_data.md5: dll.hash.md5 - winlog.event_data.Imphash: dll.pe.imphash + fileSHA256: dll.hash.sha256 + fileSHA1: dll.hash.sha1 + fileMD5: dll.hash.md5 + fileIMPHASH: dll.pe.imphash rule_conditions: - type: logsource product: windows @@ -81,10 +82,10 @@ transformations: - id: hashes_driver-load type: field_name_mapping mapping: - winlog.event_data.sha256: dll.hash.sha256 - winlog.event_data.sha1: dll.hash.sha1 - winlog.event_data.md5: dll.hash.md5 - winlog.event_data.Imphash: dll.pe.imphash + fileSHA256: dll.hash.sha256 + fileSHA1: dll.hash.sha1 + fileMD5: dll.hash.md5 + fileIMPHASH: dll.pe.imphash rule_conditions: - type: logsource product: windows From 9f838539221661bb36b5194a37daff90f4a57ee3 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 31 Dec 2024 13:44:20 -0600 Subject: [PATCH 4/6] Zeek QUIC support Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/elasticsearch/files/ingest/zeek.quic | 18 ++++++++++++++++++ salt/soc/defaults.yaml | 3 +++ 2 files changed, 21 insertions(+) create mode 100644 salt/elasticsearch/files/ingest/zeek.quic diff --git a/salt/elasticsearch/files/ingest/zeek.quic b/salt/elasticsearch/files/ingest/zeek.quic new file mode 100644 index 000000000..9a58bda82 --- /dev/null +++ b/salt/elasticsearch/files/ingest/zeek.quic @@ -0,0 +1,18 @@ +{ + "description" : "zeek.quic", + "processors" : [ + { "set": { "field": "event.dataset", "value": "quic" } }, + { "set": { "field": "network.transport", "value": "udp" } }, + { "json": { "field": "message", "target_field": "message2", "ignore_failure": true } }, + { "rename": { "field": "message2.version", "target_field": "quic.version", "ignore_missing": true } }, + { "rename": { "field": "message2.client_initial_dcid", "target_field": "quic.client_initial_dcid", "ignore_missing": true } }, + { "rename": { "field": "message2.client_scid", "target_field": "quic.client_scid", "ignore_missing": true } }, + { "rename": { "field": "message2.server_scid", "target_field": "quic.server_scid", "ignore_missing": true } }, + { "rename": { "field": "message2.server_name", "target_field": "quic.server_name", "ignore_missing": true } }, + { "rename": { "field": "message2.client_protocol", "target_field": "quic.client_protocol", "ignore_missing": true } }, + { "rename": { "field": "message2.history", "target_field": "quic.history", "ignore_missing": true } }, + { "remove": { "field": "message2.tags", "ignore_failure": true } }, + { "remove": { "field": ["host"], "ignore_failure": true } }, + { "pipeline": { "name": "zeek.common" } } + ] +} \ No newline at end of file diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 580b6993f..6f672843f 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1950,6 +1950,9 @@ soc: - name: PE description: PE (Portable Executable) files transferred via network traffic query: 'tags:pe | groupby file.machine | groupby -sankey file.machine file.os | groupby file.os | groupby -sankey file.os file.subsystem | groupby file.subsystem | groupby file.section_names | groupby file.is_exe | groupby file.is_64bit' + - name: QUIC + description: QUIC network metadata + query: 'tags:quic | groupby quic.server_name | groupby source.ip | groupby destination.ip | groupby -sankey source.ip quic.server_name | groupby destination.port | groupby -sankey source.ip quic.client_initial_dcid quic.client_scid destination_geo.organization_name | groupby quic.server_scid | groupby quic.version | groupby quic.client_protocol' - name: RADIUS description: RADIUS (Remote Authentication Dial-In User Service) network metadata query: 'tags:radius | groupby user.name | groupby -sankey user.name source.ip | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' From 927b618ec97e2caeb3c8c0367651da620e6a2107 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Thu, 2 Jan 2025 06:57:56 -0500 Subject: [PATCH 5/6] Update Zeek QUIC dashboard, add Hunt query, add quic.server.name as column in Events table --- salt/soc/defaults.yaml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 6f672843f..813b54223 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -339,6 +339,16 @@ soc: - file.os - file.subsystem - log.id.fuid + '::quic': + - soc_timestamp + - event.dataset + - source.ip + - source.port + - destination.ip + - destination.port + - quic.server_name + - log.id.uid + - network.community_id '::radius': - soc_timestamp - event.dataset @@ -1732,6 +1742,10 @@ soc: description: PE files list query: 'tags:pe | groupby file.machine file.os file.subsystem' showSubtitle: true + - name: QUIC + description: QUIC connections + query: 'tags:quic | groupby quic.server_name | groupby source.ip quic.server_name destination.ip' + showSubtitle: true - name: RADIUS description: RADIUS grouped by username query: 'tags:radius | groupby user.name' @@ -1952,7 +1966,7 @@ soc: query: 'tags:pe | groupby file.machine | groupby -sankey file.machine file.os | groupby file.os | groupby -sankey file.os file.subsystem | groupby file.subsystem | groupby file.section_names | groupby file.is_exe | groupby file.is_64bit' - name: QUIC description: QUIC network metadata - query: 'tags:quic | groupby quic.server_name | groupby source.ip | groupby destination.ip | groupby -sankey source.ip quic.server_name | groupby destination.port | groupby -sankey source.ip quic.client_initial_dcid quic.client_scid destination_geo.organization_name | groupby quic.server_scid | groupby quic.version | groupby quic.client_protocol' + query: 'tags:quic | groupby quic.server_name | groupby -sankey quic.server_name source.ip | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name | groupby quic.server_scid | groupby quic.version | groupby quic.client_protocol' - name: RADIUS description: RADIUS (Remote Authentication Dial-In User Service) network metadata query: 'tags:radius | groupby user.name | groupby -sankey user.name source.ip | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' From 3d3f0460fad532c1a5e207fabe3f327dd4ea167b Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 6 Jan 2025 14:42:16 -0600 Subject: [PATCH 6/6] move addon integration script run to elasticfleet state Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/elasticfleet/enabled.sls | 4 ++++ salt/elasticsearch/enabled.sls | 7 ------- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/salt/elasticfleet/enabled.sls b/salt/elasticfleet/enabled.sls index f91074b39..5a52f3a41 100644 --- a/salt/elasticfleet/enabled.sls +++ b/salt/elasticfleet/enabled.sls @@ -151,6 +151,10 @@ so-elastic-fleet-integration-upgrade: cmd.run: - name: /usr/sbin/so-elastic-fleet-integration-upgrade +so-elastic-fleet-addon-integrations: + cmd.run: + - name: /usr/sbin/so-elastic-fleet-optional-integrations-load + {% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %} so-elastic-defend-manage-filters-file-watch: cmd.run: diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index fb3f877df..4ed4b1b98 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -172,13 +172,6 @@ so-elasticsearch-ilm-policy-load: - onchanges: - file: so-elasticsearch-ilm-policy-load-script -configure-addon-fleet-integrations: - cmd.run: - - name: /usr/sbin/so-elastic-fleet-optional-integrations-load - - cwd: /opt/so - - require: - - docker_container: so-elasticsearch - so-elasticsearch-templates-reload: file.absent: - name: /opt/so/state/estemplates.txt