From 6d0350793db4fe29d879036f122da6453147b0aa Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Sun, 23 Feb 2025 14:02:17 -0500 Subject: [PATCH 01/11] Remove old defend json --- salt/manager/tools/sbin/soup | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 5da116e05..27123c660 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -533,9 +533,6 @@ post_to_2.4.120() { # Manually rollover suricata alerts index to ensure data_stream.dataset expected mapping is set to 'suricata' rollover_index "logs-suricata.alerts-so" - echo "Regenerating Elastic Agent Installers" - /sbin/so-elastic-agent-gen-installers - POSTVERSION=2.4.120 } @@ -546,6 +543,9 @@ post_to_2.4.130() { echo "Updating Kibana default space" /usr/sbin/so-kibana-space-defaults + echo "Regenerating Elastic Agent Installers" + /sbin/so-elastic-agent-gen-installers + POSTVERSION=2.4.130 } @@ -757,12 +757,12 @@ up_to_2.4.120() { } up_to_2.4.130() { + # Remove any old Elastic Defend config files + rm -f /opt/so/conf/elastic-fleet/integrations/endpoints-initial/elastic-defend-endpoints.json + # Elastic Update for this release, so download Elastic Agent files determine_elastic_agent_upgrade - # Integrations policies need to be updated - rm -f /opt/so/state/eaintegrations.txt - # Ensure override exists to allow nmcli access to other devices touch /etc/NetworkManager/conf.d/10-globally-managed-devices.conf From 3f2b0973af3aeb86d8adae58ea2a49d013b0e71d Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 24 Feb 2025 08:59:59 -0600 Subject: [PATCH 02/11] manually create unused logs-soc@package for successful elasticsearch templates load --- .../component/elastic-agent/logs-soc@package.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 salt/elasticsearch/templates/component/elastic-agent/logs-soc@package.json diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-soc@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-soc@package.json new file mode 100644 index 000000000..bf3c4f649 --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/logs-soc@package.json @@ -0,0 +1,7 @@ +{ + "package": { + "name": "log" + }, + "managed_by": "fleet", + "managed": true +} \ No newline at end of file From d7c06e5ff4622b5c370baacd4f20b3baf06d6d0b Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 24 Feb 2025 09:02:56 -0600 Subject: [PATCH 03/11] run elasticsearch state, right before completing soup to ensure templates for optional integrations are loaded --- salt/manager/tools/sbin/soup | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 27123c660..a6c9b7693 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -537,6 +537,10 @@ post_to_2.4.120() { } post_to_2.4.130() { + # Optional integrations are loaded AFTER initial successful load of core ES templates (/opt/so/state/estemplates.txt) + # Dynamic templates are created in elasticsearch.enabled for every optional integration based on output of so-elastic-fleet-optional-integrations-load script + echo "Ensuring Elasticsearch templates are up to date after updating package registry" + salt-call state.apply elasticsearch queue=True # Update kibana default space salt-call state.apply kibana.config queue=True From e2772e899e8d53d48f070242f008eb565a47509d Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 24 Feb 2025 10:24:11 -0600 Subject: [PATCH 04/11] component template missing metadata field --- .../templates/component/elastic-agent/logs-soc@package.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-soc@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-soc@package.json index bf3c4f649..a2ad15b79 100644 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-soc@package.json +++ b/salt/elasticsearch/templates/component/elastic-agent/logs-soc@package.json @@ -1,7 +1,10 @@ { + "template": {}, + "_meta": { "package": { "name": "log" }, "managed_by": "fleet", "managed": true + } } \ No newline at end of file From 17edc06987b70add37fa5408b8dc1790f1a29efb Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 24 Feb 2025 14:45:43 -0600 Subject: [PATCH 05/11] allow installing integrations that require an elastic license --- ...o-elastic-fleet-optional-integrations-load | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) rename salt/elasticfleet/tools/{sbin => sbin_jinja}/so-elastic-fleet-optional-integrations-load (80%) diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load similarity index 80% rename from salt/elasticfleet/tools/sbin/so-elastic-fleet-optional-integrations-load rename to salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load index dface5a72..f97ed577b 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-optional-integrations-load +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load @@ -3,6 +3,7 @@ # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. +{% set SUB = salt['pillar.get']('elasticfleet:config:subscription_integrations', default=false) %} . /usr/sbin/so-common . /usr/sbin/so-elastic-fleet-common @@ -16,7 +17,6 @@ BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json -SKIP_SUBSCRIPTION=true PENDING_UPDATE=false # Integrations which are included in the package registry, but excluded from automatic installation via this script. @@ -63,7 +63,8 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then bulk_package=$(echo "$package" | jq '{name: .name, version: .latest_version}' ) if [[ ! "${EXCLUDED_INTEGRATIONS[@]}" =~ "$package_name" ]]; then - if $SKIP_SUBSCRIPTION && [[ "$subscription" != "basic" && "$subscription" != "null" && -n "$subscription" ]]; then + {% if not SUB %} + if [[ "$subscription" != "basic" && "$subscription" != "null" && -n "$subscription" ]]; then # pass over integrations that require non-basic elastic license echo "$package_name integration requires an Elastic license of $subscription or greater... skipping" continue @@ -83,6 +84,20 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then fi fi fi + {% else %} + if [[ "$installed_version" == "null" || -z "$installed_version" ]]; then + echo "$package_name is not installed... Adding to next update." + jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST + PENDING_UPDATE=true + else + results=$(compare_versions "$latest_version" "$installed_version") + if [ $results == "greater" ]; then + echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update." + jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST + PENDING_UPDATE=true + fi + fi + {% endif %} else echo "Skipping $package_name..." fi From 80fed1e0451a4941e9ed47b2d43ebc8ec6fa9034 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 25 Feb 2025 21:47:56 -0600 Subject: [PATCH 06/11] default capinfos to use start/end time arg --- salt/common/tools/sbin_jinja/so-import-pcap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin_jinja/so-import-pcap b/salt/common/tools/sbin_jinja/so-import-pcap index d3886305e..5f2370a4a 100755 --- a/salt/common/tools/sbin_jinja/so-import-pcap +++ b/salt/common/tools/sbin_jinja/so-import-pcap @@ -63,7 +63,7 @@ function status { function pcapinfo() { PCAP=$1 ARGS=$2 - docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap $ARGS + docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap -ae $ARGS } function pcapfix() { From 9bc64bf4537bbc096dbca45da00308a92dc3273f Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 27 Feb 2025 16:48:07 -0600 Subject: [PATCH 07/11] managed int multiline input --- salt/elasticsearch/soc_elasticsearch.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index adce41bff..47013e48f 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -80,6 +80,7 @@ elasticsearch: managed_integrations: description: List of integrations to add into SOC config UI. Enter the full or partial integration name. Eg. 1password, 1pass forcedType: "[]string" + multiline: True global: True advanced: True helpLink: elasticsearch.html From e53f4fd1f1d089414f4317159689878bb55d6299 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Sun, 2 Mar 2025 05:54:30 -0500 Subject: [PATCH 08/11] Update defaults.yaml to quote the process.entity_id value --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index b97ba11e6..665b4106b 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -83,7 +83,7 @@ soc: icon: fa-users-between-lines target: '' links: - - '/#/hunt?q=({:process.entity_id}) | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.name | groupby process.command_line | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path' + - '/#/hunt?q="{:process.entity_id}" | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.name | groupby process.command_line | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path' - name: actionProcessAncestors description: actionProcessAncestorsHelp icon: fa-people-roof From 4bd83f8983aa7ae4b7e0df29a3da74e3e9ae38d6 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 3 Mar 2025 10:48:06 -0600 Subject: [PATCH 09/11] zeek traceroute & ntp Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/elasticsearch/files/ingest/zeek.ntp | 16 ++++++++++++++++ salt/elasticsearch/files/ingest/zeek.traceroute | 10 ++++++++++ 2 files changed, 26 insertions(+) create mode 100644 salt/elasticsearch/files/ingest/zeek.ntp create mode 100644 salt/elasticsearch/files/ingest/zeek.traceroute diff --git a/salt/elasticsearch/files/ingest/zeek.ntp b/salt/elasticsearch/files/ingest/zeek.ntp new file mode 100644 index 000000000..6c500c770 --- /dev/null +++ b/salt/elasticsearch/files/ingest/zeek.ntp @@ -0,0 +1,16 @@ +{ + "description" : "zeek.ntp", + "processors":[ + {"set": {"field":"event.dataset", "value":"ntp", "ignore_failure":true}}, + {"json": {"field":"message", "target_field":"message2", "ignore_failure":true}}, + {"rename": {"field":"message2.version", "target_field":"ntp.version", "ignore_missing":true}}, + {"rename": {"field":"message2.mode", "target_field":"ntp.mode", "ignore_missing":true}}, + {"rename": {"field":"message2.poll", "target_field":"ntp.poll", "ignore_missing":true}}, + {"rename": {"field":"message2.precision", "target_field":"ntp.precision", "ignore_missing":true}}, + {"rename": {"field":"message2.org_time", "target_field":"ntp.org_time", "ignore_missing":true}}, + {"rename": {"field":"message2.xmt_time", "target_field":"ntp.xmt_time", "ignore_missing":true}}, + {"date": {"field":"ntp.org_time", "target_field":"ntp.org_time", "formats":["UNIX", "UNIX_MS"], "ignore_failure": true, "if":"ctx?.ntp?.org_time != null"}}, + {"date": {"field":"ntp.xmt_time", "target_field":"ntp.xmt_time", "formats":["UNIX", "UNIX_MS"], "ignore_failure": true, "if":"ctx?.ntp?.xmt_time != null"}}, + {"pipeline":{"name":"zeek.common"}} + ] +} diff --git a/salt/elasticsearch/files/ingest/zeek.traceroute b/salt/elasticsearch/files/ingest/zeek.traceroute new file mode 100644 index 000000000..f34f58cb0 --- /dev/null +++ b/salt/elasticsearch/files/ingest/zeek.traceroute @@ -0,0 +1,10 @@ +{ + "description":"zeek.traceroute", + "processors":[ + {"set": {"field":"event.dataset", "value":"traceroute" }}, + {"json": {"field":"message", "target_field":"message2" }}, + {"rename": {"field":"message2.src", "target_field":"source.ip", "ignore_missing":true,"ignore_failure":true}}, + {"rename": {"field":"message2.dst", "target_field":"destination.ip", "ignore_missing":true,"ignore_failure":true}}, + {"pipeline": {"name":"zeek.common"}} + ] +} From 44535cba8c0403ed3797a98d5d4ab8d1e28b873b Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 4 Mar 2025 06:46:56 -0500 Subject: [PATCH 10/11] FIX: Elastic Agent Security Events dashboard should reference user.effective.name #14325 --- salt/soc/defaults.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 665b4106b..962d1096b 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1256,7 +1256,7 @@ soc: - soc_timestamp - event.dataset - host.name - - user.name + - user.effective.name - process.executable - event.action - event.outcome @@ -1918,7 +1918,7 @@ soc: query: 'event.dataset:endpoint.events.registry | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby -sankey user.name process.name | groupby process.name | groupby event.action | groupby registry.path' - name: Elastic Agent Security Events description: Security events from Elastic Agents - query: 'event.dataset:endpoint.events.security | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby -sankey user.name process.executable | groupby process.executable | groupby event.action | groupby event.outcome' + query: 'event.dataset:endpoint.events.security | groupby host.name | groupby -sankey host.name user.effective.name | groupby user.effective.name | groupby -sankey user.effective.name process.executable | groupby process.executable | groupby event.action | groupby event.outcome' - name: Host Overview description: Overview of all host data types query: '((event.category:registry OR event.category:host OR event.category:process OR event.category:driver OR event.category:configuration) OR (event.category:file AND _exists_:process.executable) OR (event.category:network AND _exists_:host.name)) | groupby event.dataset* event.category* event.action* | groupby event.type | groupby -sankey event.type host.name | groupby host.name | groupby user.name | groupby file.name | groupby process.executable' From e1c8bee71abd2b975b666a22be70f7562796bcdb Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 4 Mar 2025 08:58:41 -0600 Subject: [PATCH 11/11] install bc package --- salt/common/packages.sls | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/common/packages.sls b/salt/common/packages.sls index c5d2729fd..86bad228a 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -27,6 +27,7 @@ commonpkgs: - vim - tar - unzip + - bc {% if grains.oscodename != 'focal' %} - python3-rich {% endif %} @@ -56,6 +57,7 @@ commonpkgs: - skip_suggestions: True - pkgs: - python3-dnf-plugin-versionlock + - bc - curl - device-mapper-persistent-data - fuse