diff --git a/salt/common/packages.sls b/salt/common/packages.sls index c5d2729fd..86bad228a 100644 --- a/salt/common/packages.sls +++ b/salt/common/packages.sls @@ -27,6 +27,7 @@ commonpkgs: - vim - tar - unzip + - bc {% if grains.oscodename != 'focal' %} - python3-rich {% endif %} @@ -56,6 +57,7 @@ commonpkgs: - skip_suggestions: True - pkgs: - python3-dnf-plugin-versionlock + - bc - curl - device-mapper-persistent-data - fuse diff --git a/salt/common/tools/sbin_jinja/so-import-pcap b/salt/common/tools/sbin_jinja/so-import-pcap index d3886305e..5f2370a4a 100755 --- a/salt/common/tools/sbin_jinja/so-import-pcap +++ b/salt/common/tools/sbin_jinja/so-import-pcap @@ -63,7 +63,7 @@ function status { function pcapinfo() { PCAP=$1 ARGS=$2 - docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap $ARGS + docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap -ae $ARGS } function pcapfix() { diff --git a/salt/elasticfleet/tools/sbin/so-elastic-fleet-optional-integrations-load b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load similarity index 80% rename from salt/elasticfleet/tools/sbin/so-elastic-fleet-optional-integrations-load rename to salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load index dface5a72..f97ed577b 100644 --- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-optional-integrations-load +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load @@ -3,6 +3,7 @@ # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use # this file except in compliance with the Elastic License 2.0. +{% set SUB = salt['pillar.get']('elasticfleet:config:subscription_integrations', default=false) %} . /usr/sbin/so-common . /usr/sbin/so-elastic-fleet-common @@ -16,7 +17,6 @@ BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json -SKIP_SUBSCRIPTION=true PENDING_UPDATE=false # Integrations which are included in the package registry, but excluded from automatic installation via this script. @@ -63,7 +63,8 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then bulk_package=$(echo "$package" | jq '{name: .name, version: .latest_version}' ) if [[ ! "${EXCLUDED_INTEGRATIONS[@]}" =~ "$package_name" ]]; then - if $SKIP_SUBSCRIPTION && [[ "$subscription" != "basic" && "$subscription" != "null" && -n "$subscription" ]]; then + {% if not SUB %} + if [[ "$subscription" != "basic" && "$subscription" != "null" && -n "$subscription" ]]; then # pass over integrations that require non-basic elastic license echo "$package_name integration requires an Elastic license of $subscription or greater... skipping" continue @@ -83,6 +84,20 @@ if [[ -f $STATE_FILE_SUCCESS ]]; then fi fi fi + {% else %} + if [[ "$installed_version" == "null" || -z "$installed_version" ]]; then + echo "$package_name is not installed... Adding to next update." + jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST + PENDING_UPDATE=true + else + results=$(compare_versions "$latest_version" "$installed_version") + if [ $results == "greater" ]; then + echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update." + jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST + PENDING_UPDATE=true + fi + fi + {% endif %} else echo "Skipping $package_name..." fi diff --git a/salt/elasticsearch/files/ingest/zeek.ntp b/salt/elasticsearch/files/ingest/zeek.ntp new file mode 100644 index 000000000..6c500c770 --- /dev/null +++ b/salt/elasticsearch/files/ingest/zeek.ntp @@ -0,0 +1,16 @@ +{ + "description" : "zeek.ntp", + "processors":[ + {"set": {"field":"event.dataset", "value":"ntp", "ignore_failure":true}}, + {"json": {"field":"message", "target_field":"message2", "ignore_failure":true}}, + {"rename": {"field":"message2.version", "target_field":"ntp.version", "ignore_missing":true}}, + {"rename": {"field":"message2.mode", "target_field":"ntp.mode", "ignore_missing":true}}, + {"rename": {"field":"message2.poll", "target_field":"ntp.poll", "ignore_missing":true}}, + {"rename": {"field":"message2.precision", "target_field":"ntp.precision", "ignore_missing":true}}, + {"rename": {"field":"message2.org_time", "target_field":"ntp.org_time", "ignore_missing":true}}, + {"rename": {"field":"message2.xmt_time", "target_field":"ntp.xmt_time", "ignore_missing":true}}, + {"date": {"field":"ntp.org_time", "target_field":"ntp.org_time", "formats":["UNIX", "UNIX_MS"], "ignore_failure": true, "if":"ctx?.ntp?.org_time != null"}}, + {"date": {"field":"ntp.xmt_time", "target_field":"ntp.xmt_time", "formats":["UNIX", "UNIX_MS"], "ignore_failure": true, "if":"ctx?.ntp?.xmt_time != null"}}, + {"pipeline":{"name":"zeek.common"}} + ] +} diff --git a/salt/elasticsearch/files/ingest/zeek.traceroute b/salt/elasticsearch/files/ingest/zeek.traceroute new file mode 100644 index 000000000..f34f58cb0 --- /dev/null +++ b/salt/elasticsearch/files/ingest/zeek.traceroute @@ -0,0 +1,10 @@ +{ + "description":"zeek.traceroute", + "processors":[ + {"set": {"field":"event.dataset", "value":"traceroute" }}, + {"json": {"field":"message", "target_field":"message2" }}, + {"rename": {"field":"message2.src", "target_field":"source.ip", "ignore_missing":true,"ignore_failure":true}}, + {"rename": {"field":"message2.dst", "target_field":"destination.ip", "ignore_missing":true,"ignore_failure":true}}, + {"pipeline": {"name":"zeek.common"}} + ] +} diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index adce41bff..47013e48f 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -80,6 +80,7 @@ elasticsearch: managed_integrations: description: List of integrations to add into SOC config UI. Enter the full or partial integration name. Eg. 1password, 1pass forcedType: "[]string" + multiline: True global: True advanced: True helpLink: elasticsearch.html diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-soc@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-soc@package.json new file mode 100644 index 000000000..a2ad15b79 --- /dev/null +++ b/salt/elasticsearch/templates/component/elastic-agent/logs-soc@package.json @@ -0,0 +1,10 @@ +{ + "template": {}, + "_meta": { + "package": { + "name": "log" + }, + "managed_by": "fleet", + "managed": true + } +} \ No newline at end of file diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index d44ca5fa7..3d5ca338f 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -533,19 +533,23 @@ post_to_2.4.120() { # Manually rollover suricata alerts index to ensure data_stream.dataset expected mapping is set to 'suricata' rollover_index "logs-suricata.alerts-so" - echo "Regenerating Elastic Agent Installers" - /sbin/so-elastic-agent-gen-installers - POSTVERSION=2.4.120 } post_to_2.4.130() { + # Optional integrations are loaded AFTER initial successful load of core ES templates (/opt/so/state/estemplates.txt) + # Dynamic templates are created in elasticsearch.enabled for every optional integration based on output of so-elastic-fleet-optional-integrations-load script + echo "Ensuring Elasticsearch templates are up to date after updating package registry" + salt-call state.apply elasticsearch queue=True # Update kibana default space salt-call state.apply kibana.config queue=True echo "Updating Kibana default space" /usr/sbin/so-kibana-space-defaults + echo "Regenerating Elastic Agent Installers" + /sbin/so-elastic-agent-gen-installers + POSTVERSION=2.4.130 } @@ -757,12 +761,12 @@ up_to_2.4.120() { } up_to_2.4.130() { + # Remove any old Elastic Defend config files + rm -f /opt/so/conf/elastic-fleet/integrations/endpoints-initial/elastic-defend-endpoints.json + # Elastic Update for this release, so download Elastic Agent files determine_elastic_agent_upgrade - # Integrations policies need to be updated - rm -f /opt/so/state/eaintegrations.txt - # Ensure override exists to allow nmcli access to other devices touch /etc/NetworkManager/conf.d/10-globally-managed-devices.conf diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index b97ba11e6..962d1096b 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -83,7 +83,7 @@ soc: icon: fa-users-between-lines target: '' links: - - '/#/hunt?q=({:process.entity_id}) | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.name | groupby process.command_line | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path' + - '/#/hunt?q="{:process.entity_id}" | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.name | groupby process.command_line | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path' - name: actionProcessAncestors description: actionProcessAncestorsHelp icon: fa-people-roof @@ -1256,7 +1256,7 @@ soc: - soc_timestamp - event.dataset - host.name - - user.name + - user.effective.name - process.executable - event.action - event.outcome @@ -1918,7 +1918,7 @@ soc: query: 'event.dataset:endpoint.events.registry | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby -sankey user.name process.name | groupby process.name | groupby event.action | groupby registry.path' - name: Elastic Agent Security Events description: Security events from Elastic Agents - query: 'event.dataset:endpoint.events.security | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby -sankey user.name process.executable | groupby process.executable | groupby event.action | groupby event.outcome' + query: 'event.dataset:endpoint.events.security | groupby host.name | groupby -sankey host.name user.effective.name | groupby user.effective.name | groupby -sankey user.effective.name process.executable | groupby process.executable | groupby event.action | groupby event.outcome' - name: Host Overview description: Overview of all host data types query: '((event.category:registry OR event.category:host OR event.category:process OR event.category:driver OR event.category:configuration) OR (event.category:file AND _exists_:process.executable) OR (event.category:network AND _exists_:host.name)) | groupby event.dataset* event.category* event.action* | groupby event.type | groupby -sankey event.type host.name | groupby host.name | groupby user.name | groupby file.name | groupby process.executable'