Merge branch '2.4/dev' into jertel/wip

salt/common/packages.sls

@@ -27,6 +27,7 @@ commonpkgs:
       - vim
       - tar
       - unzip
+      - bc
       {% if grains.oscodename != 'focal' %}
       - python3-rich
       {% endif %}
@@ -56,6 +57,7 @@ commonpkgs:
     - skip_suggestions: True
     - pkgs:
       - python3-dnf-plugin-versionlock
+      - bc
       - curl
       - device-mapper-persistent-data
       - fuse

salt/common/tools/sbin_jinja/so-import-pcap

@@ -63,7 +63,7 @@ function status {
 function pcapinfo() {
   PCAP=$1
   ARGS=$2
-  docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap $ARGS
+  docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap -ae $ARGS
 }
 
 function pcapfix() {

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load

@@ -3,6 +3,7 @@
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
 # this file except in compliance with the Elastic License 2.0.
+{% set SUB = salt['pillar.get']('elasticfleet:config:subscription_integrations', default=false) %}
 
 . /usr/sbin/so-common
 . /usr/sbin/so-elastic-fleet-common
@@ -16,7 +17,6 @@ BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
 BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
 PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
 
-SKIP_SUBSCRIPTION=true
 PENDING_UPDATE=false
 
 # Integrations which are included in the package registry, but excluded from automatic installation via this script.
@@ -63,7 +63,8 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
             bulk_package=$(echo "$package" | jq '{name: .name, version: .latest_version}' )
 
             if [[ ! "${EXCLUDED_INTEGRATIONS[@]}" =~ "$package_name" ]]; then
-                if $SKIP_SUBSCRIPTION && [[ "$subscription" != "basic" && "$subscription" != "null" && -n "$subscription" ]]; then
+            {% if not SUB %}
+                if [[ "$subscription" != "basic" && "$subscription" != "null" && -n "$subscription" ]]; then
                     # pass over integrations that require non-basic elastic license
                     echo "$package_name integration requires an Elastic license of $subscription or greater... skipping"
                     continue
@@ -83,6 +84,20 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
                         fi
                     fi
                 fi
+            {% else %}
+                if [[ "$installed_version" == "null" || -z "$installed_version" ]]; then
+                    echo "$package_name is not installed... Adding to next update."
+                    jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
+                    PENDING_UPDATE=true
+                else
+                    results=$(compare_versions "$latest_version" "$installed_version")
+                    if [ $results == "greater" ]; then
+                        echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
+                        jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
+                        PENDING_UPDATE=true
+                    fi
+                fi
+            {% endif %}
             else
                 echo "Skipping $package_name..."
             fi

salt/elasticsearch/files/ingest/zeek.ntp

@@ -0,0 +1,16 @@
+{
+  "description" : "zeek.ntp",
+  "processors":[
+    {"set":     {"field":"event.dataset",       "value":"ntp",                       "ignore_failure":true}},
+    {"json":    {"field":"message",             "target_field":"message2",           "ignore_failure":true}},
+    {"rename":  {"field":"message2.version",    "target_field":"ntp.version",        "ignore_missing":true}},
+    {"rename":  {"field":"message2.mode",       "target_field":"ntp.mode",           "ignore_missing":true}},
+    {"rename":  {"field":"message2.poll",       "target_field":"ntp.poll",           "ignore_missing":true}},
+    {"rename":  {"field":"message2.precision",  "target_field":"ntp.precision",      "ignore_missing":true}},
+    {"rename":  {"field":"message2.org_time",   "target_field":"ntp.org_time",       "ignore_missing":true}},
+    {"rename":  {"field":"message2.xmt_time",   "target_field":"ntp.xmt_time",       "ignore_missing":true}},
+    {"date":    {"field":"ntp.org_time",        "target_field":"ntp.org_time",  "formats":["UNIX", "UNIX_MS"], "ignore_failure": true, "if":"ctx?.ntp?.org_time != null"}},
+    {"date":    {"field":"ntp.xmt_time",        "target_field":"ntp.xmt_time",  "formats":["UNIX", "UNIX_MS"], "ignore_failure": true, "if":"ctx?.ntp?.xmt_time != null"}},
+    {"pipeline":{"name":"zeek.common"}}
+  ]
+}

salt/elasticsearch/files/ingest/zeek.traceroute

@@ -0,0 +1,10 @@
+{
+  "description":"zeek.traceroute",
+  "processors":[
+    {"set":         {"field":"event.dataset",       "value":"traceroute" }},
+    {"json":        {"field":"message",             "target_field":"message2" }},
+    {"rename":      {"field":"message2.src",        "target_field":"source.ip",         "ignore_missing":true,"ignore_failure":true}},
+    {"rename":      {"field":"message2.dst",        "target_field":"destination.ip",    "ignore_missing":true,"ignore_failure":true}},
+    {"pipeline":    {"name":"zeek.common"}}
+  ]
+}

salt/elasticsearch/soc_elasticsearch.yaml

@@ -80,6 +80,7 @@ elasticsearch:
   managed_integrations:
     description: List of integrations to add into SOC config UI. Enter the full or partial integration name. Eg. 1password, 1pass
     forcedType: "[]string"
+    multiline: True
     global: True
     advanced: True
     helpLink: elasticsearch.html

salt/elasticsearch/templates/component/elastic-agent/logs-soc@package.json

@@ -0,0 +1,10 @@
+{
+  "template": {},
+  "_meta": {
+    "package": {
+      "name": "log"
+    },
+    "managed_by": "fleet",
+    "managed": true
+  }
+}

salt/manager/tools/sbin/soup

@@ -533,19 +533,23 @@ post_to_2.4.120() {
   # Manually rollover suricata alerts index to ensure data_stream.dataset expected mapping is set to 'suricata'
   rollover_index "logs-suricata.alerts-so"
 
-  echo "Regenerating Elastic Agent Installers"
-  /sbin/so-elastic-agent-gen-installers
-
   POSTVERSION=2.4.120
 }
 
 post_to_2.4.130() {
+  # Optional integrations are loaded AFTER initial successful load of core ES templates (/opt/so/state/estemplates.txt)
+  #   Dynamic templates are created in elasticsearch.enabled for every optional integration based on output of so-elastic-fleet-optional-integrations-load script
+  echo "Ensuring Elasticsearch templates are up to date after updating package registry"
+  salt-call state.apply elasticsearch queue=True
 
   # Update kibana default space
   salt-call state.apply kibana.config queue=True
   echo "Updating Kibana default space"
   /usr/sbin/so-kibana-space-defaults
 
+  echo "Regenerating Elastic Agent Installers"
+  /sbin/so-elastic-agent-gen-installers
+
   POSTVERSION=2.4.130
 }
 
@@ -757,12 +761,12 @@ up_to_2.4.120() {
 }
 
 up_to_2.4.130() {
+  # Remove any old Elastic Defend config files
+  rm -f /opt/so/conf/elastic-fleet/integrations/endpoints-initial/elastic-defend-endpoints.json
+
   # Elastic Update for this release, so download Elastic Agent files
   determine_elastic_agent_upgrade
 
-  # Integrations policies need to be updated
-  rm -f /opt/so/state/eaintegrations.txt
-
   # Ensure override exists to allow nmcli access to other devices
   touch /etc/NetworkManager/conf.d/10-globally-managed-devices.conf
 

salt/soc/defaults.yaml

@@ -83,7 +83,7 @@ soc:
         icon: fa-users-between-lines
         target: ''
         links:
-          - '/#/hunt?q=({:process.entity_id}) | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.name | groupby process.command_line | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path'
+          - '/#/hunt?q="{:process.entity_id}" | groupby event.dataset | groupby -sankey event.dataset event.action | groupby event.action | groupby process.name | groupby process.command_line | groupby host.name user.name | groupby source.ip source.port destination.ip destination.port | groupby dns.question.name | groupby dns.answers.data | groupby file.path | groupby registry.path | groupby dll.path'
       - name: actionProcessAncestors
         description: actionProcessAncestorsHelp
         icon: fa-people-roof
@@ -1256,7 +1256,7 @@ soc:
         -  soc_timestamp
         -  event.dataset
         -  host.name
-        -  user.name
+        -  user.effective.name
         -  process.executable
         -  event.action
         -  event.outcome
@@ -1918,7 +1918,7 @@ soc:
               query: 'event.dataset:endpoint.events.registry | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby -sankey user.name process.name | groupby process.name | groupby event.action | groupby registry.path'
             - name: Elastic Agent Security Events
               description: Security events from Elastic Agents
-              query: 'event.dataset:endpoint.events.security | groupby host.name | groupby -sankey host.name user.name | groupby user.name | groupby -sankey user.name process.executable | groupby process.executable | groupby event.action | groupby event.outcome'
+              query: 'event.dataset:endpoint.events.security | groupby host.name | groupby -sankey host.name user.effective.name | groupby user.effective.name | groupby -sankey user.effective.name process.executable | groupby process.executable | groupby event.action | groupby event.outcome'
             - name: Host Overview
               description: Overview of all host data types
               query: '((event.category:registry OR event.category:host OR event.category:process OR event.category:driver OR event.category:configuration) OR (event.category:file  AND _exists_:process.executable) OR (event.category:network AND _exists_:host.name)) | groupby event.dataset* event.category* event.action* | groupby event.type | groupby -sankey event.type host.name | groupby host.name | groupby user.name | groupby file.name | groupby process.executable'