Merge remote-tracking branch 'origin/2.4/dev' into reyesj2/es-integ-tmp

salt/elasticsearch/files/ingest/zeek.quic

@@ -0,0 +1,18 @@
+{
+  "description" : "zeek.quic",
+  "processors" : [
+    { "set":      { "field": "event.dataset",                           "value": "quic" } },
+    { "set":      { "field": "network.transport",                       "value": "udp"  } },
+    { "json":     { "field": "message",                                 "target_field": "message2",                       "ignore_failure": true    } },
+    { "rename": 	{ "field": "message2.version",	                    "target_field": "quic.version",		              "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_initial_dcid",	        "target_field": "quic.client_initial_dcid",		  "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_scid",	                "target_field": "quic.client_scid",		          "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server_scid",	                "target_field": "quic.server_scid",		          "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server_name",	                "target_field": "quic.server_name",		          "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_protocol",	            "target_field": "quic.client_protocol",		      "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.history",	                    "target_field": "quic.history",                   "ignore_missing": true 	} },
+    { "remove":		{ "field": "message2.tags",		                                                                      "ignore_failure": true    } },
+    { "remove":   { "field": ["host"],                                                                                    "ignore_failure": true    } },
+    { "pipeline":       { "name": "zeek.common" } }
+  ]
+}

salt/manager/tools/sbin/soup

@@ -404,7 +404,8 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.4.80 ]] && up_to_2.4.90
     [[ "$INSTALLEDVERSION" == 2.4.90 ]] && up_to_2.4.100
     [[ "$INSTALLEDVERSION" == 2.4.100 ]] && up_to_2.4.110
-    [[ "$INSTALLEDVERSION" == 2.4.110 ]] && up_to_2.4.120
+    [[ "$INSTALLEDVERSION" == 2.4.110 ]] && up_to_2.4.111
+    [[ "$INSTALLEDVERSION" == 2.4.111 ]] && up_to_2.4.120
     true
 }
 
@@ -519,6 +520,11 @@ post_to_2.4.110() {
   POSTVERSION=2.4.110
 }
 
+post_to_2.4.111() {
+  echo "Nothing to apply"
+  POSTVERSION=2.4.111
+}
+
 post_to_2.4.120() {
   update_elasticsearch_index_settings
   POSTVERSION=2.4.120
@@ -714,6 +720,12 @@ up_to_2.4.110() {
   INSTALLEDVERSION=2.4.110
 }
 
+up_to_2.4.111() {
+  echo "Nothing to do for 2.4.111"
+
+  INSTALLEDVERSION=2.4.111
+}
+
 up_to_2.4.120() {
   add_hydra_pillars
 

salt/soc/defaults.yaml

@@ -339,6 +339,16 @@ soc:
         - file.os
         - file.subsystem
         - log.id.fuid
+      '::quic':
+        - soc_timestamp
+        - event.dataset
+        - source.ip
+        - source.port
+        - destination.ip
+        - destination.port
+        - quic.server_name
+        - log.id.uid
+        - network.community_id
       '::radius':
         - soc_timestamp
         - event.dataset
@@ -1732,6 +1742,10 @@ soc:
               description: PE files list
               query: 'tags:pe | groupby file.machine file.os file.subsystem'
               showSubtitle: true
+            - name: QUIC
+              description: QUIC connections
+              query: 'tags:quic | groupby quic.server_name | groupby source.ip quic.server_name destination.ip'
+              showSubtitle: true
             - name: RADIUS
               description: RADIUS grouped by username
               query: 'tags:radius | groupby user.name'
@@ -1950,6 +1964,9 @@ soc:
             - name: PE
               description: PE (Portable Executable) files transferred via network traffic
               query: 'tags:pe | groupby file.machine | groupby -sankey file.machine file.os | groupby file.os | groupby -sankey file.os file.subsystem | groupby file.subsystem | groupby file.section_names | groupby file.is_exe | groupby file.is_64bit'
+            - name: QUIC
+              description: QUIC network metadata
+              query: 'tags:quic | groupby quic.server_name | groupby -sankey quic.server_name source.ip | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name | groupby quic.server_scid | groupby quic.version | groupby quic.client_protocol'
             - name: RADIUS
               description: RADIUS (Remote Authentication Dial-In User Service) network metadata
               query: 'tags:radius | groupby user.name | groupby -sankey user.name source.ip | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name'

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -45,24 +45,25 @@ transformations:
       rule_conditions:
       - type: logsource
         category: antivirus
-    # Drops the Hashes field which is specific to Sysmon logs
+    # Transforms the `Hashes` field to ECS fields
-    # Ingested sysmon logs will have the Hashes field mapped to ECS specific fields
+    # ECS fields are used by the hash fields emitted by Elastic Defend
-    - id: hashes_drop_sysmon-specific-field
+    # If shipped with Elastic Agent, sysmon logs will also have hashes mapped to ECS fields    
-      type: drop_detection_item
+    - id: hashes_break_out_field
+      type: hashes_fields
+      valid_hash_algos: ["MD5", "SHA1", "SHA256", "SHA512", "IMPHASH"]
+      field_prefix: "file"
+      drop_algo_prefix: False
       field_name_conditions:
         - type: include_fields
           fields:
-          - winlog.event_data.Hashes
+            - winlog.event_data.Hashes
-      rule_conditions:
-      - type: logsource
-        product: windows
     - id: hashes_process-creation
       type: field_name_mapping
       mapping:
-        winlog.event_data.sha256: process.hash.sha256
+        fileSHA256: process.hash.sha256
-        winlog.event_data.sha1: process.hash.sha1
+        fileSHA1: process.hash.sha1
-        winlog.event_data.md5: process.hash.md5
+        fileMD5: process.hash.md5
-        winlog.event_data.Imphash: process.pe.imphash
+        fileIMPHASH: process.pe.imphash
       rule_conditions:
       - type: logsource
         product: windows
@@ -70,10 +71,10 @@ transformations:
     - id: hashes_image-load
       type: field_name_mapping
       mapping:
-        winlog.event_data.sha256: dll.hash.sha256
+        fileSHA256: dll.hash.sha256
-        winlog.event_data.sha1: dll.hash.sha1
+        fileSHA1: dll.hash.sha1
-        winlog.event_data.md5: dll.hash.md5
+        fileMD5: dll.hash.md5
-        winlog.event_data.Imphash: dll.pe.imphash
+        fileIMPHASH: dll.pe.imphash
       rule_conditions:
       - type: logsource
         product: windows
@@ -81,10 +82,10 @@ transformations:
     - id: hashes_driver-load
       type: field_name_mapping
       mapping:
-        winlog.event_data.sha256: dll.hash.sha256
+        fileSHA256: dll.hash.sha256
-        winlog.event_data.sha1: dll.hash.sha1
+        fileSHA1: dll.hash.sha1
-        winlog.event_data.md5: dll.hash.md5
+        fileMD5: dll.hash.md5
-        winlog.event_data.Imphash: dll.pe.imphash
+        fileIMPHASH: dll.pe.imphash
       rule_conditions:
       - type: logsource
         product: windows

setup/so-functions

@@ -962,7 +962,12 @@ docker_seed_update() {
 docker_seed_registry() {
 	local VERSION="$SOVERSION"
 
-	if ! [ -f /nsm/docker-registry/docker/registry.tar ]; then
+	if [ -f /nsm/docker-registry/docker/registry.tar ]; then
+		logCmd "tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker"
+		logCmd "rm /nsm/docker-registry/docker/registry.tar"
+	elif [ -d /nsm/docker-registry/docker/registry ] && [ -f /etc/SOCLOUD ]; then
+		echo "Using existing docker registry content for cloud install"
+	else
 		if [ "$install_type" == 'IMPORT' ]; then
 			container_list 'so-import'	
 		else
@@ -972,9 +977,6 @@ docker_seed_registry() {
 		docker_seed_update_percent=25
 
 		update_docker_containers 'netinstall' '' 'docker_seed_update' '/dev/stdout' 2>&1 | tee -a "$setup_log"
-	else
-		logCmd "tar xvf /nsm/docker-registry/docker/registry.tar -C /nsm/docker-registry/docker"
-		logCmd "rm /nsm/docker-registry/docker/registry.tar"
 	fi
 }