soup: require green Elasticsearch cluster before upgrading

Change the pre-flight cluster-health gate to wait_for_status=green instead of
yellow, so soup only proceeds when the cluster is fully green.

salt/elasticfleet/defaults.yaml

@@ -1,6 +1,5 @@
 elasticfleet:
   enabled: False
-  patch_version: 9.3.3+build202604082258 # Elastic Agent specific patch release.
   enable_manager_output: True
   config:
     server:

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -5,7 +5,7 @@
   "package": {
     "name": "endpoint",
     "title": "Elastic Defend",
-    "version": "9.3.0",
+    "version": "9.3.1",
     "requires_root": true
   },
   "enabled": true,

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -29,7 +29,7 @@
               "\\.gz$"
             ],
             "include_files": [],
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.15.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.15.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.15.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.20.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.8.3\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.20.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.20.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.8.3\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ],

salt/elasticsearch/defaults.yaml

@@ -1,6 +1,6 @@
 elasticsearch:
   enabled: false
-  version: 9.3.3
+  version: 9.3.7
   index_clean: true
   data_retention_method: DLM
   vm:

salt/elasticsearch/files/ingest/logs-pfsense.log-1.25.4

@@ -118,70 +118,70 @@
     {
       "pipeline": {
         "tag": "pipeline_e16851a7",
-        "name": "logs-pfsense.log-1.25.2-firewall",
+        "name": "logs-pfsense.log-1.25.4-firewall",
         "if": "ctx.event.provider == 'filterlog'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_828590b5",
-        "name": "logs-pfsense.log-1.25.2-openvpn",
+        "name": "logs-pfsense.log-1.25.4-openvpn",
         "if": "ctx.event.provider == 'openvpn'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_9d37039c",
-        "name": "logs-pfsense.log-1.25.2-ipsec",
+        "name": "logs-pfsense.log-1.25.4-ipsec",
         "if": "ctx.event.provider == 'charon'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_ad56bbca",
-        "name": "logs-pfsense.log-1.25.2-dhcp",
+        "name": "logs-pfsense.log-1.25.4-dhcp",
         "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\", \"dnsmasq-dhcp\"].contains(ctx.event.provider)"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_dd85553d",
-        "name": "logs-pfsense.log-1.25.2-unbound",
+        "name": "logs-pfsense.log-1.25.4-unbound",
         "if": "ctx.event.provider == 'unbound'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_720ed255",
-        "name": "logs-pfsense.log-1.25.2-haproxy",
+        "name": "logs-pfsense.log-1.25.4-haproxy",
         "if": "ctx.event.provider == 'haproxy'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_456beba5",
-        "name": "logs-pfsense.log-1.25.2-php-fpm",
+        "name": "logs-pfsense.log-1.25.4-php-fpm",
         "if": "ctx.event.provider == 'php-fpm'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_a0d89375",
-        "name": "logs-pfsense.log-1.25.2-squid",
+        "name": "logs-pfsense.log-1.25.4-squid",
         "if": "ctx.event.provider == 'squid'"
       }
     },
     {
       "pipeline": {
         "tag": "pipeline_c2f1ed55",
-        "name": "logs-pfsense.log-1.25.2-snort",
+        "name": "logs-pfsense.log-1.25.4-snort",
         "if": "ctx.event.provider == 'snort'"
       }
     },
     {
       "pipeline": {
         "tag":"pipeline_33db1c9e",
-        "name": "logs-pfsense.log-1.25.2-suricata",
+        "name": "logs-pfsense.log-1.25.4-suricata",
         "if": "ctx.event.provider == 'suricata'"
       }
     },

salt/kibana/defaults.yaml

@@ -22,7 +22,7 @@ kibana:
           - default
           - file
     migrations:
-      discardCorruptObjects: "9.3.3"
+      discardCorruptObjects: "9.3.7"
     telemetry:
       enabled: False
     xpack:

salt/manager/tools/sbin/soup

@@ -12,7 +12,17 @@
 UPDATE_DIR=/tmp/sogh/securityonion
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
 INSTALLEDVERSION=$(cat /etc/soversion)
-POSTVERSION=$INSTALLEDVERSION
+# /etc/sopostversion is a soup-owned marker (no salt state manages it) tracking how
+# far the post-upgrade walk has progressed. Its presence means a prior upgrade did
+# not finish its post-upgrade steps; its contents are the resume point. It is read
+# here before preupgrade_changes mutates INSTALLEDVERSION and before any highstate
+# stamps /etc/soversion from the pillar.
+POSTVERSION_FILE=/etc/sopostversion
+if [ -f "$POSTVERSION_FILE" ]; then
+  POSTVERSION=$(cat "$POSTVERSION_FILE")
+else
+  POSTVERSION=$INSTALLEDVERSION
+fi
 INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
 BATCHSIZE=5
 SOUP_LOG=/root/soup.log
@@ -23,6 +33,10 @@ NOTIFYCUSTOMELASTICCONFIG=false
 TOPFILE=/opt/so/saltstack/default/salt/top.sls
 BACKUPTOPFILE=/opt/so/saltstack/default/salt/top.sls.backup
 SALTUPGRADED=false
+# Set true once soup begins modifying the system (past the pre-flight checks), so the
+# EXIT trap can tell the user the update did not finish and must be re-run. Only the
+# pre-flight gates (ES compatibility, disk, network) fail before this is set.
+SOUP_UPGRADE_STARTED=false
 SALT_CLOUD_INSTALLED=false
 SALT_CLOUD_CONFIGURED=false
 # Check if salt-cloud is installed
@@ -123,6 +137,28 @@ check_err() {
 
     echo "SOUP XTRACE debug log (if enabled) at $SOUP_DEBUG_LOG. Re-run soup with SOUP_DEBUG=1 to create $SOUP_DEBUG_LOG"
 
+    # If soup had already started modifying the system, make it unmistakable that the
+    # update is incomplete and must be re-run. soup is resumable: a version upgrade
+    # picks up from the /etc/sopostversion marker, and a hotfix re-applies because
+    # /etc/sohotfix is only advanced after a successful highstate.
+    if [[ "$SOUP_UPGRADE_STARTED" == "true" ]]; then
+      echo ""
+      echo "=============================================================================="
+      echo " UPGRADE INCOMPLETE"
+      echo "=============================================================================="
+      echo " This soup run did NOT finish. Your Security Onion installation may be in a"
+      echo " partially-updated state and is not yet fully upgraded."
+      echo ""
+      echo " Review the error above and $SOUP_LOG, resolve the underlying problem, then"
+      echo " run soup again to resume and complete the update:"
+      echo ""
+      echo "     sudo soup"
+      echo ""
+      echo " soup is resumable -- re-running it continues from where this run stopped."
+      echo "=============================================================================="
+      echo ""
+    fi
+
     exit $exit_code
   fi
 
@@ -291,6 +327,30 @@ check_pillar_items() {
   fi
 }
 
+check_cluster_health() {
+  echo "Checking Elasticsearch cluster health."
+  # Require a 'green' cluster before upgrading; anything less (yellow, red, or
+  # unreachable) blocks. Modeled on the wait used in so-elasticsearch-roles-load.
+  if so-elasticsearch-query "_cluster/health?wait_for_status=green&timeout=120s" --fail > /dev/null 2>&1; then
+    printf "\nThe Elasticsearch cluster is healthy (green). We can proceed with SOUP.\n\n"
+  else
+    printf "\nThe Elasticsearch cluster is not green. Please resolve the cluster health issue so the cluster is green before running SOUP again.\n\n"
+    exit 0
+  fi
+}
+
+check_fleet_server() {
+  echo "Checking that Elastic Fleet Server is responding."
+  # Modeled on the wait_for_so-elastic-fleet state check in elasticfleet/enabled.sls,
+  # which waits for HTTP 200 from the Fleet Server status API.
+  if curl -sk --fail --retry 3 --retry-delay 10 --max-time 30 "https://localhost:8220/api/status" > /dev/null 2>&1; then
+    printf "\nElastic Fleet Server is responding. We can proceed with SOUP.\n\n"
+  else
+    printf "\nElastic Fleet Server is not responding at https://localhost:8220/api/status. Please ensure Elastic Fleet is healthy before running SOUP again.\n\n"
+    exit 0
+  fi
+}
+
 check_saltmaster_status() {
   set +e
   echo "Waiting on the Salt Master service to be ready."
@@ -414,6 +474,13 @@ preupgrade_changes() {
     true
 }
 
+set_postversion() {
+    # Persist post-upgrade walk progress so an interrupted upgrade can resume the
+    # remaining steps on the next soup run (see /etc/sopostversion handling).
+    POSTVERSION="$1"
+    echo "$POSTVERSION" > "$POSTVERSION_FILE"
+}
+
 postupgrade_changes() {
     # This function is to add any new pillar items if needed.
     echo "Running post upgrade processes."
@@ -421,6 +488,8 @@ postupgrade_changes() {
     [[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0
     [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0
     [[ "$POSTVERSION" == "3.1.0" ]] && post_to_3.2.0
+    # All applicable post-upgrade steps completed; clear the resume marker.
+    rm -f "$POSTVERSION_FILE"
     true
 }
 
@@ -513,7 +582,7 @@ post_to_3.0.0() {
   # convert yes/no in suricata pillars to true/false
   convert_suricata_yes_no
 
-  POSTVERSION=3.0.0
+  set_postversion 3.0.0
 }
 
 ### 3.0.0 End ###
@@ -740,7 +809,6 @@ fix_logstash_0013_lumberjack_pipeline_name() {
 up_to_3.1.0() {
   ensure_postgres_local_pillar
   ensure_postgres_secret
-  determine_elastic_agent_upgrade
   elasticsearch_backup_index_templates
   # Clear existing component template state file.
   rm -f /opt/so/state/esfleet_component_templates.json
@@ -777,7 +845,7 @@ post_to_3.1.0() {
   # Check for unhealthy / unauthorized integration transform jobs and attempt reauthorizations
   check_transform_health_and_reauthorize || true
 
-  POSTVERSION=3.1.0
+  set_postversion 3.1.0
 }
 
 ### 3.1.0 End ###
@@ -888,6 +956,9 @@ update_kafka_metadata() {
 }
 
 up_to_3.2.0() {
+  # download 9.3.7 elastic agent packages
+  determine_elastic_agent_upgrade
+
   fix_logstash_0013_lumberjack_pipeline_name
 
   pin_elasticsearch_data_retention_method
@@ -901,7 +972,7 @@ post_to_3.2.0() {
 
   bootstrap_so_soc_database
 
-  # Including agent regen script here since it was missed in post_to_3.1.0
+  # Generate 9.3.7 elastic agent installers
   echo "Regenerating Elastic Agent Installers"
   /sbin/so-elastic-agent-gen-installers
 
@@ -909,7 +980,7 @@ post_to_3.2.0() {
 
   update_kafka_metadata "4.3"
 
-  POSTVERSION=3.2.0
+  set_postversion 3.2.0
 }
 
 ### 3.2.0 End ###
@@ -1061,8 +1132,20 @@ upgrade_check() {
   fi
   [[ -f /etc/sohotfix ]] && CURRENTHOTFIX=$(cat /etc/sohotfix)
   if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then
+    # A leftover post-version marker means a previous upgrade to this version
+    # advanced /etc/soversion (the highstate stamps it from the pillar) but did not
+    # finish its post-upgrade steps. Resume the upgrade instead of reporting "latest".
+    if [ -f "$POSTVERSION_FILE" ] && [ "$(cat "$POSTVERSION_FILE")" != "$NEWVERSION" ]; then
+      echo "A previous upgrade to $NEWVERSION did not complete its post-upgrade steps; resuming."
+      is_hotfix=false
+      return 0
+    fi
     echo "Checking to see if there are hotfixes needed"
     if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then
+      # Reaching here means we are at the target version and NOT resuming (the resume
+      # check above returned otherwise). Clear any stale resume marker so a completed
+      # upgrade is never mistaken for a partial one and re-run on a later invocation.
+      rm -f "$POSTVERSION_FILE"
       echo "You are already running the latest version of Security Onion."
       exit 0
     else
@@ -1174,7 +1257,8 @@ verify_es_version_compatibility() {
         ["8.18.4"]="8.18.6 8.18.8 9.0.8"
 	    ["8.18.6"]="8.18.8 9.0.8"
 	    ["8.18.8"]="9.0.8"
-        ["9.0.8"]="9.3.3"
+        ["9.0.8"]="9.3.3 9.3.7"
+        ["9.3.3"]="9.3.7"
     )
 
     # Elasticsearch MUST upgrade through these versions
@@ -1757,6 +1841,15 @@ main() {
   set_minionid
   MINION_ROLE=$(lookup_role)
   echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
+  # /etc/soversion is stamped to the target version before the upgrade fully
+  # completes, so a lingering resume marker means this grid is only partially
+  # upgraded even though the line above shows the target version. Make that explicit
+  # so it is not mistaken for a finished upgrade.
+  if [ -f "$POSTVERSION_FILE" ] && [ "$(cat "$POSTVERSION_FILE")" != "$INSTALLEDVERSION" ]; then
+    echo ""
+    echo "NOTE: A previous upgrade to $INSTALLEDVERSION did not finish. This grid is"
+    echo "      partially upgraded and this soup run will resume and complete it."
+  fi
   echo ""
   check_minimum_version
 
@@ -1785,6 +1878,12 @@ main() {
   echo "Verifying Elasticsearch version compatibility across the grid before upgrading."
   verify_es_version_compatibility
 
+  # Pre-flight health checks: confirm the grid is in a good state before we change
+  # anything. These run before any modifications, so a failure exits cleanly and the
+  # operator can fix the issue and re-run soup.
+  check_cluster_health
+  check_fleet_server
+
   echo "Checking for Salt Master and Minion updates."
   upgrade_check_salt
   set -e
@@ -1801,6 +1900,7 @@ main() {
   fi
 
   if [ "$is_hotfix" == "true" ]; then
+    SOUP_UPGRADE_STARTED=true
     echo "Applying $HOTFIXVERSION hotfix"
     # since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars
     if [[ ! "$MINION_ROLE" == "import" ]]; then
@@ -1811,10 +1911,16 @@ main() {
     create_local_directories "/opt/so/saltstack/default"
     apply_hotfix
     echo "Hotfix applied"
-    update_version
     enable_highstate
     highstate
+    # Record the hotfix only after the highstate succeeds. /etc/sohotfix is written
+    # solely by soup (no salt state manages it), so deferring the write means a failed
+    # hotfix highstate leaves the old hotfix value and re-running soup re-applies it,
+    # rather than reporting "already latest". The soversion/pillar writes in
+    # update_version are no-ops here since the version is unchanged for a hotfix.
+    update_version
   else
+    SOUP_UPGRADE_STARTED=true
     echo ""
     echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
     echo ""
@@ -1870,6 +1976,10 @@ main() {
     copy_new_files
     echo ""
     create_local_directories "/opt/so/saltstack/default"
+    # Seed the resume marker before the highstate stamps /etc/soversion to the new
+    # version, so an interrupted upgrade is detectable as "not finished" on re-run.
+    # POSTVERSION still holds the pre-upgrade (or prior resume) version here.
+    [ -f "$POSTVERSION_FILE" ] || echo "$POSTVERSION" > "$POSTVERSION_FILE"
     update_version
 
     echo ""

salt/nginx/etc/nginx.conf

@@ -399,7 +399,7 @@ http {
 		error_page 429 = @error429;
 
 		location @error401 {
-			if ($request_uri ~* (^/api/.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) {
+			if ($request_uri ~* (^.*/api/.*|^/connect/.*|^/oauth2/.*|^/.*\.map$)) {
 				return    401;
 			}
 

salt/soc/files/soc/playbook_placeholder_map.yaml

@@ -14,22 +14,18 @@
 
 CommandLine: process.command_line
 CurrentDirectory: process.working_directory
-Details: registry.data.strings
 Image: process.executable
-ImageLoaded: dll.path
+ImageLoaded: dll.name
 ParentImage: process.parent.executable
 ParentName: process.parent.name
 ParentProcessGuid: process.parent.entity_id
 ProcessGuid: process.entity_id
-TargetFilename: file.path
+TargetFilename: file.name
 TargetObject: registry.path
 TargetUserName: user.target.name
 User: user.name
 community_id: network.community_id
 dns_resolved_ip: dns.resolved_ip
-QueryName: dns.question.name
-dll_basename: dll.name
-file_basename: file.name
 document_id: soc_id
 dst_ip: destination.ip
 dst_port: destination.port
@@ -44,15 +40,10 @@ public_ip: network.public_ip
 related_hosts: related.hosts
 related_ip: related.ip
 src_ip: source.ip
-dns_query_name: dns.question.name
+dns_query_name: dns.query_name
 flow_id: log.id.uid
 payload: network.data.decoded
 rule_category: rule.category
 rule_name: rule.name
 rule_uuid: rule.uuid
 src_port: source.port
-# PowerShell channel (ps_script EID 4104 / classic EID 400)
-ScriptBlockText: powershell.file.script_block_text
-script_block_id: powershell.file.script_block_id
-script_block_hash: powershell.file.script_block_hash
-EngineVersion: powershell.engine.version

salt/soc/files/soc/sigma_playbook_pipeline.yaml

@@ -1,9 +1,8 @@
 name: Security Onion - Playbook Pipeline
 priority: 97
 transformations:
-    # Route to lowercase-normalized .caseless subfields for case-insensitive matching.
-    # file.path.caseless exists on Defend only (Sysmon file events lack it);
-    # registry.path / dll.path / file.name have no .caseless on any source.
+    # Route string fields to their lowercase-normalized .caseless subfield so wildcard
+    # matches are case-insensitive.
     - id: case_insensitive_string_fields
       type: field_name_mapping
       mapping:
@@ -11,121 +10,3 @@ transformations:
         process.parent.executable: process.parent.executable.caseless
         process.command_line: process.command_line.caseless
         process.parent.command_line: process.parent.command_line.caseless
-        file.path: file.path.caseless
-    # file_activity: playbook-only pseudo-category spanning all file operations.
-    - id: playbook_file_activity_add-fields
-      type: add_condition
-      conditions:
-        event.category: 'file'
-      rule_conditions:
-      - type: logsource
-        category: file_activity
-    # EventType -> event.action is safe as a projection only — the values differ
-    # (Sysmon DeleteFile/SetValue vs ECS deletion/modification).
-    - id: playbook_file_activity_event_type
-      type: field_name_mapping
-      mapping:
-        winlog.event_data.EventType: event.action
-      rule_conditions:
-      - type: logsource
-        category: file_activity
-    - id: playbook_file_event_event_type
-      type: field_name_mapping
-      mapping:
-        winlog.event_data.EventType: event.action
-      rule_conditions:
-      - type: logsource
-        category: file_event
-    - id: playbook_file_delete_event_type
-      type: field_name_mapping
-      mapping:
-        winlog.event_data.EventType: event.action
-      rule_conditions:
-      - type: logsource
-        category: file_delete
-    - id: playbook_file_rename_event_type
-      type: field_name_mapping
-      mapping:
-        winlog.event_data.EventType: event.action
-      rule_conditions:
-      - type: logsource
-        category: file_rename
-    - id: playbook_registry_set_event_type
-      type: field_name_mapping
-      mapping:
-        winlog.event_data.EventType: event.action
-      rule_conditions:
-      - type: logsource
-        category: registry_set
-    # WMI-Activity events carry no event.category; event.dataset is source-specific
-    - id: playbook_wmi_activity_add-fields
-      type: add_condition
-      conditions:
-        event.dataset: 'wmi.operational'
-      rule_conditions:
-      - type: logsource
-        category: wmi
-    - id: playbook_wmi_event_activity_add-fields
-      type: add_condition
-      conditions:
-        event.dataset: 'wmi.operational'
-      rule_conditions:
-      - type: logsource
-        category: wmi_event
-    - id: playbook_ps_script_field-mapping
-      type: field_name_mapping
-      mapping:
-        ScriptBlockText: powershell.file.script_block_text
-        Path: file.path
-      rule_conditions:
-      - type: logsource
-        category: ps_script
-    - id: playbook_ps_script_add-fields
-      type: add_condition
-      conditions:
-        event.dataset: 'windows.powershell_operational'
-        event.code: '4104'
-      rule_conditions:
-      - type: logsource
-        category: ps_script
-    # Data (raw EID 400/600 message blob) -> process.command_line: the winlog
-    # integration parses HostApplication into it; `message` is analyzed text and
-    # wildcard matches on it silently return zero. No .caseless on this dataset.
-    - id: playbook_ps_classic_start_field-mapping
-      type: field_name_mapping
-      mapping:
-        Data: process.command_line
-      rule_conditions:
-      - type: logsource
-        category: ps_classic_start
-    - id: playbook_ps_classic_start_add-fields
-      type: add_condition
-      conditions:
-        event.dataset: 'windows.powershell'
-        event.code: '400'
-      rule_conditions:
-      - type: logsource
-        category: ps_classic_start
-    - id: playbook_ps_classic_provider_start_field-mapping
-      type: field_name_mapping
-      mapping:
-        Data: process.command_line
-      rule_conditions:
-      - type: logsource
-        category: ps_classic_provider_start
-    - id: playbook_ps_classic_provider_start_add-fields
-      type: add_condition
-      conditions:
-        event.dataset: 'windows.powershell'
-        event.code: '600'
-      rule_conditions:
-      - type: logsource
-        category: ps_classic_provider_start
-    # Alert docs nest the host under event_data.host.name; no top-level host.name.
-    - id: playbook_alert_host_field
-      type: field_name_mapping
-      mapping:
-        host.name: event_data.host.name
-      rule_conditions:
-      - type: logsource
-        category: alert

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -131,22 +131,13 @@ transformations:
       type: field_name_mapping
       mapping:
         winlog.event_data.Details: registry.data.strings
+        # field rename only; EventType values (SetValue/CreateKey) still differ from
+        # event.action values (modification/creation)
+        winlog.event_data.EventType: event.action
       rule_conditions:
       - type: logsource
         product: windows
         category: registry_set
-    # ecs_windows renames Initiated -> network.direction without translating the value,
-    # so rules compile to network.direction:"true" and never match (field holds
-    # ingress/egress). network.initiated carries the boolean on both Defend and Sysmon.
-    # Keyed on network.direction because ecs_windows has already renamed by this layer.
-    - id: ecs_fix_network_connection_initiated
-      type: field_name_mapping
-      mapping:
-        network.direction: network.initiated
-      rule_conditions:
-      - type: logsource
-        product: windows
-        category: network_connection
     - id: ecs_fix_image_load
       type: field_name_mapping
       mapping:
@@ -159,30 +150,6 @@ transformations:
       - type: logsource
         product: windows
         category: image_load
-    # Defend reports the driver image in dll.path (file.path is null on driver events) —
-    # same renames as image_load.
-    - id: ecs_fix_driver_load
-      type: field_name_mapping
-      mapping:
-        file.path: dll.path
-        file.code_signature.signed: dll.code_signature.exists
-        winlog.event_data.Signature: dll.code_signature.subject_name
-        file.code_signature.status: dll.code_signature.status
-        winlog.event_data.Hashes: dll.hash.sha256
-      rule_conditions:
-      - type: logsource
-        product: windows
-        category: driver_load
-    - id: ecs_fix_file_rename
-      type: field_name_mapping
-      mapping:
-        # Defend records the pre-rename path in file.Ext.original.path; Sysmon's
-        # SourceFilename has no ECS equivalent.
-        winlog.event_data.SourceFilename: file.Ext.original.path
-      rule_conditions:
-      - type: logsource
-        product: windows
-        category: file_rename
     - id: linux_security_add-fields
       type: add_condition
       conditions:
@@ -356,33 +323,6 @@ transformations:
       rule_conditions:
       - type: logsource
         category: file_event
-    # Without event.type scoping, file_delete rules also match creations.
-    - id: endpoint_file_delete_add-fields
-      type: add_condition
-      conditions:
-        event.category: 'file'
-        event.type: 'deletion'
-      rule_conditions:
-      - type: logsource
-        category: file_delete
-    # Defend reports renames as event.action:rename with event.type:change.
-    - id: endpoint_file_rename_add-fields
-      type: add_condition
-      conditions:
-        event.category: 'file'
-        event.action: 'rename'
-      rule_conditions:
-      - type: logsource
-        category: file_rename
-    # event.type:change selects writes and excludes Defend's read-only registry events.
-    - id: endpoint_registry_set_add-fields
-      type: add_condition
-      conditions:
-        event.category: 'registry'
-        event.type: 'change'
-      rule_conditions:
-      - type: logsource
-        category: registry_set
     # Scope image_load rules to Elastic Endpoint library events (event.category:library, dll.*
     # populated).
     - id: endpoint_image_load_add-fields
@@ -411,149 +351,6 @@ transformations:
       rule_conditions:
       - type: logsource
         category: network_connection
-    # Scope on lookup actions, not network.protocol:dns (also matches Zeek port-53).
-    - id: endpoint_dns_query_add-fields
-      type: add_condition
-      conditions:
-        event.category: 'network'
-        event.action:
-          - 'lookup_requested'
-          - 'lookup_result'
-      rule_conditions:
-      - type: logsource
-        category: dns_query
-    # OS gates: without them a `product: windows` rule in these categories also matches
-    # Linux/macOS. Separate conditions (not `product:` on the mappings above) so the
-    # mappings keep scoping product-less rules.
-    - id: endpoint_file_create_windows_os-gate
-      type: add_condition
-      conditions:
-        host.os.type: 'windows'
-      rule_conditions:
-      - type: logsource
-        category: file_event
-        product: windows
-    - id: endpoint_file_create_linux_os-gate
-      type: add_condition
-      conditions:
-        host.os.type: 'linux'
-      rule_conditions:
-      - type: logsource
-        category: file_event
-        product: linux
-    - id: endpoint_file_create_macos_os-gate
-      type: add_condition
-      conditions:
-        host.os.type: 'macos'
-      rule_conditions:
-      - type: logsource
-        category: file_event
-        product: macos
-    - id: endpoint_file_delete_windows_os-gate
-      type: add_condition
-      conditions:
-        host.os.type: 'windows'
-      rule_conditions:
-      - type: logsource
-        category: file_delete
-        product: windows
-    - id: endpoint_file_delete_linux_os-gate
-      type: add_condition
-      conditions:
-        host.os.type: 'linux'
-      rule_conditions:
-      - type: logsource
-        category: file_delete
-        product: linux
-    - id: endpoint_file_delete_macos_os-gate
-      type: add_condition
-      conditions:
-        host.os.type: 'macos'
-      rule_conditions:
-      - type: logsource
-        category: file_delete
-        product: macos
-    - id: endpoint_file_rename_windows_os-gate
-      type: add_condition
-      conditions:
-        host.os.type: 'windows'
-      rule_conditions:
-      - type: logsource
-        category: file_rename
-        product: windows
-    - id: endpoint_file_rename_linux_os-gate
-      type: add_condition
-      conditions:
-        host.os.type: 'linux'
-      rule_conditions:
-      - type: logsource
-        category: file_rename
-        product: linux
-    - id: endpoint_file_rename_macos_os-gate
-      type: add_condition
-      conditions:
-        host.os.type: 'macos'
-      rule_conditions:
-      - type: logsource
-        category: file_rename
-        product: macos
-    - id: endpoint_network_connection_windows_os-gate
-      type: add_condition
-      conditions:
-        host.os.type: 'windows'
-      rule_conditions:
-      - type: logsource
-        category: network_connection
-        product: windows
-    - id: endpoint_network_connection_linux_os-gate
-      type: add_condition
-      conditions:
-        host.os.type: 'linux'
-      rule_conditions:
-      - type: logsource
-        category: network_connection
-        product: linux
-    - id: endpoint_network_connection_macos_os-gate
-      type: add_condition
-      conditions:
-        host.os.type: 'macos'
-      rule_conditions:
-      - type: logsource
-        category: network_connection
-        product: macos
-    - id: endpoint_dns_query_windows_os-gate
-      type: add_condition
-      conditions:
-        host.os.type: 'windows'
-      rule_conditions:
-      - type: logsource
-        category: dns_query
-        product: windows
-    - id: endpoint_dns_query_linux_os-gate
-      type: add_condition
-      conditions:
-        host.os.type: 'linux'
-      rule_conditions:
-      - type: logsource
-        category: dns_query
-        product: linux
-    - id: endpoint_dns_query_macos_os-gate
-      type: add_condition
-      conditions:
-        host.os.type: 'macos'
-      rule_conditions:
-      - type: logsource
-        category: dns_query
-        product: macos
-    # registry_set / image_load / driver_load are not gated — windows-only telemetry.
-    # wmi / wmi_event are not scoped here: their only handle is event.dataset
-    - id: endpoint_driver_load_add-fields
-      type: add_condition
-      conditions:
-        event.category: 'driver'
-      rule_conditions:
-      - type: logsource
-        category: driver_load
     # Maps "alert" category to SO Alert events
     - id: alert_so_add-fields
       type: add_condition