Merge pull request #16087 from Security-Onion-Solutions/sigma-pipeline

Broader support for SigmaHQ rules + Elastic Defend logs

salt/common/tools/sbin/so-log-check

@@ -133,6 +133,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Cancelling deferred write event maybeFenceReplicas because the event queue is now closed" # Kafka controller log during shutdown/restart
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Redis may have been restarted" # Redis likely restarted by salt
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|file already closed"          # Go logging race condition during container restart
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|relation \"audit_settings\" does not exist" # salt checking for changes before SOC starts
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then

salt/soc/defaults.yaml

@@ -1530,16 +1530,16 @@ soc:
               healthTimeoutSeconds: 5
           agentic: false
           agentMapping:
-            Orchestrator: sonnet
+            Orchestrator: Claude Sonnet
-            Investigator: sonnet
+            Investigator: Claude Sonnet
-            Detection Engineer: sonnet
+            Detection Engineer: Claude Sonnet
         onionconfig:
           saltstackDir: /opt/so/saltstack
           bypassEnabled: false
         postgres:
           host: ""
           port: 5432
-          sslMode: "allow"
+          sslMode: "require"
           database: securityonion
           user: ""
           password: ""

salt/soc/files/soc/playbook_placeholder_map.yaml

@@ -14,18 +14,22 @@
 
 CommandLine: process.command_line
 CurrentDirectory: process.working_directory
+Details: registry.data.strings
 Image: process.executable
-ImageLoaded: dll.name
+ImageLoaded: dll.path
 ParentImage: process.parent.executable
 ParentName: process.parent.name
 ParentProcessGuid: process.parent.entity_id
 ProcessGuid: process.entity_id
-TargetFilename: file.name
+TargetFilename: file.path
 TargetObject: registry.path
 TargetUserName: user.target.name
 User: user.name
 community_id: network.community_id
 dns_resolved_ip: dns.resolved_ip
+QueryName: dns.question.name
+dll_basename: dll.name
+file_basename: file.name
 document_id: soc_id
 dst_ip: destination.ip
 dst_port: destination.port
@@ -40,10 +44,15 @@ public_ip: network.public_ip
 related_hosts: related.hosts
 related_ip: related.ip
 src_ip: source.ip
-dns_query_name: dns.query_name
+dns_query_name: dns.question.name
 flow_id: log.id.uid
 payload: network.data.decoded
 rule_category: rule.category
 rule_name: rule.name
 rule_uuid: rule.uuid
 src_port: source.port
+# PowerShell channel (ps_script EID 4104 / classic EID 400)
+ScriptBlockText: powershell.file.script_block_text
+script_block_id: powershell.file.script_block_id
+script_block_hash: powershell.file.script_block_hash
+EngineVersion: powershell.engine.version

salt/soc/files/soc/sigma_playbook_pipeline.yaml

@@ -1,8 +1,9 @@
 name: Security Onion - Playbook Pipeline
 priority: 97
 transformations:
-    # Route string fields to their lowercase-normalized .caseless subfield so wildcard
+    # Route to lowercase-normalized .caseless subfields for case-insensitive matching.
-    # matches are case-insensitive.
+    # file.path.caseless exists on Defend only (Sysmon file events lack it);
+    # registry.path / dll.path / file.name have no .caseless on any source.
     - id: case_insensitive_string_fields
       type: field_name_mapping
       mapping:
@@ -10,3 +11,121 @@ transformations:
         process.parent.executable: process.parent.executable.caseless
         process.command_line: process.command_line.caseless
         process.parent.command_line: process.parent.command_line.caseless
+        file.path: file.path.caseless
+    # file_activity: playbook-only pseudo-category spanning all file operations.
+    - id: playbook_file_activity_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'file'
+      rule_conditions:
+      - type: logsource
+        category: file_activity
+    # EventType -> event.action is safe as a projection only — the values differ
+    # (Sysmon DeleteFile/SetValue vs ECS deletion/modification).
+    - id: playbook_file_activity_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: file_activity
+    - id: playbook_file_event_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: file_event
+    - id: playbook_file_delete_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+    - id: playbook_file_rename_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+    - id: playbook_registry_set_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: registry_set
+    # WMI-Activity events carry no event.category; event.dataset is source-specific
+    - id: playbook_wmi_activity_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'wmi.operational'
+      rule_conditions:
+      - type: logsource
+        category: wmi
+    - id: playbook_wmi_event_activity_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'wmi.operational'
+      rule_conditions:
+      - type: logsource
+        category: wmi_event
+    - id: playbook_ps_script_field-mapping
+      type: field_name_mapping
+      mapping:
+        ScriptBlockText: powershell.file.script_block_text
+        Path: file.path
+      rule_conditions:
+      - type: logsource
+        category: ps_script
+    - id: playbook_ps_script_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'windows.powershell_operational'
+        event.code: '4104'
+      rule_conditions:
+      - type: logsource
+        category: ps_script
+    # Data (raw EID 400/600 message blob) -> process.command_line: the winlog
+    # integration parses HostApplication into it; `message` is analyzed text and
+    # wildcard matches on it silently return zero. No .caseless on this dataset.
+    - id: playbook_ps_classic_start_field-mapping
+      type: field_name_mapping
+      mapping:
+        Data: process.command_line
+      rule_conditions:
+      - type: logsource
+        category: ps_classic_start
+    - id: playbook_ps_classic_start_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'windows.powershell'
+        event.code: '400'
+      rule_conditions:
+      - type: logsource
+        category: ps_classic_start
+    - id: playbook_ps_classic_provider_start_field-mapping
+      type: field_name_mapping
+      mapping:
+        Data: process.command_line
+      rule_conditions:
+      - type: logsource
+        category: ps_classic_provider_start
+    - id: playbook_ps_classic_provider_start_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'windows.powershell'
+        event.code: '600'
+      rule_conditions:
+      - type: logsource
+        category: ps_classic_provider_start
+    # Alert docs nest the host under event_data.host.name; no top-level host.name.
+    - id: playbook_alert_host_field
+      type: field_name_mapping
+      mapping:
+        host.name: event_data.host.name
+      rule_conditions:
+      - type: logsource
+        category: alert

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -131,13 +131,22 @@ transformations:
       type: field_name_mapping
       mapping:
         winlog.event_data.Details: registry.data.strings
-        # field rename only; EventType values (SetValue/CreateKey) still differ from
-        # event.action values (modification/creation)
-        winlog.event_data.EventType: event.action
       rule_conditions:
       - type: logsource
         product: windows
         category: registry_set
+    # ecs_windows renames Initiated -> network.direction without translating the value,
+    # so rules compile to network.direction:"true" and never match (field holds
+    # ingress/egress). network.initiated carries the boolean on both Defend and Sysmon.
+    # Keyed on network.direction because ecs_windows has already renamed by this layer.
+    - id: ecs_fix_network_connection_initiated
+      type: field_name_mapping
+      mapping:
+        network.direction: network.initiated
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: network_connection
     - id: ecs_fix_image_load
       type: field_name_mapping
       mapping:
@@ -150,6 +159,30 @@ transformations:
       - type: logsource
         product: windows
         category: image_load
+    # Defend reports the driver image in dll.path (file.path is null on driver events) —
+    # same renames as image_load.
+    - id: ecs_fix_driver_load
+      type: field_name_mapping
+      mapping:
+        file.path: dll.path
+        file.code_signature.signed: dll.code_signature.exists
+        winlog.event_data.Signature: dll.code_signature.subject_name
+        file.code_signature.status: dll.code_signature.status
+        winlog.event_data.Hashes: dll.hash.sha256
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: driver_load
+    - id: ecs_fix_file_rename
+      type: field_name_mapping
+      mapping:
+        # Defend records the pre-rename path in file.Ext.original.path; Sysmon's
+        # SourceFilename has no ECS equivalent.
+        winlog.event_data.SourceFilename: file.Ext.original.path
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: file_rename
     - id: linux_security_add-fields
       type: add_condition
       conditions:
@@ -323,6 +356,33 @@ transformations:
       rule_conditions:
       - type: logsource
         category: file_event
+    # Without event.type scoping, file_delete rules also match creations.
+    - id: endpoint_file_delete_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'file'
+        event.type: 'deletion'
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+    # Defend reports renames as event.action:rename with event.type:change.
+    - id: endpoint_file_rename_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'file'
+        event.action: 'rename'
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+    # event.type:change selects writes and excludes Defend's read-only registry events.
+    - id: endpoint_registry_set_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'registry'
+        event.type: 'change'
+      rule_conditions:
+      - type: logsource
+        category: registry_set
     # Scope image_load rules to Elastic Endpoint library events (event.category:library, dll.*
     # populated).
     - id: endpoint_image_load_add-fields
@@ -351,6 +411,149 @@ transformations:
       rule_conditions:
       - type: logsource
         category: network_connection
+    # Scope on lookup actions, not network.protocol:dns (also matches Zeek port-53).
+    - id: endpoint_dns_query_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'network'
+        event.action:
+          - 'lookup_requested'
+          - 'lookup_result'
+      rule_conditions:
+      - type: logsource
+        category: dns_query
+    # OS gates: without them a `product: windows` rule in these categories also matches
+    # Linux/macOS. Separate conditions (not `product:` on the mappings above) so the
+    # mappings keep scoping product-less rules.
+    - id: endpoint_file_create_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: file_event
+        product: windows
+    - id: endpoint_file_create_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: file_event
+        product: linux
+    - id: endpoint_file_create_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: file_event
+        product: macos
+    - id: endpoint_file_delete_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+        product: windows
+    - id: endpoint_file_delete_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+        product: linux
+    - id: endpoint_file_delete_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+        product: macos
+    - id: endpoint_file_rename_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+        product: windows
+    - id: endpoint_file_rename_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+        product: linux
+    - id: endpoint_file_rename_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+        product: macos
+    - id: endpoint_network_connection_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: network_connection
+        product: windows
+    - id: endpoint_network_connection_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: network_connection
+        product: linux
+    - id: endpoint_network_connection_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: network_connection
+        product: macos
+    - id: endpoint_dns_query_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: dns_query
+        product: windows
+    - id: endpoint_dns_query_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: dns_query
+        product: linux
+    - id: endpoint_dns_query_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: dns_query
+        product: macos
+    # registry_set / image_load / driver_load are not gated — windows-only telemetry.
+    # wmi / wmi_event are not scoped here: their only handle is event.dataset
+    - id: endpoint_driver_load_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'driver'
+      rule_conditions:
+      - type: logsource
+        category: driver_load
     # Maps "alert" category to SO Alert events
     - id: alert_so_add-fields
       type: add_condition

salt/soc/merged.map.jinja

@@ -88,7 +88,7 @@
           'host': GLOBALS.manager_ip,
           'password': PG_PASS,
           'port': 5432,
-          'sslMode': 'allow',
+          'sslMode': 'require',
           'user': PG_USER,
         }
       }) %}

salt/soc/soc_soc.yaml

@@ -486,7 +486,7 @@ soc:
             global: True
             advanced: True
           sslMode:
-            description: "Use encrypted connections to the PostgreSQL server. Must be one of the following values: disable, allow, prefer, require, verify-ca, verify-full.  Defaults to allow."
+            description: "Use encrypted connections to the PostgreSQL server. Must be one of the following values: disable, allow, prefer, require, verify-ca, verify-full."
             global: True
             advanced: True
           database: