Merge pull request #16105 from Security-Onion-Solutions/fixESHEAP

Add default esheap value so nodes highstate without the pillar value

salt/common/tools/sbin/so-log-check

@@ -133,6 +133,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Cancelling deferred write event maybeFenceReplicas because the event queue is now closed" # Kafka controller log during shutdown/restart
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Redis may have been restarted" # Redis likely restarted by salt
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|file already closed"          # Go logging race condition during container restart
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|relation \"audit_settings\" does not exist" # salt checking for changes before SOC starts
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then

salt/elasticsearch/defaults.yaml

@@ -1,5 +1,6 @@
 elasticsearch:
   enabled: false
+  esheap: '600m'
   version: 9.3.7
   index_clean: true
   data_retention_method: DLM

salt/manager/tools/sbin/soup

@@ -1998,11 +1998,11 @@ main() {
     # Testing that salt-master is up by checking that is it connected to itself
     check_saltmaster_status
 
-    # update the salt-minion configs here and start the minion
+    # update the salt-master and salt-minion configs here and start the minion
     # since highstate are disabled above, minion start should not trigger a highstate
     echo ""
-    echo "Ensuring salt-minion configs are up-to-date."
+    echo "Ensuring salt-master and salt-minion configs are up-to-date."
-    salt-call state.apply salt.minion -l info queue=True
+    salt-call state.apply salt.master -l info queue=True
     echo ""
 
     # ensure the mine is updated and populated before highstates run, following the salt-master restart

salt/salt/master.sls

@@ -94,7 +94,9 @@ salt_master_service:
       - file: checkmine_engine
       - file: pillarWatch_engine
       - file: engines_config
-    - order: 9002
+    - require:
+      - cmd: wait_for_salt_minion_ready
+    - order: last
 
 {% else %}
 

salt/soc/defaults.yaml

@@ -1539,7 +1539,7 @@ soc:
         postgres:
           host: ""
           port: 5432
-          sslMode: "allow"
+          sslMode: "require"
           database: securityonion
           user: ""
           password: ""

salt/soc/files/soc/playbook_placeholder_map.yaml

@@ -14,18 +14,22 @@
 
 CommandLine: process.command_line
 CurrentDirectory: process.working_directory
+Details: registry.data.strings
 Image: process.executable
-ImageLoaded: dll.name
+ImageLoaded: dll.path
 ParentImage: process.parent.executable
 ParentName: process.parent.name
 ParentProcessGuid: process.parent.entity_id
 ProcessGuid: process.entity_id
-TargetFilename: file.name
+TargetFilename: file.path
 TargetObject: registry.path
 TargetUserName: user.target.name
 User: user.name
 community_id: network.community_id
 dns_resolved_ip: dns.resolved_ip
+QueryName: dns.question.name
+dll_basename: dll.name
+file_basename: file.name
 document_id: soc_id
 dst_ip: destination.ip
 dst_port: destination.port
@@ -40,10 +44,15 @@ public_ip: network.public_ip
 related_hosts: related.hosts
 related_ip: related.ip
 src_ip: source.ip
-dns_query_name: dns.query_name
+dns_query_name: dns.question.name
 flow_id: log.id.uid
 payload: network.data.decoded
 rule_category: rule.category
 rule_name: rule.name
 rule_uuid: rule.uuid
 src_port: source.port
+# PowerShell channel (ps_script EID 4104 / classic EID 400)
+ScriptBlockText: powershell.file.script_block_text
+script_block_id: powershell.file.script_block_id
+script_block_hash: powershell.file.script_block_hash
+EngineVersion: powershell.engine.version

salt/soc/files/soc/sigma_playbook_pipeline.yaml

@@ -1,8 +1,9 @@
 name: Security Onion - Playbook Pipeline
 priority: 97
 transformations:
-    # Route string fields to their lowercase-normalized .caseless subfield so wildcard
+    # Route to lowercase-normalized .caseless subfields for case-insensitive matching.
-    # matches are case-insensitive.
+    # file.path.caseless exists on Defend only (Sysmon file events lack it);
+    # registry.path / dll.path / file.name have no .caseless on any source.
     - id: case_insensitive_string_fields
       type: field_name_mapping
       mapping:
@@ -10,3 +11,121 @@ transformations:
         process.parent.executable: process.parent.executable.caseless
         process.command_line: process.command_line.caseless
         process.parent.command_line: process.parent.command_line.caseless
+        file.path: file.path.caseless
+    # file_activity: playbook-only pseudo-category spanning all file operations.
+    - id: playbook_file_activity_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'file'
+      rule_conditions:
+      - type: logsource
+        category: file_activity
+    # EventType -> event.action is safe as a projection only — the values differ
+    # (Sysmon DeleteFile/SetValue vs ECS deletion/modification).
+    - id: playbook_file_activity_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: file_activity
+    - id: playbook_file_event_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: file_event
+    - id: playbook_file_delete_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+    - id: playbook_file_rename_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+    - id: playbook_registry_set_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: registry_set
+    # WMI-Activity events carry no event.category; event.dataset is source-specific
+    - id: playbook_wmi_activity_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'wmi.operational'
+      rule_conditions:
+      - type: logsource
+        category: wmi
+    - id: playbook_wmi_event_activity_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'wmi.operational'
+      rule_conditions:
+      - type: logsource
+        category: wmi_event
+    - id: playbook_ps_script_field-mapping
+      type: field_name_mapping
+      mapping:
+        ScriptBlockText: powershell.file.script_block_text
+        Path: file.path
+      rule_conditions:
+      - type: logsource
+        category: ps_script
+    - id: playbook_ps_script_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'windows.powershell_operational'
+        event.code: '4104'
+      rule_conditions:
+      - type: logsource
+        category: ps_script
+    # Data (raw EID 400/600 message blob) -> process.command_line: the winlog
+    # integration parses HostApplication into it; `message` is analyzed text and
+    # wildcard matches on it silently return zero. No .caseless on this dataset.
+    - id: playbook_ps_classic_start_field-mapping
+      type: field_name_mapping
+      mapping:
+        Data: process.command_line
+      rule_conditions:
+      - type: logsource
+        category: ps_classic_start
+    - id: playbook_ps_classic_start_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'windows.powershell'
+        event.code: '400'
+      rule_conditions:
+      - type: logsource
+        category: ps_classic_start
+    - id: playbook_ps_classic_provider_start_field-mapping
+      type: field_name_mapping
+      mapping:
+        Data: process.command_line
+      rule_conditions:
+      - type: logsource
+        category: ps_classic_provider_start
+    - id: playbook_ps_classic_provider_start_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'windows.powershell'
+        event.code: '600'
+      rule_conditions:
+      - type: logsource
+        category: ps_classic_provider_start
+    # Alert docs nest the host under event_data.host.name; no top-level host.name.
+    - id: playbook_alert_host_field
+      type: field_name_mapping
+      mapping:
+        host.name: event_data.host.name
+      rule_conditions:
+      - type: logsource
+        category: alert

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -131,13 +131,22 @@ transformations:
       type: field_name_mapping
       mapping:
         winlog.event_data.Details: registry.data.strings
-        # field rename only; EventType values (SetValue/CreateKey) still differ from
-        # event.action values (modification/creation)
-        winlog.event_data.EventType: event.action
       rule_conditions:
       - type: logsource
         product: windows
         category: registry_set
+    # ecs_windows renames Initiated -> network.direction without translating the value,
+    # so rules compile to network.direction:"true" and never match (field holds
+    # ingress/egress). network.initiated carries the boolean on both Defend and Sysmon.
+    # Keyed on network.direction because ecs_windows has already renamed by this layer.
+    - id: ecs_fix_network_connection_initiated
+      type: field_name_mapping
+      mapping:
+        network.direction: network.initiated
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: network_connection
     - id: ecs_fix_image_load
       type: field_name_mapping
       mapping:
@@ -150,6 +159,30 @@ transformations:
       - type: logsource
         product: windows
         category: image_load
+    # Defend reports the driver image in dll.path (file.path is null on driver events) —
+    # same renames as image_load.
+    - id: ecs_fix_driver_load
+      type: field_name_mapping
+      mapping:
+        file.path: dll.path
+        file.code_signature.signed: dll.code_signature.exists
+        winlog.event_data.Signature: dll.code_signature.subject_name
+        file.code_signature.status: dll.code_signature.status
+        winlog.event_data.Hashes: dll.hash.sha256
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: driver_load
+    - id: ecs_fix_file_rename
+      type: field_name_mapping
+      mapping:
+        # Defend records the pre-rename path in file.Ext.original.path; Sysmon's
+        # SourceFilename has no ECS equivalent.
+        winlog.event_data.SourceFilename: file.Ext.original.path
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: file_rename
     - id: linux_security_add-fields
       type: add_condition
       conditions:
@@ -323,6 +356,33 @@ transformations:
       rule_conditions:
       - type: logsource
         category: file_event
+    # Without event.type scoping, file_delete rules also match creations.
+    - id: endpoint_file_delete_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'file'
+        event.type: 'deletion'
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+    # Defend reports renames as event.action:rename with event.type:change.
+    - id: endpoint_file_rename_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'file'
+        event.action: 'rename'
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+    # event.type:change selects writes and excludes Defend's read-only registry events.
+    - id: endpoint_registry_set_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'registry'
+        event.type: 'change'
+      rule_conditions:
+      - type: logsource
+        category: registry_set
     # Scope image_load rules to Elastic Endpoint library events (event.category:library, dll.*
     # populated).
     - id: endpoint_image_load_add-fields
@@ -351,6 +411,149 @@ transformations:
       rule_conditions:
       - type: logsource
         category: network_connection
+    # Scope on lookup actions, not network.protocol:dns (also matches Zeek port-53).
+    - id: endpoint_dns_query_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'network'
+        event.action:
+          - 'lookup_requested'
+          - 'lookup_result'
+      rule_conditions:
+      - type: logsource
+        category: dns_query
+    # OS gates: without them a `product: windows` rule in these categories also matches
+    # Linux/macOS. Separate conditions (not `product:` on the mappings above) so the
+    # mappings keep scoping product-less rules.
+    - id: endpoint_file_create_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: file_event
+        product: windows
+    - id: endpoint_file_create_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: file_event
+        product: linux
+    - id: endpoint_file_create_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: file_event
+        product: macos
+    - id: endpoint_file_delete_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+        product: windows
+    - id: endpoint_file_delete_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+        product: linux
+    - id: endpoint_file_delete_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+        product: macos
+    - id: endpoint_file_rename_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+        product: windows
+    - id: endpoint_file_rename_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+        product: linux
+    - id: endpoint_file_rename_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+        product: macos
+    - id: endpoint_network_connection_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: network_connection
+        product: windows
+    - id: endpoint_network_connection_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: network_connection
+        product: linux
+    - id: endpoint_network_connection_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: network_connection
+        product: macos
+    - id: endpoint_dns_query_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: dns_query
+        product: windows
+    - id: endpoint_dns_query_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: dns_query
+        product: linux
+    - id: endpoint_dns_query_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: dns_query
+        product: macos
+    # registry_set / image_load / driver_load are not gated — windows-only telemetry.
+    # wmi / wmi_event are not scoped here: their only handle is event.dataset
+    - id: endpoint_driver_load_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'driver'
+      rule_conditions:
+      - type: logsource
+        category: driver_load
     # Maps "alert" category to SO Alert events
     - id: alert_so_add-fields
       type: add_condition

salt/soc/merged.map.jinja

@@ -88,7 +88,7 @@
           'host': GLOBALS.manager_ip,
           'password': PG_PASS,
           'port': 5432,
-          'sslMode': 'allow',
+          'sslMode': 'require',
           'user': PG_USER,
         }
       }) %}

salt/soc/soc_soc.yaml

@@ -486,7 +486,7 @@ soc:
             global: True
             advanced: True
           sslMode:
-            description: "Use encrypted connections to the PostgreSQL server. Must be one of the following values: disable, allow, prefer, require, verify-ca, verify-full.  Defaults to allow."
+            description: "Use encrypted connections to the PostgreSQL server. Must be one of the following values: disable, allow, prefer, require, verify-ca, verify-full."
             global: True
             advanced: True
           database:

salt/vars/elasticsearch.map.jinja

@@ -1,10 +1,11 @@
 {% import 'vars/init.map.jinja' as INIT %}
+{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
 
 {%
   set ELASTICSEARCH_GLOBALS = {
     'elasticsearch': {
       'es_cluster_name': INIT.PILLAR.elasticsearch.config.cluster.name,
-      'es_heap': INIT.PILLAR.elasticsearch.esheap
+      'es_heap': INIT.PILLAR.elasticsearch.get('esheap', ELASTICSEARCHDEFAULTS.elasticsearch.esheap)
     }
   }
 %}