Rename Detection Engineer agent key to DetectionEngineer

The agent name is also the agentMapping config key and a SOC setting id
segment, so the space made it awkward to target. Matches the rename in
securityonion-soc.

salt/common/tools/sbin/so-log-check

@@ -133,6 +133,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Cancelling deferred write event maybeFenceReplicas because the event queue is now closed" # Kafka controller log during shutdown/restart
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Redis may have been restarted" # Redis likely restarted by salt
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|file already closed"          # Go logging race condition during container restart
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|relation \"audit_settings\" does not exist" # salt checking for changes before SOC starts
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then

salt/logrotate/defaults.yaml

@@ -150,6 +150,16 @@ logrotate:
       - extension .log
       - dateext
       - dateyesterday
+    /opt/so/log/postgres/*_x_log:
+      - daily
+      - rotate 14
+      - missingok
+      - copytruncate
+      - compress
+      - create
+      - extension .log
+      - dateext
+      - dateyesterday
     /opt/so/log/telegraf/*_x_log:
       - daily
       - rotate 14

salt/logrotate/soc_logrotate.yaml

@@ -91,6 +91,13 @@ logrotate:
       multiline: True
       global: True
       forcedType: "[]string"
+    "/opt/so/log/postgres/*_x_log":
+      description: List of logrotate options for this file.
+      title: /opt/so/log/postgres/*.log
+      advanced: True
+      multiline: True
+      global: True
+      forcedType: "[]string"
     "/opt/so/log/telegraf/*_x_log":
       description: List of logrotate options for this file.
       title: /opt/so/log/telegraf/*.log

salt/manager/tools/sbin/soup

@@ -880,29 +880,21 @@ recollate_postgres() {
   echo ""
 }
 
-bootstrap_so_soc_database() {
-  # init-db.sh is mounted into so-postgres at /docker-entrypoint-initdb.d/init-db.sh
-  # and runs automatically only on a fresh data directory. Hosts upgrading from
-  # 3.1.0 already have /nsm/postgres populated, so the so_soc bootstrap block
-  # added in 3.2 never fires. Re-run the script explicitly; it's idempotent.
-  echo "Bootstrapping database via init-db.sh."
-  # The postgres image has no USER directive, so `docker exec` defaults to
-  # root, and the container env intentionally omits POSTGRES_USER (the upstream
-  # entrypoint defaults it transiently during first-init only). Recreate both
-  # so psql inside init-db.sh resolves the connect user correctly.
-  local exec_cmd="docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh"
-  if ! /usr/sbin/so-postgres-wait; then
-    FINAL_MESSAGE_QUEUE+=("WARNING: so-postgres was not ready during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd")
+scrub_postgres_log_passwords() {
+  # Purge plaintext passwords a pre-3.2 postgres could log on DDL errors.
+  local log=/opt/so/log/postgres/postgres.log
+  [[ -f "$log" ]] || return 0
+  if ! grep -qai "PASSWORD '" "$log" 2>/dev/null; then
+    echo "No leaked passwords found in $log."
     return 0
   fi
-  if ! $exec_cmd; then
-    FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the database may not have been bootstrapped. Re-run manually: $exec_cmd")
-    return 0
-  fi
-  echo "Database bootstrap complete."
-
-  echo "Restarting so-soc container to pick up database changes"
-  docker restart so-soc
+  echo "Removing leaked password statements from $log."
+  local tmp
+  tmp=$(mktemp)
+  # Rewrite in place (cat >) to keep the inode postgres is writing to.
+  grep -avi "PASSWORD '" "$log" > "$tmp" 2>/dev/null || true
+  cat "$tmp" > "$log"
+  rm -f "$tmp"
 }
 
 # Existing grids should keep ILM unless an admin explicitly opts in to DLM.
@@ -989,7 +981,8 @@ post_to_3.2.0() {
   # Recollate due to image OS rebase
   recollate_postgres
 
-  bootstrap_so_soc_database
+  # SOC database bootstrap is handled by the postgres.enabled highstate.
+  scrub_postgres_log_passwords
 
   # Generate 9.3.7 elastic agent installers
   echo "Regenerating Elastic Agent Installers"

salt/postgres/enabled.sls

@@ -85,6 +85,23 @@ so-postgres:
       - x509: postgres_crt
       - x509: postgres_key
 
+postgres_wait_ready:
+  cmd.run:
+    - name: /usr/sbin/so-postgres-wait
+    - require:
+      - docker_container: so-postgres
+      - file: postgres_sbin
+
+# Reconcile the SOC database (role, grants, so_telegraf) every highstate so a
+# partially-initialized cluster self-heals. POSTGRES_USER is injected because
+# the container env omits it.
+postgres_bootstrap_soc_db:
+  cmd.run:
+    - name: docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh
+    - require:
+      - cmd: postgres_wait_ready
+      - file: postgresinitdb
+
 delete_so-postgres_so-status.disabled:
   file.uncomment:
     - name: /opt/so/conf/so-status/so-status.conf

salt/postgres/files/init-db.sh

@@ -8,13 +8,17 @@ if [ -z "${SO_POSTGRES_PASS:-}" ] && [ -n "${SO_POSTGRES_PASS_FILE:-}" ] && [ -r
     SO_POSTGRES_PASS="$(< "$SO_POSTGRES_PASS_FILE")"
 fi
 psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    -- Keep the password out of postgres.log if this DDL errors.
+    SET log_min_error_statement = panic;
+    -- Idempotent, race-safe upsert: CREATE, falling back to ALTER if the role
+    -- already exists or is created concurrently.
     DO \$\$
     BEGIN
-        IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${SO_POSTGRES_USER}') THEN
+        BEGIN
             EXECUTE format('CREATE ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}');
-        ELSE
-            EXECUTE format('ALTER ROLE %I WITH PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}');
-        END IF;
+        EXCEPTION WHEN duplicate_object OR unique_violation THEN
+            EXECUTE format('ALTER ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}');
+        END;
     END
     \$\$;
     GRANT ALL ON SCHEMA public TO "$SO_POSTGRES_USER";

salt/postgres/telegraf_users.sls

@@ -8,23 +8,14 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'telegraf/map.jinja' import TELEGRAFMERGED %}
 
-{# postgres_wait_ready below requires `docker_container: so-postgres`, which is
-   declared in postgres.enabled. Include it here so state.apply postgres.telegraf_users
-   on its own (e.g. from orch.deploy_newnode) still has that ID in scope. Salt
-   de-duplicates the circular include. #}
+{# postgres.enabled declares the so-postgres container and postgres_wait_ready
+   that the requires below reference. Salt de-duplicates the circular include. #}
 include:
   - postgres.enabled
 
 {% set TG_OUT = TELEGRAFMERGED.output | upper %}
 {% if TG_OUT in ['POSTGRES', 'BOTH'] %}
 
-postgres_wait_ready:
-  cmd.run:
-    - name: /usr/sbin/so-postgres-wait
-    - require:
-      - docker_container: so-postgres
-      - file: postgres_sbin
-
 # Ensure the shared Telegraf database exists. init-db.sh only runs on a
 # fresh data dir, so hosts upgraded onto an existing /nsm/postgres volume
 # would otherwise never get so_telegraf.

salt/postgres/tools/sbin/so-telegraf-postgres

@@ -71,6 +71,8 @@ EOSQL
       -v role_user="$ROLE_USER" \
       -v role_pass="$ROLE_PASS" \
       -U postgres -d so_telegraf <<'EOSQL'
+-- Keep the password out of postgres.log if this DDL errors.
+SET log_min_error_statement = panic;
 SELECT format(
   CASE WHEN EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'role_user')
        THEN 'ALTER ROLE %I WITH LOGIN PASSWORD %L'

salt/soc/defaults.yaml

@@ -1530,16 +1530,16 @@ soc:
               healthTimeoutSeconds: 5
           agentic: false
           agentMapping:
-            Orchestrator: sonnet
-            Investigator: sonnet
-            Detection Engineer: sonnet
+            Orchestrator: Claude Sonnet
+            Investigator: Claude Sonnet
+            DetectionEngineer: Claude Sonnet
         onionconfig:
           saltstackDir: /opt/so/saltstack
           bypassEnabled: false
         postgres:
           host: ""
           port: 5432
-          sslMode: "allow"
+          sslMode: "require"
           database: securityonion
           user: ""
           password: ""

salt/soc/files/soc/playbook_placeholder_map.yaml

@@ -14,18 +14,22 @@
 
 CommandLine: process.command_line
 CurrentDirectory: process.working_directory
+Details: registry.data.strings
 Image: process.executable
-ImageLoaded: dll.name
+ImageLoaded: dll.path
 ParentImage: process.parent.executable
 ParentName: process.parent.name
 ParentProcessGuid: process.parent.entity_id
 ProcessGuid: process.entity_id
-TargetFilename: file.name
+TargetFilename: file.path
 TargetObject: registry.path
 TargetUserName: user.target.name
 User: user.name
 community_id: network.community_id
 dns_resolved_ip: dns.resolved_ip
+QueryName: dns.question.name
+dll_basename: dll.name
+file_basename: file.name
 document_id: soc_id
 dst_ip: destination.ip
 dst_port: destination.port
@@ -40,10 +44,15 @@ public_ip: network.public_ip
 related_hosts: related.hosts
 related_ip: related.ip
 src_ip: source.ip
-dns_query_name: dns.query_name
+dns_query_name: dns.question.name
 flow_id: log.id.uid
 payload: network.data.decoded
 rule_category: rule.category
 rule_name: rule.name
 rule_uuid: rule.uuid
 src_port: source.port
+# PowerShell channel (ps_script EID 4104 / classic EID 400)
+ScriptBlockText: powershell.file.script_block_text
+script_block_id: powershell.file.script_block_id
+script_block_hash: powershell.file.script_block_hash
+EngineVersion: powershell.engine.version

salt/soc/files/soc/sigma_playbook_pipeline.yaml

@@ -1,8 +1,9 @@
 name: Security Onion - Playbook Pipeline
 priority: 97
 transformations:
-    # Route string fields to their lowercase-normalized .caseless subfield so wildcard
-    # matches are case-insensitive.
+    # Route to lowercase-normalized .caseless subfields for case-insensitive matching.
+    # file.path.caseless exists on Defend only (Sysmon file events lack it);
+    # registry.path / dll.path / file.name have no .caseless on any source.
     - id: case_insensitive_string_fields
       type: field_name_mapping
       mapping:
@@ -10,3 +11,121 @@ transformations:
         process.parent.executable: process.parent.executable.caseless
         process.command_line: process.command_line.caseless
         process.parent.command_line: process.parent.command_line.caseless
+        file.path: file.path.caseless
+    # file_activity: playbook-only pseudo-category spanning all file operations.
+    - id: playbook_file_activity_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'file'
+      rule_conditions:
+      - type: logsource
+        category: file_activity
+    # EventType -> event.action is safe as a projection only — the values differ
+    # (Sysmon DeleteFile/SetValue vs ECS deletion/modification).
+    - id: playbook_file_activity_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: file_activity
+    - id: playbook_file_event_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: file_event
+    - id: playbook_file_delete_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+    - id: playbook_file_rename_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+    - id: playbook_registry_set_event_type
+      type: field_name_mapping
+      mapping:
+        winlog.event_data.EventType: event.action
+      rule_conditions:
+      - type: logsource
+        category: registry_set
+    # WMI-Activity events carry no event.category; event.dataset is source-specific
+    - id: playbook_wmi_activity_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'wmi.operational'
+      rule_conditions:
+      - type: logsource
+        category: wmi
+    - id: playbook_wmi_event_activity_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'wmi.operational'
+      rule_conditions:
+      - type: logsource
+        category: wmi_event
+    - id: playbook_ps_script_field-mapping
+      type: field_name_mapping
+      mapping:
+        ScriptBlockText: powershell.file.script_block_text
+        Path: file.path
+      rule_conditions:
+      - type: logsource
+        category: ps_script
+    - id: playbook_ps_script_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'windows.powershell_operational'
+        event.code: '4104'
+      rule_conditions:
+      - type: logsource
+        category: ps_script
+    # Data (raw EID 400/600 message blob) -> process.command_line: the winlog
+    # integration parses HostApplication into it; `message` is analyzed text and
+    # wildcard matches on it silently return zero. No .caseless on this dataset.
+    - id: playbook_ps_classic_start_field-mapping
+      type: field_name_mapping
+      mapping:
+        Data: process.command_line
+      rule_conditions:
+      - type: logsource
+        category: ps_classic_start
+    - id: playbook_ps_classic_start_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'windows.powershell'
+        event.code: '400'
+      rule_conditions:
+      - type: logsource
+        category: ps_classic_start
+    - id: playbook_ps_classic_provider_start_field-mapping
+      type: field_name_mapping
+      mapping:
+        Data: process.command_line
+      rule_conditions:
+      - type: logsource
+        category: ps_classic_provider_start
+    - id: playbook_ps_classic_provider_start_add-fields
+      type: add_condition
+      conditions:
+        event.dataset: 'windows.powershell'
+        event.code: '600'
+      rule_conditions:
+      - type: logsource
+        category: ps_classic_provider_start
+    # Alert docs nest the host under event_data.host.name; no top-level host.name.
+    - id: playbook_alert_host_field
+      type: field_name_mapping
+      mapping:
+        host.name: event_data.host.name
+      rule_conditions:
+      - type: logsource
+        category: alert

salt/soc/files/soc/sigma_so_pipeline.yaml

@@ -131,13 +131,22 @@ transformations:
       type: field_name_mapping
       mapping:
         winlog.event_data.Details: registry.data.strings
-        # field rename only; EventType values (SetValue/CreateKey) still differ from
-        # event.action values (modification/creation)
-        winlog.event_data.EventType: event.action
       rule_conditions:
       - type: logsource
         product: windows
         category: registry_set
+    # ecs_windows renames Initiated -> network.direction without translating the value,
+    # so rules compile to network.direction:"true" and never match (field holds
+    # ingress/egress). network.initiated carries the boolean on both Defend and Sysmon.
+    # Keyed on network.direction because ecs_windows has already renamed by this layer.
+    - id: ecs_fix_network_connection_initiated
+      type: field_name_mapping
+      mapping:
+        network.direction: network.initiated
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: network_connection
     - id: ecs_fix_image_load
       type: field_name_mapping
       mapping:
@@ -150,6 +159,30 @@ transformations:
       - type: logsource
         product: windows
         category: image_load
+    # Defend reports the driver image in dll.path (file.path is null on driver events) —
+    # same renames as image_load.
+    - id: ecs_fix_driver_load
+      type: field_name_mapping
+      mapping:
+        file.path: dll.path
+        file.code_signature.signed: dll.code_signature.exists
+        winlog.event_data.Signature: dll.code_signature.subject_name
+        file.code_signature.status: dll.code_signature.status
+        winlog.event_data.Hashes: dll.hash.sha256
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: driver_load
+    - id: ecs_fix_file_rename
+      type: field_name_mapping
+      mapping:
+        # Defend records the pre-rename path in file.Ext.original.path; Sysmon's
+        # SourceFilename has no ECS equivalent.
+        winlog.event_data.SourceFilename: file.Ext.original.path
+      rule_conditions:
+      - type: logsource
+        product: windows
+        category: file_rename
     - id: linux_security_add-fields
       type: add_condition
       conditions:
@@ -323,6 +356,33 @@ transformations:
       rule_conditions:
       - type: logsource
         category: file_event
+    # Without event.type scoping, file_delete rules also match creations.
+    - id: endpoint_file_delete_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'file'
+        event.type: 'deletion'
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+    # Defend reports renames as event.action:rename with event.type:change.
+    - id: endpoint_file_rename_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'file'
+        event.action: 'rename'
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+    # event.type:change selects writes and excludes Defend's read-only registry events.
+    - id: endpoint_registry_set_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'registry'
+        event.type: 'change'
+      rule_conditions:
+      - type: logsource
+        category: registry_set
     # Scope image_load rules to Elastic Endpoint library events (event.category:library, dll.*
     # populated).
     - id: endpoint_image_load_add-fields
@@ -351,6 +411,149 @@ transformations:
       rule_conditions:
       - type: logsource
         category: network_connection
+    # Scope on lookup actions, not network.protocol:dns (also matches Zeek port-53).
+    - id: endpoint_dns_query_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'network'
+        event.action:
+          - 'lookup_requested'
+          - 'lookup_result'
+      rule_conditions:
+      - type: logsource
+        category: dns_query
+    # OS gates: without them a `product: windows` rule in these categories also matches
+    # Linux/macOS. Separate conditions (not `product:` on the mappings above) so the
+    # mappings keep scoping product-less rules.
+    - id: endpoint_file_create_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: file_event
+        product: windows
+    - id: endpoint_file_create_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: file_event
+        product: linux
+    - id: endpoint_file_create_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: file_event
+        product: macos
+    - id: endpoint_file_delete_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+        product: windows
+    - id: endpoint_file_delete_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+        product: linux
+    - id: endpoint_file_delete_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: file_delete
+        product: macos
+    - id: endpoint_file_rename_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+        product: windows
+    - id: endpoint_file_rename_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+        product: linux
+    - id: endpoint_file_rename_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: file_rename
+        product: macos
+    - id: endpoint_network_connection_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: network_connection
+        product: windows
+    - id: endpoint_network_connection_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: network_connection
+        product: linux
+    - id: endpoint_network_connection_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: network_connection
+        product: macos
+    - id: endpoint_dns_query_windows_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'windows'
+      rule_conditions:
+      - type: logsource
+        category: dns_query
+        product: windows
+    - id: endpoint_dns_query_linux_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'linux'
+      rule_conditions:
+      - type: logsource
+        category: dns_query
+        product: linux
+    - id: endpoint_dns_query_macos_os-gate
+      type: add_condition
+      conditions:
+        host.os.type: 'macos'
+      rule_conditions:
+      - type: logsource
+        category: dns_query
+        product: macos
+    # registry_set / image_load / driver_load are not gated — windows-only telemetry.
+    # wmi / wmi_event are not scoped here: their only handle is event.dataset
+    - id: endpoint_driver_load_add-fields
+      type: add_condition
+      conditions:
+        event.category: 'driver'
+      rule_conditions:
+      - type: logsource
+        category: driver_load
     # Maps "alert" category to SO Alert events
     - id: alert_so_add-fields
       type: add_condition

salt/soc/merged.map.jinja

@@ -88,7 +88,7 @@
           'host': GLOBALS.manager_ip,
           'password': PG_PASS,
           'port': 5432,
-          'sslMode': 'allow',
+          'sslMode': 'require',
           'user': PG_USER,
         }
       }) %}

salt/soc/soc_soc.yaml

@@ -486,7 +486,7 @@ soc:
             global: True
             advanced: True
           sslMode:
-            description: "Use encrypted connections to the PostgreSQL server. Must be one of the following values: disable, allow, prefer, require, verify-ca, verify-full.  Defaults to allow."
+            description: "Use encrypted connections to the PostgreSQL server. Must be one of the following values: disable, allow, prefer, require, verify-ca, verify-full."
             global: True
             advanced: True
           database:
@@ -786,7 +786,7 @@ soc:
             Investigator:
               description: This agent investigates alerts, explains events and records, and hunts through event data. It can also acknowledge alerts and escalate to cases.
               global: True
-            Detection Engineer:
+            DetectionEngineer:
               description: This agent manages detections and their overrides, including tuning noisy rules and authoring rule content.
               global: True
       client: