From 48a7d66964e910b6793a36fc27621147d07d80a7 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Fri, 17 Jul 2026 15:20:20 -0400 Subject: [PATCH 1/9] Update baseline agents --- salt/soc/defaults.yaml | 3 ++- salt/soc/soc_soc.yaml | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 8df30736f..9705c1863 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1531,7 +1531,8 @@ soc: agentic: false agentMapping: Orchestrator: sonnet - Hunter: sonnet + Investigator: sonnet + Detection Engineer: sonnet onionconfig: saltstackDir: /opt/so/saltstack bypassEnabled: false diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 47c7ea3ee..f09317f39 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -783,8 +783,11 @@ soc: Orchestrator: description: The initial agent in most agentic conversations. This agent will delegate requests to specialized agents. global: True - Hunter: - description: This agent is specialized in querying events. + Investigator: + description: This agent investigates alerts, explains events and records, and hunts through event data. It can also acknowledge alerts and escalate to cases. + global: True + Detection Engineer: + description: This agent manages detections and their overrides, including tuning noisy rules and authoring rule content. global: True client: assistant: From ad78e84ccd7c114fa1719d15de267e31295570ac Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 20 Jul 2026 12:41:14 -0400 Subject: [PATCH 2/9] Prevent PostgreSQL from leaking plaintext passwords to postgres.log PostgreSQL's log_min_error_statement defaults to 'error', so whenever a CREATE/ALTER ROLE ... PASSWORD statement errored, the full statement text -- including the plaintext password -- was written to /opt/so/log/postgres/postgres.log. The role-provisioning paths (init-db.sh for so_postgres, so-telegraf-postgres for per-minion telegraf roles) both dispatch such DDL, the latter on every state.apply. - init-db.sh / so-telegraf-postgres: SET log_min_error_statement = panic before the password-bearing DDL so an error no longer emits the STATEMENT line. The ERROR message itself (no password) still logs, preserving debuggability. - logrotate: add a postgres stanza (daily, keep 14, copytruncate, compress) so postgres.log is rotated like every other service and leaked content can't persist indefinitely. copytruncate is required because the container holds the log open via redirected stderr. - soup: scrub any already-logged PASSWORD lines from postgres.log during post_to_3.2.0, rewriting in place to preserve the inode postgres is writing to. --- salt/logrotate/defaults.yaml | 10 +++++++ salt/logrotate/soc_logrotate.yaml | 7 +++++ salt/manager/tools/sbin/soup | 27 +++++++++++++++++++ salt/postgres/files/init-db.sh | 4 +++ salt/postgres/tools/sbin/so-telegraf-postgres | 4 +++ 5 files changed, 52 insertions(+) diff --git a/salt/logrotate/defaults.yaml b/salt/logrotate/defaults.yaml index 2261bb4f7..e37193d0d 100644 --- a/salt/logrotate/defaults.yaml +++ b/salt/logrotate/defaults.yaml @@ -150,6 +150,16 @@ logrotate: - extension .log - dateext - dateyesterday + /opt/so/log/postgres/*_x_log: + - daily + - rotate 14 + - missingok + - copytruncate + - compress + - create + - extension .log + - dateext + - dateyesterday /opt/so/log/telegraf/*_x_log: - daily - rotate 14 diff --git a/salt/logrotate/soc_logrotate.yaml b/salt/logrotate/soc_logrotate.yaml index f407ab48d..f329ed623 100644 --- a/salt/logrotate/soc_logrotate.yaml +++ b/salt/logrotate/soc_logrotate.yaml @@ -91,6 +91,13 @@ logrotate: multiline: True global: True forcedType: "[]string" + "/opt/so/log/postgres/*_x_log": + description: List of logrotate options for this file. + title: /opt/so/log/postgres/*.log + advanced: True + multiline: True + global: True + forcedType: "[]string" "/opt/so/log/telegraf/*_x_log": description: List of logrotate options for this file. title: /opt/so/log/telegraf/*.log diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index f5b40310e..8def509e1 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -905,6 +905,30 @@ bootstrap_so_soc_database() { docker restart so-soc } +scrub_postgres_log_passwords() { + # Before 3.2 PostgreSQL could log the full CREATE/ALTER ROLE ... PASSWORD + # statement -- with the plaintext password -- to postgres.log whenever such a + # DDL errored, because log_min_error_statement defaulted to 'error'. The leak + # is fixed going forward (the provisioning scripts now SET log_min_error_statement + # = panic), but any already-logged secret persists in the on-disk log. Purge it + # here. Only the manager runs so-postgres, so this is a local scrub. + local log=/opt/so/log/postgres/postgres.log + [[ -f "$log" ]] || return 0 + if ! grep -qai "PASSWORD '" "$log" 2>/dev/null; then + echo "No leaked passwords found in $log." + return 0 + fi + echo "Removing leaked password statements from $log." + local tmp + tmp=$(mktemp) + # Drop every line carrying a PASSWORD literal, then rewrite in place with + # `cat >` so the file keeps its inode -- postgres holds it open via redirected + # stderr and must keep logging to the same file. + grep -avi "PASSWORD '" "$log" > "$tmp" 2>/dev/null || true + cat "$tmp" > "$log" + rm -f "$tmp" +} + # Existing grids should keep ILM unless an admin explicitly opts in to DLM. pin_elasticsearch_data_retention_method() { local elasticsearch_file=/opt/so/saltstack/local/pillar/elasticsearch/soc_elasticsearch.sls @@ -991,6 +1015,9 @@ post_to_3.2.0() { bootstrap_so_soc_database + # Purge any plaintext passwords a pre-3.2 postgres logged on DDL errors. + scrub_postgres_log_passwords + # Generate 9.3.7 elastic agent installers echo "Regenerating Elastic Agent Installers" /sbin/so-elastic-agent-gen-installers diff --git a/salt/postgres/files/init-db.sh b/salt/postgres/files/init-db.sh index d12bc4c9b..d1f1e375f 100644 --- a/salt/postgres/files/init-db.sh +++ b/salt/postgres/files/init-db.sh @@ -8,6 +8,10 @@ if [ -z "${SO_POSTGRES_PASS:-}" ] && [ -n "${SO_POSTGRES_PASS_FILE:-}" ] && [ -r SO_POSTGRES_PASS="$(< "$SO_POSTGRES_PASS_FILE")" fi psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + -- Shield the plaintext password below from postgres.log if this DDL errors: + -- log_min_error_statement defaults to 'error', which would append a STATEMENT line + -- containing the full CREATE/ALTER ROLE ... PASSWORD text. panic suppresses that. + SET log_min_error_statement = panic; DO \$\$ BEGIN IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${SO_POSTGRES_USER}') THEN diff --git a/salt/postgres/tools/sbin/so-telegraf-postgres b/salt/postgres/tools/sbin/so-telegraf-postgres index ef7c3f9e6..e6cc69fa6 100644 --- a/salt/postgres/tools/sbin/so-telegraf-postgres +++ b/salt/postgres/tools/sbin/so-telegraf-postgres @@ -71,6 +71,10 @@ EOSQL -v role_user="$ROLE_USER" \ -v role_pass="$ROLE_PASS" \ -U postgres -d so_telegraf <<'EOSQL' +-- Shield the plaintext password below from postgres.log if this DDL errors: +-- log_min_error_statement defaults to 'error', which would append a STATEMENT line +-- containing the full CREATE/ALTER ROLE ... PASSWORD text. panic suppresses that. +SET log_min_error_statement = panic; SELECT format( CASE WHEN EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'role_user') THEN 'ALTER ROLE %I WITH LOGIN PASSWORD %L' From 011749ad09d383879e212e7624424130601e01b1 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 20 Jul 2026 15:04:58 -0400 Subject: [PATCH 3/9] Self-heal PostgreSQL SOC database bootstrap init-db.sh only runs on a fresh PGDATA (docker-entrypoint-initdb.d), so a cluster left partially initialized -- e.g. the container restarted mid first-init by a watch trigger -- never recovered: the so_postgres role existed but its schema grants and the so_telegraf database were missing, breaking SOC with "permission denied for schema public". - init-db.sh: make the role upsert race-safe. Try CREATE ROLE and fall back to ALTER on duplicate_object/unique_violation instead of IF NOT EXISTS, so a concurrent creator no longer aborts the script (set -e + ON_ERROR_STOP=1) before the grants and so_telegraf creation run. The whole script is now idempotent and safe to re-run. - postgres/enabled.sls: move postgres_wait_ready here (was telegraf-gated in telegraf_users.sls) and add postgres_bootstrap_soc_db, which re-runs init-db.sh every highstate so a partially-initialized cluster self-heals. - telegraf_users.sls: drop its now-duplicate postgres_wait_ready definition; it resolves from postgres.enabled via the existing include. - soup: remove bootstrap_so_soc_database and its post_to_3.2.0 call. The highstates soup runs before postupgrade now reconcile the SOC DB, making the one-shot bootstrap redundant. --- salt/manager/tools/sbin/soup | 29 +++-------------------------- salt/postgres/enabled.sls | 20 ++++++++++++++++++++ salt/postgres/files/init-db.sh | 14 ++++++++++---- salt/postgres/telegraf_users.sls | 15 ++++----------- 4 files changed, 37 insertions(+), 41 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 8def509e1..eb63a1d60 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -880,31 +880,6 @@ recollate_postgres() { echo "" } -bootstrap_so_soc_database() { - # init-db.sh is mounted into so-postgres at /docker-entrypoint-initdb.d/init-db.sh - # and runs automatically only on a fresh data directory. Hosts upgrading from - # 3.1.0 already have /nsm/postgres populated, so the so_soc bootstrap block - # added in 3.2 never fires. Re-run the script explicitly; it's idempotent. - echo "Bootstrapping database via init-db.sh." - # The postgres image has no USER directive, so `docker exec` defaults to - # root, and the container env intentionally omits POSTGRES_USER (the upstream - # entrypoint defaults it transiently during first-init only). Recreate both - # so psql inside init-db.sh resolves the connect user correctly. - local exec_cmd="docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh" - if ! /usr/sbin/so-postgres-wait; then - FINAL_MESSAGE_QUEUE+=("WARNING: so-postgres was not ready during the 3.2.0 upgrade; the so_soc database may not have been bootstrapped. Re-run manually: $exec_cmd") - return 0 - fi - if ! $exec_cmd; then - FINAL_MESSAGE_QUEUE+=("WARNING: init-db.sh failed inside so-postgres during the 3.2.0 upgrade; the database may not have been bootstrapped. Re-run manually: $exec_cmd") - return 0 - fi - echo "Database bootstrap complete." - - echo "Restarting so-soc container to pick up database changes" - docker restart so-soc -} - scrub_postgres_log_passwords() { # Before 3.2 PostgreSQL could log the full CREATE/ALTER ROLE ... PASSWORD # statement -- with the plaintext password -- to postgres.log whenever such a @@ -1013,7 +988,9 @@ post_to_3.2.0() { # Recollate due to image OS rebase recollate_postgres - bootstrap_so_soc_database + # The SOC database (so_postgres role, schema/db grants, so_telegraf) is + # bootstrapped idempotently by the postgres.enabled highstate that runs + # before this postupgrade step, so no explicit bootstrap call is needed here. # Purge any plaintext passwords a pre-3.2 postgres logged on DDL errors. scrub_postgres_log_passwords diff --git a/salt/postgres/enabled.sls b/salt/postgres/enabled.sls index b0eadd205..ad6ff7819 100644 --- a/salt/postgres/enabled.sls +++ b/salt/postgres/enabled.sls @@ -85,6 +85,26 @@ so-postgres: - x509: postgres_crt - x509: postgres_key +postgres_wait_ready: + cmd.run: + - name: /usr/sbin/so-postgres-wait + - require: + - docker_container: so-postgres + - file: postgres_sbin + +# Re-run the SOC database bootstrap on every highstate so a cluster left +# partially initialized -- e.g. the container was restarted mid first-init by a +# watch trigger, so docker-entrypoint-initdb.d never completed -- self-heals: +# the so_postgres role, its schema/database grants, and the so_telegraf database +# are all reconciled idempotently. init-db.sh is idempotent and race-safe. +# POSTGRES_USER is injected because the container env intentionally omits it. +postgres_bootstrap_soc_db: + cmd.run: + - name: docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh + - require: + - cmd: postgres_wait_ready + - file: postgresinitdb + delete_so-postgres_so-status.disabled: file.uncomment: - name: /opt/so/conf/so-status/so-status.conf diff --git a/salt/postgres/files/init-db.sh b/salt/postgres/files/init-db.sh index d1f1e375f..540980469 100644 --- a/salt/postgres/files/init-db.sh +++ b/salt/postgres/files/init-db.sh @@ -12,13 +12,19 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-E -- log_min_error_statement defaults to 'error', which would append a STATEMENT line -- containing the full CREATE/ALTER ROLE ... PASSWORD text. panic suppresses that. SET log_min_error_statement = panic; + -- Race-safe upsert: try CREATE, and if the role was created concurrently + -- (another session re-entering init) fall back to ALTER. Catching the + -- exception -- rather than an IF NOT EXISTS check -- avoids a TOCTOU window + -- where CREATE ROLE would abort the whole script with a duplicate-key error + -- and skip the grants below. Both SQLSTATEs are covered: duplicate_object + -- (role already exists) and unique_violation (racy pg_authid index insert). DO \$\$ BEGIN - IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${SO_POSTGRES_USER}') THEN + BEGIN EXECUTE format('CREATE ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}'); - ELSE - EXECUTE format('ALTER ROLE %I WITH PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}'); - END IF; + EXCEPTION WHEN duplicate_object OR unique_violation THEN + EXECUTE format('ALTER ROLE %I WITH LOGIN PASSWORD %L', '${SO_POSTGRES_USER}', '${SO_POSTGRES_PASS}'); + END; END \$\$; GRANT ALL ON SCHEMA public TO "$SO_POSTGRES_USER"; diff --git a/salt/postgres/telegraf_users.sls b/salt/postgres/telegraf_users.sls index 5e3566a95..62c33b892 100644 --- a/salt/postgres/telegraf_users.sls +++ b/salt/postgres/telegraf_users.sls @@ -8,23 +8,16 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'telegraf/map.jinja' import TELEGRAFMERGED %} -{# postgres_wait_ready below requires `docker_container: so-postgres`, which is - declared in postgres.enabled. Include it here so state.apply postgres.telegraf_users - on its own (e.g. from orch.deploy_newnode) still has that ID in scope. Salt - de-duplicates the circular include. #} +{# postgres_wait_ready and the so-postgres container are declared in + postgres.enabled; include it so the require references below resolve and so + state.apply postgres.telegraf_users on its own (e.g. from orch.deploy_newnode) + still has those IDs in scope. Salt de-duplicates the circular include. #} include: - postgres.enabled {% set TG_OUT = TELEGRAFMERGED.output | upper %} {% if TG_OUT in ['POSTGRES', 'BOTH'] %} -postgres_wait_ready: - cmd.run: - - name: /usr/sbin/so-postgres-wait - - require: - - docker_container: so-postgres - - file: postgres_sbin - # Ensure the shared Telegraf database exists. init-db.sh only runs on a # fresh data dir, so hosts upgraded onto an existing /nsm/postgres volume # would otherwise never get so_telegraf. From 387781c6292c96aec01b3f0a66b103a9a29d9057 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 20 Jul 2026 15:19:02 -0400 Subject: [PATCH 4/9] Trim verbose comments in postgres provisioning changes --- salt/manager/tools/sbin/soup | 17 +++-------------- salt/postgres/enabled.sls | 9 +++------ salt/postgres/files/init-db.sh | 12 +++--------- salt/postgres/telegraf_users.sls | 6 ++---- salt/postgres/tools/sbin/so-telegraf-postgres | 4 +--- 5 files changed, 12 insertions(+), 36 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index eb63a1d60..9f02eb32a 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -881,12 +881,7 @@ recollate_postgres() { } scrub_postgres_log_passwords() { - # Before 3.2 PostgreSQL could log the full CREATE/ALTER ROLE ... PASSWORD - # statement -- with the plaintext password -- to postgres.log whenever such a - # DDL errored, because log_min_error_statement defaulted to 'error'. The leak - # is fixed going forward (the provisioning scripts now SET log_min_error_statement - # = panic), but any already-logged secret persists in the on-disk log. Purge it - # here. Only the manager runs so-postgres, so this is a local scrub. + # Purge plaintext passwords a pre-3.2 postgres could log on DDL errors. local log=/opt/so/log/postgres/postgres.log [[ -f "$log" ]] || return 0 if ! grep -qai "PASSWORD '" "$log" 2>/dev/null; then @@ -896,9 +891,7 @@ scrub_postgres_log_passwords() { echo "Removing leaked password statements from $log." local tmp tmp=$(mktemp) - # Drop every line carrying a PASSWORD literal, then rewrite in place with - # `cat >` so the file keeps its inode -- postgres holds it open via redirected - # stderr and must keep logging to the same file. + # Rewrite in place (cat >) to keep the inode postgres is writing to. grep -avi "PASSWORD '" "$log" > "$tmp" 2>/dev/null || true cat "$tmp" > "$log" rm -f "$tmp" @@ -988,11 +981,7 @@ post_to_3.2.0() { # Recollate due to image OS rebase recollate_postgres - # The SOC database (so_postgres role, schema/db grants, so_telegraf) is - # bootstrapped idempotently by the postgres.enabled highstate that runs - # before this postupgrade step, so no explicit bootstrap call is needed here. - - # Purge any plaintext passwords a pre-3.2 postgres logged on DDL errors. + # SOC database bootstrap is handled by the postgres.enabled highstate. scrub_postgres_log_passwords # Generate 9.3.7 elastic agent installers diff --git a/salt/postgres/enabled.sls b/salt/postgres/enabled.sls index ad6ff7819..fa91d146a 100644 --- a/salt/postgres/enabled.sls +++ b/salt/postgres/enabled.sls @@ -92,12 +92,9 @@ postgres_wait_ready: - docker_container: so-postgres - file: postgres_sbin -# Re-run the SOC database bootstrap on every highstate so a cluster left -# partially initialized -- e.g. the container was restarted mid first-init by a -# watch trigger, so docker-entrypoint-initdb.d never completed -- self-heals: -# the so_postgres role, its schema/database grants, and the so_telegraf database -# are all reconciled idempotently. init-db.sh is idempotent and race-safe. -# POSTGRES_USER is injected because the container env intentionally omits it. +# Reconcile the SOC database (role, grants, so_telegraf) every highstate so a +# partially-initialized cluster self-heals. POSTGRES_USER is injected because +# the container env omits it. postgres_bootstrap_soc_db: cmd.run: - name: docker exec -u postgres -e POSTGRES_USER=postgres so-postgres bash /docker-entrypoint-initdb.d/init-db.sh diff --git a/salt/postgres/files/init-db.sh b/salt/postgres/files/init-db.sh index 540980469..4d65b0c97 100644 --- a/salt/postgres/files/init-db.sh +++ b/salt/postgres/files/init-db.sh @@ -8,16 +8,10 @@ if [ -z "${SO_POSTGRES_PASS:-}" ] && [ -n "${SO_POSTGRES_PASS_FILE:-}" ] && [ -r SO_POSTGRES_PASS="$(< "$SO_POSTGRES_PASS_FILE")" fi psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - -- Shield the plaintext password below from postgres.log if this DDL errors: - -- log_min_error_statement defaults to 'error', which would append a STATEMENT line - -- containing the full CREATE/ALTER ROLE ... PASSWORD text. panic suppresses that. + -- Keep the password out of postgres.log if this DDL errors. SET log_min_error_statement = panic; - -- Race-safe upsert: try CREATE, and if the role was created concurrently - -- (another session re-entering init) fall back to ALTER. Catching the - -- exception -- rather than an IF NOT EXISTS check -- avoids a TOCTOU window - -- where CREATE ROLE would abort the whole script with a duplicate-key error - -- and skip the grants below. Both SQLSTATEs are covered: duplicate_object - -- (role already exists) and unique_violation (racy pg_authid index insert). + -- Idempotent, race-safe upsert: CREATE, falling back to ALTER if the role + -- already exists or is created concurrently. DO \$\$ BEGIN BEGIN diff --git a/salt/postgres/telegraf_users.sls b/salt/postgres/telegraf_users.sls index 62c33b892..4c63d40b0 100644 --- a/salt/postgres/telegraf_users.sls +++ b/salt/postgres/telegraf_users.sls @@ -8,10 +8,8 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'telegraf/map.jinja' import TELEGRAFMERGED %} -{# postgres_wait_ready and the so-postgres container are declared in - postgres.enabled; include it so the require references below resolve and so - state.apply postgres.telegraf_users on its own (e.g. from orch.deploy_newnode) - still has those IDs in scope. Salt de-duplicates the circular include. #} +{# postgres.enabled declares the so-postgres container and postgres_wait_ready + that the requires below reference. Salt de-duplicates the circular include. #} include: - postgres.enabled diff --git a/salt/postgres/tools/sbin/so-telegraf-postgres b/salt/postgres/tools/sbin/so-telegraf-postgres index e6cc69fa6..17e4da744 100644 --- a/salt/postgres/tools/sbin/so-telegraf-postgres +++ b/salt/postgres/tools/sbin/so-telegraf-postgres @@ -71,9 +71,7 @@ EOSQL -v role_user="$ROLE_USER" \ -v role_pass="$ROLE_PASS" \ -U postgres -d so_telegraf <<'EOSQL' --- Shield the plaintext password below from postgres.log if this DDL errors: --- log_min_error_statement defaults to 'error', which would append a STATEMENT line --- containing the full CREATE/ALTER ROLE ... PASSWORD text. panic suppresses that. +-- Keep the password out of postgres.log if this DDL errors. SET log_min_error_statement = panic; SELECT format( CASE WHEN EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = :'role_user') From c4c1464b2a8b17a7652d1a70ae83a7abe72c6293 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 21 Jul 2026 10:07:33 -0400 Subject: [PATCH 5/9] Better support SigmaHQ rules --- .../files/soc/playbook_placeholder_map.yaml | 15 +- .../files/soc/sigma_playbook_pipeline.yaml | 123 ++++++++++- salt/soc/files/soc/sigma_so_pipeline.yaml | 209 +++++++++++++++++- 3 files changed, 339 insertions(+), 8 deletions(-) diff --git a/salt/soc/files/soc/playbook_placeholder_map.yaml b/salt/soc/files/soc/playbook_placeholder_map.yaml index eeddb4474..aac5a12ac 100644 --- a/salt/soc/files/soc/playbook_placeholder_map.yaml +++ b/salt/soc/files/soc/playbook_placeholder_map.yaml @@ -14,18 +14,22 @@ CommandLine: process.command_line CurrentDirectory: process.working_directory +Details: registry.data.strings Image: process.executable -ImageLoaded: dll.name +ImageLoaded: dll.path ParentImage: process.parent.executable ParentName: process.parent.name ParentProcessGuid: process.parent.entity_id ProcessGuid: process.entity_id -TargetFilename: file.name +TargetFilename: file.path TargetObject: registry.path TargetUserName: user.target.name User: user.name community_id: network.community_id dns_resolved_ip: dns.resolved_ip +QueryName: dns.question.name +dll_basename: dll.name +file_basename: file.name document_id: soc_id dst_ip: destination.ip dst_port: destination.port @@ -40,10 +44,15 @@ public_ip: network.public_ip related_hosts: related.hosts related_ip: related.ip src_ip: source.ip -dns_query_name: dns.query_name +dns_query_name: dns.question.name flow_id: log.id.uid payload: network.data.decoded rule_category: rule.category rule_name: rule.name rule_uuid: rule.uuid src_port: source.port +# PowerShell channel (ps_script EID 4104 / classic EID 400) +ScriptBlockText: powershell.file.script_block_text +script_block_id: powershell.file.script_block_id +script_block_hash: powershell.file.script_block_hash +EngineVersion: powershell.engine.version diff --git a/salt/soc/files/soc/sigma_playbook_pipeline.yaml b/salt/soc/files/soc/sigma_playbook_pipeline.yaml index 6fcdedbb4..04bd2ffaf 100644 --- a/salt/soc/files/soc/sigma_playbook_pipeline.yaml +++ b/salt/soc/files/soc/sigma_playbook_pipeline.yaml @@ -1,8 +1,9 @@ name: Security Onion - Playbook Pipeline priority: 97 transformations: - # Route string fields to their lowercase-normalized .caseless subfield so wildcard - # matches are case-insensitive. + # Route to lowercase-normalized .caseless subfields for case-insensitive matching. + # file.path.caseless exists on Defend only (Sysmon file events lack it); + # registry.path / dll.path / file.name have no .caseless on any source. - id: case_insensitive_string_fields type: field_name_mapping mapping: @@ -10,3 +11,121 @@ transformations: process.parent.executable: process.parent.executable.caseless process.command_line: process.command_line.caseless process.parent.command_line: process.parent.command_line.caseless + file.path: file.path.caseless + # file_activity: playbook-only pseudo-category spanning all file operations. + - id: playbook_file_activity_add-fields + type: add_condition + conditions: + event.category: 'file' + rule_conditions: + - type: logsource + category: file_activity + # EventType -> event.action is safe as a projection only — the values differ + # (Sysmon DeleteFile/SetValue vs ECS deletion/modification). + - id: playbook_file_activity_event_type + type: field_name_mapping + mapping: + winlog.event_data.EventType: event.action + rule_conditions: + - type: logsource + category: file_activity + - id: playbook_file_event_event_type + type: field_name_mapping + mapping: + winlog.event_data.EventType: event.action + rule_conditions: + - type: logsource + category: file_event + - id: playbook_file_delete_event_type + type: field_name_mapping + mapping: + winlog.event_data.EventType: event.action + rule_conditions: + - type: logsource + category: file_delete + - id: playbook_file_rename_event_type + type: field_name_mapping + mapping: + winlog.event_data.EventType: event.action + rule_conditions: + - type: logsource + category: file_rename + - id: playbook_registry_set_event_type + type: field_name_mapping + mapping: + winlog.event_data.EventType: event.action + rule_conditions: + - type: logsource + category: registry_set + # WMI-Activity events carry no event.category; event.dataset is source-specific + - id: playbook_wmi_activity_add-fields + type: add_condition + conditions: + event.dataset: 'wmi.operational' + rule_conditions: + - type: logsource + category: wmi + - id: playbook_wmi_event_activity_add-fields + type: add_condition + conditions: + event.dataset: 'wmi.operational' + rule_conditions: + - type: logsource + category: wmi_event + - id: playbook_ps_script_field-mapping + type: field_name_mapping + mapping: + ScriptBlockText: powershell.file.script_block_text + Path: file.path + rule_conditions: + - type: logsource + category: ps_script + - id: playbook_ps_script_add-fields + type: add_condition + conditions: + event.dataset: 'windows.powershell_operational' + event.code: '4104' + rule_conditions: + - type: logsource + category: ps_script + # Data (raw EID 400/600 message blob) -> process.command_line: the winlog + # integration parses HostApplication into it; `message` is analyzed text and + # wildcard matches on it silently return zero. No .caseless on this dataset. + - id: playbook_ps_classic_start_field-mapping + type: field_name_mapping + mapping: + Data: process.command_line + rule_conditions: + - type: logsource + category: ps_classic_start + - id: playbook_ps_classic_start_add-fields + type: add_condition + conditions: + event.dataset: 'windows.powershell' + event.code: '400' + rule_conditions: + - type: logsource + category: ps_classic_start + - id: playbook_ps_classic_provider_start_field-mapping + type: field_name_mapping + mapping: + Data: process.command_line + rule_conditions: + - type: logsource + category: ps_classic_provider_start + - id: playbook_ps_classic_provider_start_add-fields + type: add_condition + conditions: + event.dataset: 'windows.powershell' + event.code: '600' + rule_conditions: + - type: logsource + category: ps_classic_provider_start + # Alert docs nest the host under event_data.host.name; no top-level host.name. + - id: playbook_alert_host_field + type: field_name_mapping + mapping: + host.name: event_data.host.name + rule_conditions: + - type: logsource + category: alert diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index dee93c84e..281acb8a9 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -131,13 +131,22 @@ transformations: type: field_name_mapping mapping: winlog.event_data.Details: registry.data.strings - # field rename only; EventType values (SetValue/CreateKey) still differ from - # event.action values (modification/creation) - winlog.event_data.EventType: event.action rule_conditions: - type: logsource product: windows category: registry_set + # ecs_windows renames Initiated -> network.direction without translating the value, + # so rules compile to network.direction:"true" and never match (field holds + # ingress/egress). network.initiated carries the boolean on both Defend and Sysmon. + # Keyed on network.direction because ecs_windows has already renamed by this layer. + - id: ecs_fix_network_connection_initiated + type: field_name_mapping + mapping: + network.direction: network.initiated + rule_conditions: + - type: logsource + product: windows + category: network_connection - id: ecs_fix_image_load type: field_name_mapping mapping: @@ -150,6 +159,30 @@ transformations: - type: logsource product: windows category: image_load + # Defend reports the driver image in dll.path (file.path is null on driver events) — + # same renames as image_load. + - id: ecs_fix_driver_load + type: field_name_mapping + mapping: + file.path: dll.path + file.code_signature.signed: dll.code_signature.exists + winlog.event_data.Signature: dll.code_signature.subject_name + file.code_signature.status: dll.code_signature.status + winlog.event_data.Hashes: dll.hash.sha256 + rule_conditions: + - type: logsource + product: windows + category: driver_load + - id: ecs_fix_file_rename + type: field_name_mapping + mapping: + # Defend records the pre-rename path in file.Ext.original.path; Sysmon's + # SourceFilename has no ECS equivalent. + winlog.event_data.SourceFilename: file.Ext.original.path + rule_conditions: + - type: logsource + product: windows + category: file_rename - id: linux_security_add-fields type: add_condition conditions: @@ -323,6 +356,33 @@ transformations: rule_conditions: - type: logsource category: file_event + # Without event.type scoping, file_delete rules also match creations. + - id: endpoint_file_delete_add-fields + type: add_condition + conditions: + event.category: 'file' + event.type: 'deletion' + rule_conditions: + - type: logsource + category: file_delete + # Defend reports renames as event.action:rename with event.type:change. + - id: endpoint_file_rename_add-fields + type: add_condition + conditions: + event.category: 'file' + event.action: 'rename' + rule_conditions: + - type: logsource + category: file_rename + # event.type:change selects writes and excludes Defend's read-only registry events. + - id: endpoint_registry_set_add-fields + type: add_condition + conditions: + event.category: 'registry' + event.type: 'change' + rule_conditions: + - type: logsource + category: registry_set # Scope image_load rules to Elastic Endpoint library events (event.category:library, dll.* # populated). - id: endpoint_image_load_add-fields @@ -351,6 +411,149 @@ transformations: rule_conditions: - type: logsource category: network_connection + # Scope on lookup actions, not network.protocol:dns (also matches Zeek port-53). + - id: endpoint_dns_query_add-fields + type: add_condition + conditions: + event.category: 'network' + event.action: + - 'lookup_requested' + - 'lookup_result' + rule_conditions: + - type: logsource + category: dns_query + # OS gates: without them a `product: windows` rule in these categories also matches + # Linux/macOS. Separate conditions (not `product:` on the mappings above) so the + # mappings keep scoping product-less rules. + - id: endpoint_file_create_windows_os-gate + type: add_condition + conditions: + host.os.type: 'windows' + rule_conditions: + - type: logsource + category: file_event + product: windows + - id: endpoint_file_create_linux_os-gate + type: add_condition + conditions: + host.os.type: 'linux' + rule_conditions: + - type: logsource + category: file_event + product: linux + - id: endpoint_file_create_macos_os-gate + type: add_condition + conditions: + host.os.type: 'macos' + rule_conditions: + - type: logsource + category: file_event + product: macos + - id: endpoint_file_delete_windows_os-gate + type: add_condition + conditions: + host.os.type: 'windows' + rule_conditions: + - type: logsource + category: file_delete + product: windows + - id: endpoint_file_delete_linux_os-gate + type: add_condition + conditions: + host.os.type: 'linux' + rule_conditions: + - type: logsource + category: file_delete + product: linux + - id: endpoint_file_delete_macos_os-gate + type: add_condition + conditions: + host.os.type: 'macos' + rule_conditions: + - type: logsource + category: file_delete + product: macos + - id: endpoint_file_rename_windows_os-gate + type: add_condition + conditions: + host.os.type: 'windows' + rule_conditions: + - type: logsource + category: file_rename + product: windows + - id: endpoint_file_rename_linux_os-gate + type: add_condition + conditions: + host.os.type: 'linux' + rule_conditions: + - type: logsource + category: file_rename + product: linux + - id: endpoint_file_rename_macos_os-gate + type: add_condition + conditions: + host.os.type: 'macos' + rule_conditions: + - type: logsource + category: file_rename + product: macos + - id: endpoint_network_connection_windows_os-gate + type: add_condition + conditions: + host.os.type: 'windows' + rule_conditions: + - type: logsource + category: network_connection + product: windows + - id: endpoint_network_connection_linux_os-gate + type: add_condition + conditions: + host.os.type: 'linux' + rule_conditions: + - type: logsource + category: network_connection + product: linux + - id: endpoint_network_connection_macos_os-gate + type: add_condition + conditions: + host.os.type: 'macos' + rule_conditions: + - type: logsource + category: network_connection + product: macos + - id: endpoint_dns_query_windows_os-gate + type: add_condition + conditions: + host.os.type: 'windows' + rule_conditions: + - type: logsource + category: dns_query + product: windows + - id: endpoint_dns_query_linux_os-gate + type: add_condition + conditions: + host.os.type: 'linux' + rule_conditions: + - type: logsource + category: dns_query + product: linux + - id: endpoint_dns_query_macos_os-gate + type: add_condition + conditions: + host.os.type: 'macos' + rule_conditions: + - type: logsource + category: dns_query + product: macos + # registry_set / image_load / driver_load are not gated — windows-only telemetry. + # wmi / wmi_event are not scoped here: their only handle is event.dataset + - id: endpoint_driver_load_add-fields + type: add_condition + conditions: + event.category: 'driver' + rule_conditions: + - type: logsource + category: driver_load # Maps "alert" category to SO Alert events - id: alert_so_add-fields type: add_condition From 4e1935f8a0bad057dee6a15c7360ed7d7e3ef958 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 21 Jul 2026 11:58:11 -0400 Subject: [PATCH 6/9] postgress updates --- salt/common/tools/sbin/so-log-check | 1 + salt/soc/defaults.yaml | 2 +- salt/soc/merged.map.jinja | 2 +- salt/soc/soc_soc.yaml | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 42ceea47b..56c0278da 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -133,6 +133,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Cancelling deferred write event maybeFenceReplicas because the event queue is now closed" # Kafka controller log during shutdown/restart EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Redis may have been restarted" # Redis likely restarted by salt EXCLUDED_ERRORS="$EXCLUDED_ERRORS|file already closed" # Go logging race condition during container restart + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|relation \"audit_settings\" does not exist" # salt checking for changes before SOC starts fi if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 9705c1863..e04b966c0 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1539,7 +1539,7 @@ soc: postgres: host: "" port: 5432 - sslMode: "allow" + sslMode: "require" database: securityonion user: "" password: "" diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index a421711e2..452fba0b9 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -88,7 +88,7 @@ 'host': GLOBALS.manager_ip, 'password': PG_PASS, 'port': 5432, - 'sslMode': 'allow', + 'sslMode': 'require', 'user': PG_USER, } }) %} diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index f09317f39..eb4cd3154 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -486,7 +486,7 @@ soc: global: True advanced: True sslMode: - description: "Use encrypted connections to the PostgreSQL server. Must be one of the following values: disable, allow, prefer, require, verify-ca, verify-full. Defaults to allow." + description: "Use encrypted connections to the PostgreSQL server. Must be one of the following values: disable, allow, prefer, require, verify-ca, verify-full." global: True advanced: True database: From 894d323323fe3cad8204721f054d1fd26d4d74bd Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 21 Jul 2026 15:08:24 -0400 Subject: [PATCH 7/9] Map default agents to model displayName, not id agentMapping values are model displayNames (the canonical selector); the stock config used the model id, which only resolved via the legacy id@adapter fallback. Use the Claude Sonnet displayName so agent-to-model resolution matches the documented contract. --- salt/soc/defaults.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 9705c1863..c0e01f80d 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1530,9 +1530,9 @@ soc: healthTimeoutSeconds: 5 agentic: false agentMapping: - Orchestrator: sonnet - Investigator: sonnet - Detection Engineer: sonnet + Orchestrator: Claude Sonnet + Investigator: Claude Sonnet + Detection Engineer: Claude Sonnet onionconfig: saltstackDir: /opt/so/saltstack bypassEnabled: false From ac46636196387378d30d15b2bd7844500c2944ca Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 21 Jul 2026 16:05:18 -0400 Subject: [PATCH 8/9] ensure salt-master service restarted last --- salt/salt/master.sls | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/salt/master.sls b/salt/salt/master.sls index 2c47a95c6..12ad0e344 100644 --- a/salt/salt/master.sls +++ b/salt/salt/master.sls @@ -94,7 +94,9 @@ salt_master_service: - file: checkmine_engine - file: pillarWatch_engine - file: engines_config - - order: 9002 + - require: + - cmd: wait_for_salt_minion_ready + - order: last {% else %} From f77fad80871b504649ca7d168743982a5424b8e1 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Tue, 21 Jul 2026 16:29:24 -0400 Subject: [PATCH 9/9] apply salt.master state instead of just salt.minion --- salt/manager/tools/sbin/soup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 9f02eb32a..174d6fd7b 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1998,11 +1998,11 @@ main() { # Testing that salt-master is up by checking that is it connected to itself check_saltmaster_status - # update the salt-minion configs here and start the minion + # update the salt-master and salt-minion configs here and start the minion # since highstate are disabled above, minion start should not trigger a highstate echo "" - echo "Ensuring salt-minion configs are up-to-date." - salt-call state.apply salt.minion -l info queue=True + echo "Ensuring salt-master and salt-minion configs are up-to-date." + salt-call state.apply salt.master -l info queue=True echo "" # ensure the mine is updated and populated before highstates run, following the salt-master restart