From b55cb257b67558a57f74c672aa913cf9a5860919 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 19 May 2025 13:25:27 -0400 Subject: [PATCH 1/5] Add parsing for Playbook --- salt/elasticsearch/files/ingest/global@custom | 2 + .../elasticsearch/files/ingest/suricata.alert | 1 + .../files/ingest/suricata.common | 7 ++ salt/soc/files/soc/sigma_so_pipeline.yaml | 92 ++++++++++--------- salt/zeek/defaults.yaml | 1 + .../securityonion/community-id-extended.zeek | 40 ++++++++ 6 files changed, 102 insertions(+), 41 deletions(-) create mode 100644 salt/zeek/policy/securityonion/community-id-extended.zeek diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom index 828ee0a03..85154b908 100644 --- a/salt/elasticsearch/files/ingest/global@custom +++ b/salt/elasticsearch/files/ingest/global@custom @@ -24,6 +24,8 @@ { "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } }, { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, + {"append": {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network'","ignore_failure":true}}, + {"foreach": {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"ignore_failure":true}}, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } } ] } diff --git a/salt/elasticsearch/files/ingest/suricata.alert b/salt/elasticsearch/files/ingest/suricata.alert index 71e346728..3d0241e48 100644 --- a/salt/elasticsearch/files/ingest/suricata.alert +++ b/salt/elasticsearch/files/ingest/suricata.alert @@ -9,6 +9,7 @@ { "rename":{ "field": "rule.signature_id", "target_field": "rule.uuid", "ignore_failure": true } }, { "rename":{ "field": "rule.signature_id", "target_field": "rule.signature", "ignore_failure": true } }, { "rename":{ "field": "message2.payload_printable", "target_field": "network.data.decoded", "ignore_failure": true } }, + { "dissect": { "field": "rule.rule", "pattern": "%{?prefix}content:\"%{dns.query_name}\"%{?remainder}", "ignore_missing": true, "ignore_failure": true } }, { "pipeline": { "name": "common.nids" } } ] } \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/suricata.common b/salt/elasticsearch/files/ingest/suricata.common index 8143882c7..102b5dac8 100644 --- a/salt/elasticsearch/files/ingest/suricata.common +++ b/salt/elasticsearch/files/ingest/suricata.common @@ -18,6 +18,13 @@ { "set": { "field": "event.ingested", "value": "{{@timestamp}}" } }, { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } }, { "remove":{ "field": "agent", "ignore_failure": true } }, + {"append":{"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"ignore_failure":true}}, + { + "script": { + "source": "boolean isPrivate(def ip) { if (ip == null) return false; int dot1 = ip.indexOf('.'); if (dot1 == -1) return false; int dot2 = ip.indexOf('.', dot1 + 1); if (dot2 == -1) return false; int first = Integer.parseInt(ip.substring(0, dot1)); if (first == 10) return true; if (first == 192 && ip.startsWith('168.', dot1 + 1)) return true; if (first == 172) { int second = Integer.parseInt(ip.substring(dot1 + 1, dot2)); return second >= 16 && second <= 31; } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx[field]?.ip; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); if (isPrivate(ip)) { if (ctx.network.private_ip == null) ctx.network.private_ip = new ArrayList(); if (!ctx.network.private_ip.contains(ip)) ctx.network.private_ip.add(ip); } else { if (ctx.network.public_ip == null) ctx.network.public_ip = new ArrayList(); if (!ctx.network.public_ip.contains(ip)) ctx.network.public_ip.add(ip); } } }", + "ignore_failure": false + } + }, { "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}" } } ] } diff --git a/salt/soc/files/soc/sigma_so_pipeline.yaml b/salt/soc/files/soc/sigma_so_pipeline.yaml index 88abcc200..5cee0cfd3 100644 --- a/salt/soc/files/soc/sigma_so_pipeline.yaml +++ b/salt/soc/files/soc/sigma_so_pipeline.yaml @@ -1,45 +1,6 @@ name: Security Onion Baseline Pipeline priority: 90 transformations: -vars: - document_id: - - '{soc_id}' - hostname: - - '{event_data.host.name}' - ProcessGuid: - - '{event_data.process.entity_id}' - User: - - '{user.name}' - private_ip: - - '{network.private.ip}' - public_ip: - - '{network.public.ip}' - related_ip: - - '{event_data.related.ip}' - related.hosts: - - '{event_data.related.hosts' - CurrentDirectory: - - '{event_data.process.working_directory}' - ParentProcessGuid: - - '{ParentProcessGuid}' - Image: - - '{process.executable}' - community_id: - - '{network.community_id}' -transformations: - - type: value_placeholders - include: - - 'community_id' - - 'document_id' - - 'ProcessGuid' - - 'hostname' - - 'User' - - 'CurrentDirectory' - - 'ParentProcessGuid' - - 'Image' - - 'related_ip' - - 'private_ip' - - 'public_ip' - id: baseline_field_name_mapping type: field_name_mapping mapping: @@ -64,14 +25,17 @@ transformations: CommandLine: process.command_line CurrentDirectory: process.working_directory ParentProcessGuid: process.parent.entity_id - ParentProcessId: process.parent.pid" + ParentProcessId: process.parent.pid ParentImage: process.parent.executable ParentCommandLine: process.parent.command_line + User: user.name ## End Temp Linux Mappings ## + document_id: _id rule.type: event.module related_ip: related.ip community_id: network.community_id event_dataset: event.dataset + hostname: host.name # Maps "opencanary" product to SO IDH logs - id: opencanary_idh_add-fields type: add_condition @@ -181,7 +145,15 @@ transformations: rule_conditions: - type: logsource category: file_event - category: file_event + # Maps network rules to all network logs + # This targets all network logs, all services, generated from endpoints and network + - id: network_add-fields + type: add_condition + conditions: + event.category: 'network' + rule_conditions: + - type: logsource + category: network # Maps network_connection rules to endpoint network creation logs # This is an OS-agnostic mapping, to account for logs that don't specify source OS - id: endpoint_network_connection_add-fields @@ -218,3 +190,41 @@ transformations: - type: logsource category: network service: dns + # Maps "network + file" to SO file logs + - id: network_file_so_add-fields + type: add_condition + conditions: + event.category: 'network' + tags: 'file' + rule_conditions: + - type: logsource + category: network + service: file + # Maps "network + x509" to SO x509 logs + - id: network_x509_so_add-fields + type: add_condition + conditions: + event.category: 'network' + tags: 'x509' + rule_conditions: + - type: logsource + category: network + service: x509 + # Maps "network + ssl" to SO ssl logs + - id: network_ssl_so_add-fields + type: add_condition + conditions: + event.category: 'network' + tags: 'ssl' + rule_conditions: + - type: logsource + category: network + service: ssl + # Maps file to host or network file events + - id: file_so_add-fields + type: add_condition + conditions: + tags: '*file' + rule_conditions: + - type: logsource + category: file \ No newline at end of file diff --git a/salt/zeek/defaults.yaml b/salt/zeek/defaults.yaml index d41ead0e8..1daf77102 100644 --- a/salt/zeek/defaults.yaml +++ b/salt/zeek/defaults.yaml @@ -57,6 +57,7 @@ zeek: - cve-2020-0601 - securityonion/bpfconf - securityonion/file-extraction + - securityonion/community-id-extended - oui-logging - icsnpp-modbus - icsnpp-dnp3 diff --git a/salt/zeek/policy/securityonion/community-id-extended.zeek b/salt/zeek/policy/securityonion/community-id-extended.zeek new file mode 100644 index 000000000..e8df10066 --- /dev/null +++ b/salt/zeek/policy/securityonion/community-id-extended.zeek @@ -0,0 +1,40 @@ +##! Extends community ID logging to Files, and SSL by copying +##! the community_id from the parent connection. +##! +##! Note: Requires that protocols/conn/community-id-logging is loaded + +module CommunityIDExt; + +@load base/protocols/ssl +@load protocols/conn/community-id-logging + +export { + redef record SSL::Info += { + community_id: string &optional &log; + }; + + redef record Files::Info += { + community_id: string &optional &log; + }; +} + +# Files +event file_new(f: fa_file) { + if ( f?$conns ) { + # Take community_id from first connection that has it + for ( cid in f$conns ) { + local c = f$conns[cid]; + if ( c?$conn && c$conn?$community_id ) { + f$info$community_id = c$conn$community_id; + break; + } + } + } +} + +# SSL Connections +event ssl_established(c: connection) { + if ( c?$conn && c$conn?$community_id && c?$ssl ) { + c$ssl$community_id = c$conn$community_id; + } +} From 58f4db95ea1505c6a0517e9fcff21093ff447595 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Mon, 19 May 2025 15:31:50 -0400 Subject: [PATCH 2/5] Create playbooks dir --- salt/soc/config.sls | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/soc/config.sls b/salt/soc/config.sls index e19e3eb14..78a495e0a 100644 --- a/salt/soc/config.sls +++ b/salt/soc/config.sls @@ -52,6 +52,13 @@ socsaltdir: - mode: 770 - makedirs: True +socplaybooksdir: + file.directory: + - name: /opt/so/conf/soc/playbooks + - user: 939 + - group: 939 + - makedirs: True + socanalytics: file.managed: - name: /opt/so/conf/soc/analytics.js From 11fb33fdeb942aadebcc0a4664b130f514223628 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Mon, 19 May 2025 14:19:56 -0600 Subject: [PATCH 3/5] Add RulesetName to Rule Repos Fill in `rulesetName` in the rules repos of the ElastAlert and Strelka engines. These will act as an example to anybody adding their repos to these lists. The field is not required, but helps avoid collisions when managing repos as the value is used for the folder name. When not present, the final folder of the repo url is used as the rulesetName and as the folder name on disk. Note that rulesetNames including a `/` will create extra folders in the path but the rulesetName will contain the slash, i.e. `rulesetName="joesecurity/sigma-rules"` will create the nested structure of `reposFolder/joesecurity/sigma-rules" containing the contents of the repo. All rules imported from this repo will have the ruleset of `joesecurity/sigma-rules`. --- salt/soc/defaults.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index fe190ea69..d756489e1 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1415,17 +1415,21 @@ soc: license: Elastic-2.0 folder: sigma/stable community: true + rulesetName: securityonion-resources - repo: file:///nsm/rules/custom-local-repos/local-sigma license: Elastic-2.0 community: false + rulesetName: local-sigma airgap: - repo: file:///nsm/rules/detect-sigma/repos/securityonion-resources license: Elastic-2.0 folder: sigma/stable community: true + rulesetName: securityonion-resources - repo: file:///nsm/rules/custom-local-repos/local-sigma license: Elastic-2.0 community: false + rulesetName: local-sigma sigmaRulePackages: - core - emerging_threats_addon @@ -1500,16 +1504,20 @@ soc: - repo: https://github.com/Security-Onion-Solutions/securityonion-yara license: DRL community: true + rulesetName: securityonion-yara - repo: file:///nsm/rules/custom-local-repos/local-yara license: Elastic-2.0 community: false + rulesetName: local-yara airgap: - repo: file:///nsm/rules/detect-yara/repos/securityonion-yara license: DRL community: true + rulesetName: securityonion-yara - repo: file:///nsm/rules/custom-local-repos/local-yara license: Elastic-2.0 community: false + rulesetName: local-yara yaraRulesFolder: /opt/sensoroni/yara/rules stateFilePath: /opt/sensoroni/fingerprints/strelkaengine.state integrityCheckFrequencySeconds: 1200 From 39f74fe547d519a69e5b391512ccb44eba19cc31 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Mon, 19 May 2025 15:37:33 -0600 Subject: [PATCH 4/5] Use the new JSON object editor for RulesRepos config entries --- salt/soc/soc_soc.yaml | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 91ab6e3c1..58560e89e 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -344,6 +344,23 @@ soc: advanced: True forcedType: "[]{}" helpLink: sigma.html + syntax: json + uiElements: + - field: rulesetName + label: Ruleset Name + - field: repo + label: Repo URL + required: True + - field: branch + label: Branch + - field: license + label: License + required: True + - field: folder + label: Folder + - field: community + label: Community + forcedType: bool airgap: *eerulesRepos sigmaRulePackages: description: 'Defines the Sigma Community Ruleset you want to run. One of these (core | core+ | core++ | all ) as well as an optional Add-on (emerging_threats_addon). Once you have changed the ruleset here, the new settings will be applied within 15 minutes. At that point, you will need to wait for the scheduled rule update to take place (by default, every 24 hours), or you can force the update by nagivating to Detections --> Options dropdown menu --> Elastalert --> Full Update. WARNING! Changing the ruleset will remove all existing non-overlapping Sigma rules of the previous ruleset and their associated overrides. This removal cannot be undone.' @@ -459,6 +476,23 @@ soc: advanced: True forcedType: "[]{}" helpLink: yara.html + syntax: json + uiElements: + - field: rulesetName + label: Ruleset Name + - field: repo + label: Repo URL + required: True + - field: branch + label: Branch + - field: license + label: License + required: True + - field: folder + label: Folder + - field: community + label: Community + forcedType: bool airgap: *serulesRepos suricataengine: aiRepoUrl: @@ -592,7 +626,7 @@ soc: label: Query required: True - field: showSubtitle - label: Show Query in Dropdown. + label: Show Query in Dropdown. forcedType: bool queryToggleFilters: description: Customize togglable query filters that apply to all queries. Exclusive toggles will invert the filter if toggled off rather than omitting the filter from the query. From b753d40861bb0e07bfadd18e14c6e4d40fba25d1 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 20 May 2025 17:06:11 -0400 Subject: [PATCH 5/5] Tighten parsing --- salt/elasticsearch/files/ingest/global@custom | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom index 85154b908..5457a2703 100644 --- a/salt/elasticsearch/files/ingest/global@custom +++ b/salt/elasticsearch/files/ingest/global@custom @@ -24,8 +24,8 @@ { "set": { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } }, { "rename": { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } }, { "set": { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } }, - {"append": {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network'","ignore_failure":true}}, - {"foreach": {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"ignore_failure":true}}, + {"append": {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network' && ctx?.source?.ip != null","ignore_failure":true}}, + {"foreach": {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"if":"ctx?.event?.module == 'endpoint'","description":"Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"}}, { "remove": { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } } ] }