From 0eebe48492c4e818509e5e72e27345129f788592 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sat, 16 Nov 2024 21:20:24 -0500 Subject: [PATCH 01/18] soup corrections --- salt/manager/tools/sbin/soup | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 2c91430e7..2e48f0981 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1356,6 +1356,10 @@ main() { echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes." (wait_for_salt_minion "$MINIONID" "5" '/dev/stdout' || fail "Salt minion was not running or ready.") 2>&1 | tee -a "$SOUP_LOG" + + # Stop long-running scripts to allow potentially updated scripts to load on the next execution. + killall salt-relay.sh + highstate postupgrade_changes [[ $is_airgap -eq 0 ]] && unmount_update From 918f26962aa9d3ceb427892c3537da62c405b196 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Sun, 17 Nov 2024 12:21:06 -0500 Subject: [PATCH 02/18] ignore fp from hydra --- salt/common/tools/sbin/so-log-check | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index 300732587..fc855e9f7 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -150,6 +150,7 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error" # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0 EXCLUDED_ERRORS="$EXCLUDED_ERRORS|is not an ip string literal" # false positive (Open Canary logging out blank IP addresses) EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncing rule" # false positive (rule sync log line includes rule name which can contain 'error') + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|request_unauthorized" # false positive (login failures to Hydra result in an 'error' log) fi if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then From 56d6857cd6dbfd5878857071dd0772743bbd52b4 Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Mon, 18 Nov 2024 09:03:17 -0500 Subject: [PATCH 03/18] Addl customization for autoenable sigma --- salt/soc/defaults.yaml | 52 ++++++++++++++++++++++++++++++++++-------- salt/soc/soc_soc.yaml | 7 +++++- 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 6147af73b..552cd7020 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1327,16 +1327,48 @@ soc: showAiSummaries: true autoUpdateEnabled: true autoEnabledSigmaRules: - default: - - core+critical - - securityonion-resources+critical - - securityonion-resources+high - so-eval: - - securityonion-resources+critical - - securityonion-resources+high - so-import: - - securityonion-resources+critical - - securityonion-resources+high + default: |- + Enabled_On_Import: + # SOS - resources ruleset + - ruleset: ["securityonion-resources"] + level: ["critical", "high"] + product: ["*"] + category: ["*"] + service: ["*"] + # SigmaHQ - Core ruleset - Logsource: System events supported by Elastic Agent + - ruleset: ["core"] + level: ["critical"] + product: ["*"] + category: ["process_creation", "file_event", "registry_event", "network_connection", "dns_query"] + service: ["*"] + # SigmaHQ - Core ruleset - Logsource: Windows eventlogs + - ruleset: ["core"] + level: ["critical"] + product: ["windows"] + category: ["*"] + service: ["security", "system", "dns-client", "application"] + # SigmaHQ - Core ruleset - Logsource: misc + - ruleset: ["core"] + level: ["critical"] + product: ["*"] + category: ["antivirus"] + service: ["*"] + so-eval: |- + Enabled_On_Import: + # SOS - resources ruleset + - ruleset: ["securityonion-resources"] + level: ["critical", "high"] + product: ["*"] + category: ["*"] + service: ["*"] + so-import: |- + Enabled_On_Import: + # SOS - resources ruleset + - ruleset: ["securityonion-resources"] + level: ["critical", "high"] + product: ["*"] + category: ["*"] + service: ["*"] communityRulesImportFrequencySeconds: 86400 communityRulesImportErrorSeconds: 300 failAfterConsecutiveErrorCount: 10 diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index c27228ab6..180ef96e4 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -217,10 +217,15 @@ soc: jinjaEscaped: True autoEnabledSigmaRules: default: &autoEnabledSigmaRules - description: 'Sigma rules to automatically enable on initial import. Format is $Ruleset+$Level - for example, for the core community ruleset and critical level rules: core+critical. These will be applied based on role if defined and default if not.' + description: 'Sigma rules to automatically enable on initial import. The format is a YAML list, with the ability to filter for ruleset, level, product, category and service. Refer to the documentation for further details. These will be applied based on role if defined and default if not.' global: True advanced: True helpLink: sigma.html + multiline: True + syntax: yaml + duplicates: True + forcedType: string + jinjaEscaped: True so-eval: *autoEnabledSigmaRules so-import: *autoEnabledSigmaRules communityRulesImportFrequencySeconds: From d86c009f55e62a26e18e3ccba45817627298707b Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Mon, 18 Nov 2024 14:06:06 -0700 Subject: [PATCH 04/18] Add Annotations to Existing Detections Options The autoUpdateEnabled setting has been present for awhile and now have annotations. --- salt/soc/soc_soc.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index c27228ab6..4e81307bd 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -223,6 +223,10 @@ soc: helpLink: sigma.html so-eval: *autoEnabledSigmaRules so-import: *autoEnabledSigmaRules + autoUpdateEnabled: + description: 'Automatically update Sigma rules on a regular basis. This will update the rules based on the configured frequency.' + global: True + advanced: True communityRulesImportFrequencySeconds: description: 'How often to check for new Sigma rules (in seconds). This applies to both Community Rule Packages and any configured Git repos.' global: True @@ -329,6 +333,10 @@ soc: showAiSummaries: description: Show AI summaries for Strelka rules. global: True + autoUpdateEnabled: + description: 'Automatically update YARA rules on a regular basis. This will update the rules based on the configured frequency.' + global: True + advanced: True autoEnabledYaraRules: description: 'YARA rules to automatically enable on initial import. Format is $Ruleset - for example, for the default shipped ruleset: securityonion-yara' global: True @@ -367,6 +375,10 @@ soc: showAiSummaries: description: Show AI summaries for Suricata rules. global: True + autoUpdateEnabled: + description: 'Automatically update Suricata rules on a regular basis. This will update the rules based on the configured frequency.' + global: True + advanced: True communityRulesImportFrequencySeconds: description: 'How often to check for new Suricata rules (in seconds).' global: True From 92ddf2ec6cf3389048f1c76f11fcb63217a659de Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 19 Nov 2024 09:27:26 -0500 Subject: [PATCH 05/18] MFA issuer name shouldn't be an advanced setting --- salt/kratos/soc_kratos.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/kratos/soc_kratos.yaml b/salt/kratos/soc_kratos.yaml index 86227aa37..82490184b 100644 --- a/salt/kratos/soc_kratos.yaml +++ b/salt/kratos/soc_kratos.yaml @@ -126,7 +126,6 @@ kratos: issuer: description: The name to show in the MFA authenticator app. Useful for differentiating between installations that share the same user email address. global: True - advanced: True helpLink: kratos.html webauthn: enabled: From 3fcf197bc14d844bac8c069385fb7fced12fd188 Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Tue, 19 Nov 2024 11:54:15 -0500 Subject: [PATCH 06/18] Tweak structure --- salt/soc/defaults.yaml | 75 ++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 39 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 552cd7020..d9a5d4cb3 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1328,47 +1328,44 @@ soc: autoUpdateEnabled: true autoEnabledSigmaRules: default: |- - Enabled_On_Import: - # SOS - resources ruleset - - ruleset: ["securityonion-resources"] - level: ["critical", "high"] - product: ["*"] - category: ["*"] - service: ["*"] - # SigmaHQ - Core ruleset - Logsource: System events supported by Elastic Agent - - ruleset: ["core"] - level: ["critical"] - product: ["*"] - category: ["process_creation", "file_event", "registry_event", "network_connection", "dns_query"] - service: ["*"] - # SigmaHQ - Core ruleset - Logsource: Windows eventlogs - - ruleset: ["core"] - level: ["critical"] - product: ["windows"] - category: ["*"] - service: ["security", "system", "dns-client", "application"] - # SigmaHQ - Core ruleset - Logsource: misc - - ruleset: ["core"] - level: ["critical"] - product: ["*"] - category: ["antivirus"] - service: ["*"] + # SOS - resources ruleset + - ruleset: ["securityonion-resources"] + level: ["critical", "high"] + product: ["*"] + category: ["*"] + service: ["*"] + # SigmaHQ - Core ruleset - Logsource: System events supported by Elastic Agent + - ruleset: ["core"] + level: ["critical"] + product: ["*"] + category: ["process_creation", "file_event", "registry_event", "network_connection", "dns_query"] + service: ["*"] + # SigmaHQ - Core ruleset - Logsource: Windows eventlogs + - ruleset: ["core"] + level: ["critical"] + product: ["windows"] + category: ["*"] + service: ["security", "system", "dns-client", "application"] + # SigmaHQ - Core ruleset - Logsource: misc + - ruleset: ["core"] + level: ["critical"] + product: ["*"] + category: ["antivirus"] + service: ["*"] so-eval: |- - Enabled_On_Import: - # SOS - resources ruleset - - ruleset: ["securityonion-resources"] - level: ["critical", "high"] - product: ["*"] - category: ["*"] - service: ["*"] + # SOS - resources ruleset + - ruleset: ["securityonion-resources"] + level: ["critical", "high"] + product: ["*"] + category: ["*"] + service: ["*"] so-import: |- - Enabled_On_Import: - # SOS - resources ruleset - - ruleset: ["securityonion-resources"] - level: ["critical", "high"] - product: ["*"] - category: ["*"] - service: ["*"] + # SOS - resources ruleset + - ruleset: ["securityonion-resources"] + level: ["critical", "high"] + product: ["*"] + category: ["*"] + service: ["*"] communityRulesImportFrequencySeconds: 86400 communityRulesImportErrorSeconds: 300 failAfterConsecutiveErrorCount: 10 From 8958da83b30a2876ccdb5760334675c973e24aa4 Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Wed, 20 Nov 2024 18:00:26 -0500 Subject: [PATCH 07/18] Deprecate instead --- salt/soc/defaults.yaml | 4 ++++ salt/soc/merged.map.jinja | 10 +++++++++- salt/soc/soc_soc.yaml | 12 ++++++++++-- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index d9a5d4cb3..4c2f07aff 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1327,6 +1327,10 @@ soc: showAiSummaries: true autoUpdateEnabled: true autoEnabledSigmaRules: + default: [] + so-eval: [] + so-import: [] + enabledSigmaRules: default: |- # SOS - resources ruleset - ruleset: ["securityonion-resources"] diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index 38794c903..25b62683c 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -35,13 +35,21 @@ {# since cases is not a valid soc config item and only used for the map files, remove it from being placed in the config #} {% do SOCMERGED.config.server.modules.pop('cases') %} -{# set Sigma rules based on role if defined and default if not #} +{# set enabled Sigma rules based on role if defined and default if not #} +{# this particular config is deprecated as of 2.4.120 - use enabledSigmaRules instead #} {% if GLOBALS.role in SOCMERGED.config.server.modules.elastalertengine.autoEnabledSigmaRules %} {% do SOCMERGED.config.server.modules.elastalertengine.update({'autoEnabledSigmaRules': SOCMERGED.config.server.modules.elastalertengine.autoEnabledSigmaRules[GLOBALS.role]}) %} {% else %} {% do SOCMERGED.config.server.modules.elastalertengine.update({'autoEnabledSigmaRules': SOCMERGED.config.server.modules.elastalertengine.autoEnabledSigmaRules.default}) %} {% endif %} +{# set enabled Sigma rules based on role if defined and default if not #} +{% if GLOBALS.role in SOCMERGED.config.server.modules.elastalertengine.enabledSigmaRules %} +{% do SOCMERGED.config.server.modules.elastalertengine.update({'enabledSigmaRules': SOCMERGED.config.server.modules.elastalertengine.enabledSigmaRules[GLOBALS.role]}) %} +{% else %} +{% do SOCMERGED.config.server.modules.elastalertengine.update({'enabledSigmaRules': SOCMERGED.config.server.modules.elastalertengine.enabledSigmaRules.default}) %} +{% endif %} + {# set elastalertengine.rulesRepos and strelkaengine.rulesRepos based on airgap or not #} {% if GLOBALS.airgap %} {% do SOCMERGED.config.server.modules.elastalertengine.update({'rulesRepos': SOCMERGED.config.server.modules.elastalertengine.rulesRepos.airgap}) %} diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 180ef96e4..e8875ce14 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -215,8 +215,8 @@ soc: duplicates: True forcedType: string jinjaEscaped: True - autoEnabledSigmaRules: - default: &autoEnabledSigmaRules + enabledSigmaRules: + default: &enabledSigmaRules description: 'Sigma rules to automatically enable on initial import. The format is a YAML list, with the ability to filter for ruleset, level, product, category and service. Refer to the documentation for further details. These will be applied based on role if defined and default if not.' global: True advanced: True @@ -226,6 +226,14 @@ soc: duplicates: True forcedType: string jinjaEscaped: True + so-eval: *enabledSigmaRules + so-import: *enabledSigmaRules + autoEnabledSigmaRules: + default: &autoEnabledSigmaRules + description: 'DEPRECATED: Will be removed in a future release - use enabledSigmaRules instead.' + global: True + advanced: True + helpLink: sigma.html so-eval: *autoEnabledSigmaRules so-import: *autoEnabledSigmaRules communityRulesImportFrequencySeconds: From b1c4e32123ebd4842dcc80357028f30a557e9194 Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Thu, 21 Nov 2024 09:11:44 -0500 Subject: [PATCH 08/18] Remove duplicate option --- salt/soc/soc_soc.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index e8875ce14..697c3b647 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -223,7 +223,6 @@ soc: helpLink: sigma.html multiline: True syntax: yaml - duplicates: True forcedType: string jinjaEscaped: True so-eval: *enabledSigmaRules From f61bf1bd67689897f1746a015a49b35bb2d5a85d Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Thu, 21 Nov 2024 09:15:29 -0500 Subject: [PATCH 09/18] Remove adv --- salt/soc/soc_soc.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 697c3b647..3149bafa5 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -219,7 +219,6 @@ soc: default: &enabledSigmaRules description: 'Sigma rules to automatically enable on initial import. The format is a YAML list, with the ability to filter for ruleset, level, product, category and service. Refer to the documentation for further details. These will be applied based on role if defined and default if not.' global: True - advanced: True helpLink: sigma.html multiline: True syntax: yaml From efa6a533c30b11e168b06f82cebcc70e9b1f0c55 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 25 Nov 2024 15:47:47 -0600 Subject: [PATCH 10/18] add missing ilm to index template Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/elasticsearch/defaults.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 56ac2475c..7a9364148 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -3230,6 +3230,8 @@ elasticsearch: template: settings: index: + lifecycle: + name: so-logs-cisco_secure_email_gateway.log-logs number_of_replicas: 0 policy: phases: @@ -10462,6 +10464,8 @@ elasticsearch: template: settings: index: + lifecycle: + name: so-logs-rapid7_threat_command.alert-logs number_of_replicas: 0 policy: phases: @@ -10506,6 +10510,8 @@ elasticsearch: template: settings: index: + lifecycle: + name: so-logs-rapid7_threat_command.ioc-logs number_of_replicas: 0 policy: phases: @@ -10550,6 +10556,8 @@ elasticsearch: template: settings: index: + lifecycle: + name: so-logs-rapid7_threat_command.vulnerability-logs number_of_replicas: 0 policy: phases: From 993d56cb58383d0ab85f461f6e57499be286e31d Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 25 Nov 2024 15:51:49 -0600 Subject: [PATCH 11/18] ti_rapid7* Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/elasticsearch/defaults.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 7a9364148..9f0d3576c 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -10465,7 +10465,7 @@ elasticsearch: settings: index: lifecycle: - name: so-logs-rapid7_threat_command.alert-logs + name: so-logs-ti_rapid7_threat_command.alert-logs number_of_replicas: 0 policy: phases: @@ -10511,7 +10511,7 @@ elasticsearch: settings: index: lifecycle: - name: so-logs-rapid7_threat_command.ioc-logs + name: so-logs-ti_rapid7_threat_command.ioc-logs number_of_replicas: 0 policy: phases: @@ -10557,7 +10557,7 @@ elasticsearch: settings: index: lifecycle: - name: so-logs-rapid7_threat_command.vulnerability-logs + name: so-logs-ti_rapid7_threat_command.vulnerability-logs number_of_replicas: 0 policy: phases: From aade3db80d1ce1860cdba7f59bda5e08812b59fa Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Thu, 28 Nov 2024 07:00:23 -0500 Subject: [PATCH 12/18] Generate MSI --- .../tools/sbin_jinja/so-elastic-agent-gen-installers | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index e2b7d734b..7647086ad 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -76,5 +76,11 @@ do printf "\n### $GOOS/$GOARCH Installer Generated...\n" done +printf "\n\n### Generating MSI...\n" +docker run \ +--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \ +{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o /output/so-elastic-agent_msi --arch x64 /workspace/so-elastic-agent.wxs +printf "\n### MSI Generated...\n" + printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace\n" rm -rf /nsm/elastic-agent-workspace From 5b9f6b2d52fcbd2c37493d50531b50d0cf10c1f1 Mon Sep 17 00:00:00 2001 From: defensivedepth Date: Mon, 2 Dec 2024 14:42:56 -0500 Subject: [PATCH 13/18] fix path --- .../tools/sbin_jinja/so-elastic-agent-gen-installers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers index 7647086ad..850669ba0 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers @@ -79,7 +79,7 @@ done printf "\n\n### Generating MSI...\n" docker run \ --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \ -{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o /output/so-elastic-agent_msi --arch x64 /workspace/so-elastic-agent.wxs +{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o /output/so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs printf "\n### MSI Generated...\n" printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace\n" From 754d28e95db9ab58241e82704b12a6e5e8322480 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 5 Dec 2024 09:52:55 -0600 Subject: [PATCH 14/18] add openvpn & ipsec support to Zeek --- salt/elasticsearch/files/ingest/global@custom | 2 +- salt/elasticsearch/files/ingest/zeek.common | 1 + salt/elasticsearch/files/ingest/zeek.conn | 2 + salt/elasticsearch/files/ingest/zeek.ipsec | 38 +++++++++ .../templates/component/ecs/zeek.json | 83 +++++++++++++++++++ salt/soc/defaults.yaml | 6 ++ salt/zeek/defaults.yaml | 2 + 7 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 salt/elasticsearch/files/ingest/zeek.ipsec diff --git a/salt/elasticsearch/files/ingest/global@custom b/salt/elasticsearch/files/ingest/global@custom index dbf215fb1..27acce74c 100644 --- a/salt/elasticsearch/files/ingest/global@custom +++ b/salt/elasticsearch/files/ingest/global@custom @@ -10,7 +10,7 @@ { "split": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp" } }, { "set": { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } }, { "gsub": { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } }, - { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}" } }, + { "append": { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}", "allow_duplicates": false } }, { "set": { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } }, { "set": { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } }, { "set": { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } }, diff --git a/salt/elasticsearch/files/ingest/zeek.common b/salt/elasticsearch/files/ingest/zeek.common index 3265e20be..8cf4df9be 100644 --- a/salt/elasticsearch/files/ingest/zeek.common +++ b/salt/elasticsearch/files/ingest/zeek.common @@ -18,6 +18,7 @@ { "set": { "if": "ctx.destination?.ip != null", "field": "server.ip", "value": "{{destination.ip}}" } }, { "set": { "if": "ctx.destination?.port != null", "field": "server.port", "value": "{{destination.port}}" } }, { "set": { "field": "observer.name", "value": "{{agent.name}}" } }, + { "append": { "if": "ctx.network?.protocol != null && ctx.network?.protocol.contains(\"openvpn\")","field": "tags","value": ["{{network.protocol}}"],"allow_duplicates": false,"ignore_failure": true}}, { "date": { "field": "message2.ts", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "ignore_failure": true } }, { "remove": { "field": ["agent"], "ignore_failure": true } }, { "pipeline": { "name": "common" } } diff --git a/salt/elasticsearch/files/ingest/zeek.conn b/salt/elasticsearch/files/ingest/zeek.conn index 8b458e032..bd98192d8 100644 --- a/salt/elasticsearch/files/ingest/zeek.conn +++ b/salt/elasticsearch/files/ingest/zeek.conn @@ -38,6 +38,8 @@ { "set": { "if": "ctx.connection?.state == 'SH'", "field": "connection.state_description", "value": "Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was 'half' open)" } }, { "set": { "if": "ctx.connection?.state == 'SHR'", "field": "connection.state_description", "value": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator" } }, { "set": { "if": "ctx.connection?.state == 'OTH'", "field": "connection.state_description", "value": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)" } }, + { "set": { "if": "ctx.network?.protocol.contains(\"ipsec\")", "field": "network.protocol", "value": "ipsec"}}, + { "set": { "if": "ctx.network?.protocol.contains(\"openvpn\")", "field": "network.protocol", "value": "openvpn"}}, { "pipeline": { "name": "zeek.common" } } ] } diff --git a/salt/elasticsearch/files/ingest/zeek.ipsec b/salt/elasticsearch/files/ingest/zeek.ipsec new file mode 100644 index 000000000..bf2872ebc --- /dev/null +++ b/salt/elasticsearch/files/ingest/zeek.ipsec @@ -0,0 +1,38 @@ +{ + "description": "zeek.ipsec", + "processors": [ + {"set": { "field": "event.dataset","value": "ipsec"}}, + {"json": { "field": "message","target_field": "message2","ignore_failure": true}}, + {"rename": {"field": "message2.initiator_spi","target_field": "ipsec.initiator_spi","ignore_missing": true}}, + {"rename": {"field": "message2.responder_spi","target_field": "ipsec.responder_spi","ignore_missing": true}}, + {"rename": {"field": "message2.maj_ver","target_field": "ipsec.maj_version","ignore_missing": true}}, + {"rename": {"field": "message2.min_ver","target_field": "ipsec.min_version","ignore_missing": true}}, + {"set": {"ignore_failure": true,"field": "ipsec.version","value": "{{ipsec.maj_version}}.{{ipsec.min_version}}"}}, + {"rename": {"field": "message2.exchange_type","target_field": "ipsec.exchange_type","ignore_missing": true}}, + {"rename": {"field": "message2.flag_e","target_field": "ipsec.flag_e","ignore_missing": true}}, + {"rename": {"field": "message2.flag_c","target_field": "ipsec.flag_c","ignore_missing": true}}, + {"rename": {"field": "message2.flag_a","target_field": "ipsec.flag_a","ignore_missing": true}}, + {"rename": {"field": "message2.flag_i","target_field": "ipsec.flag_i","ignore_missing": true}}, + {"rename": {"field": "message2.flag_v","target_field": "ipsec.flag_v","ignore_missing": true}}, + {"rename": {"field": "message2.flag_r","target_field": "ipsec.flag_r","ignore_missing": true}}, + {"rename": {"field": "message2.message_id","target_field": "ipsec.message_id","ignore_missing": true}}, + {"rename": {"field": "message2.vendor_ids","target_field": "ipsec.vendor_ids","ignore_missing": true}}, + {"rename": {"field": "message2.notify_messages","target_field": "ipsec.notify_messages","ignore_missing": true}}, + {"rename": {"field": "message2.transforms","target_field": "ipsec.transforms","ignore_missing": true}}, + {"rename": {"field": "message2.ke_dh_groups","target_field": "ipsec.ke_dh_groups","ignore_missing": true}}, + {"rename": {"field": "message2.proposals","target_field": "ipsec.proposals","ignore_missing": true}}, + {"rename": {"field": "message2.certificates","target_field": "ipsec.certificates","ignore_missing": true}}, + {"rename": {"field": "message2.transform_attributes","target_field": "ipsec.transform_attributes","ignore_missing": true}}, + {"rename": {"field": "message2.length","target_field": "ipsec.length","ignore_missing": true}}, + {"rename": {"field": "message2.hash","target_field": "ipsec.hash","ignore_missing": true}}, + {"rename": {"field": "message2.doi","target_field": "ipsec.doi","ignore_missing": true}}, + {"rename": {"field": "message2.situation","target_field": "ipsec.situation","ignore_missing": true}}, + {"script": { + "lang": "painless", + "description": "Remove ipsec fields with empty arrays", + "source": "if (ctx.containsKey('ipsec') && ctx.ipsec instanceof Map) {\n for (String field : ['certificates', 'ke_dh_groups', 'notify_messages', 'proposals', 'transforms', 'transform_attributes', 'vendor_ids']) {\n if (ctx.ipsec[field] instanceof List && ctx.ipsec[field].isEmpty()) {\n ctx.ipsec.remove(field);\n }\n }\n }", + "ignore_failure": true + }}, + {"pipeline": {"name": "zeek.common"}} + ] +} \ No newline at end of file diff --git a/salt/elasticsearch/templates/component/ecs/zeek.json b/salt/elasticsearch/templates/component/ecs/zeek.json index f14349263..0f8fb2cfa 100644 --- a/salt/elasticsearch/templates/component/ecs/zeek.json +++ b/salt/elasticsearch/templates/component/ecs/zeek.json @@ -603,6 +603,89 @@ } } }, + "ipsec": { + "properties": { + "certificates": { + "ignore_above": 1024, + "type": "keyword" + }, + "exchange_type": { + "type": "short" + }, + "flag_a": { + "type": "boolean" + }, + "flag_c": { + "type": "boolean" + }, + "flag_e": { + "type": "boolean" + }, + "flag_i": { + "type": "boolean" + }, + "flag_r": { + "type": "boolean" + }, + "flag_v": { + "type": "boolean" + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "initiator_spi": { + "ignore_above": 1024, + "type": "keyword" + }, + "ke_dh_groups": { + "type": "short" + }, + "length": { + "type": "long" + }, + "maj_version": { + "type": "short" + }, + "message_id": { + "type": "long" + }, + "min_version": { + "type": "short" + }, + "notify_messages": { + "ignore_above": 1024, + "type": "keyword" + }, + "proposals": { + "type": "long" + }, + "responder_spi": { + "ignore_above": 1024, + "type": "keyword" + }, + "situation": { + "ignore_above": 1024, + "type": "keyword" + }, + "transform_attributes": { + "ignore_above": 1024, + "type": "keyword" + }, + "transforms": { + "ignore_above": 1024, + "type": "keyword" + }, + "vendor_ids": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "irc": { "properties": { "addl": { diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 7521c9582..580b6993f 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1929,6 +1929,9 @@ soc: - name: Intel description: Zeek Intel framework hits query: 'tags:intel | groupby intel.indicator | groupby -sankey intel.indicator source.ip | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby intel.indicator_type | groupby intel.seen_where' + - name: IPSec + description: IPSec VPN connection metadata + query: 'tags:ipsec | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby destination.geo.country_name | groupby ipsec.version' - name: IRC description: IRC (Internet Relay Chat) network metadata query: 'tags:irc | groupby irc.command.type | groupby -sankey irc.command.type irc.username | groupby irc.username | groupby irc.nickname | groupby irc.command.value | groupby irc.command.info | groupby source.ip | groupby destination.ip | groupby destination.port | groupby destination_geo.organization_name' @@ -1941,6 +1944,9 @@ soc: - name: NTLM description: NTLM (New Technology LAN Manager) network metadata query: 'tags:ntlm | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby ntlm.server.dns.name | groupby ntlm.server.nb.name | groupby ntlm.server.tree.name | groupby ntlm.success | groupby source.ip | groupby destination.ip' + - name: OpenVPN + description: OpenVPN connection metadata + query: 'tags:openvpn | groupby source.ip | groupby -sankey source.ip destination.ip | groupby destination.ip | groupby destination.port | groupby destination.geo.country_name' - name: PE description: PE (Portable Executable) files transferred via network traffic query: 'tags:pe | groupby file.machine | groupby -sankey file.machine file.os | groupby file.os | groupby -sankey file.os file.subsystem | groupby file.subsystem | groupby file.section_names | groupby file.is_exe | groupby file.is_64bit' diff --git a/salt/zeek/defaults.yaml b/salt/zeek/defaults.yaml index b4291640a..83d8baff0 100644 --- a/salt/zeek/defaults.yaml +++ b/salt/zeek/defaults.yaml @@ -70,6 +70,8 @@ zeek: - zeek-spicy-wireguard - zeek-spicy-stun - http2 + - zeek-spicy-ipsec + - zeek-spicy-openvpn load-sigs: - frameworks/signatures/detect-windows-shells redef: From 9532f21c7b037522bf1b1c6f51fe58a00c1d5a39 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 5 Dec 2024 13:49:44 -0600 Subject: [PATCH 15/18] check zeek reporter.log --- salt/common/tools/sbin/so-log-check | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index fc855e9f7..9b54cf586 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -211,6 +211,7 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integrity check failed" # Detections: Exclude false positive due to automated testing EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncErrors" # Detections: Not an actual error EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Initialized license manager" # SOC log: before fields.status was changed to fields.licenseStatus + EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log fi RESULT=0 @@ -249,6 +250,9 @@ exclude_log "agentstatus.log" # ignore this log since it tracks agents in error exclude_log "detections_runtime-status_yara.log" # temporarily ignore this log until Detections is more stable exclude_log "/nsm/kafka/data/" # ignore Kafka data directory from log check. +# Include Zeek reporter.log to detect errors after running known good pcap(s) through sensor +echo "/nsm/zeek/spool/logger/reporter.log" >> /tmp/log_check_files + for log_file in $(cat /tmp/log_check_files); do status "Checking log file $log_file" tail -n $RECENT_LOG_LINES $log_file > /tmp/log_check From ad8b339a3b658947e0658db44d877eaac5f970c1 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 6 Dec 2024 09:07:16 -0600 Subject: [PATCH 16/18] fix error due to null reference Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/elasticsearch/files/ingest/zeek.conn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/ingest/zeek.conn b/salt/elasticsearch/files/ingest/zeek.conn index bd98192d8..6051d93a1 100644 --- a/salt/elasticsearch/files/ingest/zeek.conn +++ b/salt/elasticsearch/files/ingest/zeek.conn @@ -38,8 +38,8 @@ { "set": { "if": "ctx.connection?.state == 'SH'", "field": "connection.state_description", "value": "Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was 'half' open)" } }, { "set": { "if": "ctx.connection?.state == 'SHR'", "field": "connection.state_description", "value": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator" } }, { "set": { "if": "ctx.connection?.state == 'OTH'", "field": "connection.state_description", "value": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)" } }, - { "set": { "if": "ctx.network?.protocol.contains(\"ipsec\")", "field": "network.protocol", "value": "ipsec"}}, - { "set": { "if": "ctx.network?.protocol.contains(\"openvpn\")", "field": "network.protocol", "value": "openvpn"}}, + { "set": { "if": "ctx.network?.protocol != null && ctx.network?.protocol.contains(\"ipsec\")", "field": "network.protocol", "value": "ipsec"}}, + { "set": { "if": "ctx.network?.protocol != null && ctx.network?.protocol.contains(\"openvpn\")", "field": "network.protocol", "value": "openvpn"}}, { "pipeline": { "name": "zeek.common" } } ] } From 1de20e9d43b1a0d2e94b628d5b1d5d443a0b4665 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 6 Dec 2024 09:55:56 -0600 Subject: [PATCH 17/18] fix zeek file extract Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/zeek/defaults.yaml | 1 + salt/zeek/files/zeekctl.cfg.jinja | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/zeek/defaults.yaml b/salt/zeek/defaults.yaml index b4291640a..c66124a86 100644 --- a/salt/zeek/defaults.yaml +++ b/salt/zeek/defaults.yaml @@ -23,6 +23,7 @@ zeek: CfgDir: /opt/zeek/etc CompressLogs: 1 ZeekPort: 27760 + FileExtractDir: "" local: load: - misc/loaded-scripts diff --git a/salt/zeek/files/zeekctl.cfg.jinja b/salt/zeek/files/zeekctl.cfg.jinja index 0a6be371c..018ac738c 100644 --- a/salt/zeek/files/zeekctl.cfg.jinja +++ b/salt/zeek/files/zeekctl.cfg.jinja @@ -1,6 +1,6 @@ ## Global ZeekControl configuration file. -{%- set ALLOWEDOPTIONS = ['commtimeout','commandtimeout','compresscmd','compressextension','compresslogs','compresslogsinflight','controltopic','crashexpireinterval','croncmd','debug','env_vars','havenfs','keeplogs','logdir','logexpireinterval','logrotationinterval','mailalarmsinterval','mailalarmsto','mailarchivelogfail','mailconnectionsummary','mailfrom','mailhostupdown','mailreceivingpackets','mailreplyto','mailsubjectprefix','mailto','makearchivename','memlimit','mindiskspace','pfringclusterid','pfringclustertype','pfringfirstappinstance','prefixes','savetraces','sendmail','sitepluginpath','sitepolicypath','sitepolicyscripts','statslogenable','statslogexpireinterval','statuscmdshowall','stoptimeout','stopwait','timefmt','timemachinehost','timemachineport','zeekargs','zeekport','bindir','capstatspath','cfgdir','debuglog','defaultstoredir','helperdir','libdir','libdir64','libdirinternal','localnetscfg','lockfile','logexpireminutes','nodecfg','os','pcapbufsize','pcapsnaplen','plugindir','pluginzeekdir','policydir','policydirsiteinstall','policydirsiteinstallauto','postprocdir','scriptsdir','spooldir','standalone','statefile','staticdir','statsdir','statslog','time','tmpdir','tmpexecdir','tracesummary','version','zeek','zeekbase'] %} +{%- set ALLOWEDOPTIONS = ['commtimeout','commandtimeout','compresscmd','compressextension','compresslogs','compresslogsinflight','controltopic','crashexpireinterval','croncmd','debug','env_vars','fileextractdir','havenfs','keeplogs','logdir','logexpireinterval','logrotationinterval','mailalarmsinterval','mailalarmsto','mailarchivelogfail','mailconnectionsummary','mailfrom','mailhostupdown','mailreceivingpackets','mailreplyto','mailsubjectprefix','mailto','makearchivename','memlimit','mindiskspace','pfringclusterid','pfringclustertype','pfringfirstappinstance','prefixes','savetraces','sendmail','sitepluginpath','sitepolicypath','sitepolicyscripts','statslogenable','statslogexpireinterval','statuscmdshowall','stoptimeout','stopwait','timefmt','timemachinehost','timemachineport','zeekargs','zeekport','bindir','capstatspath','cfgdir','debuglog','defaultstoredir','helperdir','libdir','libdir64','libdirinternal','localnetscfg','lockfile','logexpireminutes','nodecfg','os','pcapbufsize','pcapsnaplen','plugindir','pluginzeekdir','policydir','policydirsiteinstall','policydirsiteinstallauto','postprocdir','scriptsdir','spooldir','standalone','statefile','staticdir','statsdir','statslog','time','tmpdir','tmpexecdir','tracesummary','version','zeek','zeekbase'] %} {%- for option in ZEEKCTL|sort %} {%- if option|lower in ALLOWEDOPTIONS %} From e4db2f4819d6dddc96f7ba0108a6eb7cfb7a9585 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 10 Dec 2024 17:19:15 -0500 Subject: [PATCH 18/18] Update defaults.yaml --- salt/docker/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/docker/defaults.yaml b/salt/docker/defaults.yaml index 21cdf606c..7c776937d 100644 --- a/salt/docker/defaults.yaml +++ b/salt/docker/defaults.yaml @@ -82,6 +82,7 @@ docker: - 443:443 - 8443:8443 - 7788:7788 + - 7789:7789 custom_bind_mounts: [] extra_hosts: [] extra_env: []