From dbfb178556fb7b7458d195120d0496434c3aa339 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 16 Apr 2024 12:22:53 -0400 Subject: [PATCH 1/5] Add test --- salt/soc/defaults.yaml | 2 ++ salt/soc/soc_soc.yaml | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index b8fcf0581..c39b7bd90 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1310,6 +1310,8 @@ soc: - rbac/users_roles strelkaengine: allowRegex: '' + autoEnabledYaraRules: + - securityonion-yara autoUpdateEnabled: true communityRulesImportFrequencySeconds: 28800 compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 42b80a3f0..2001fb0c1 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -186,6 +186,11 @@ soc: global: True advanced: True helpLink: yara.html + autoEnabledYaraRules: + description: 'Yara rules to automatically enable on initial import. Format is $Ruleset - for example, for the default shipped ruleset: securityonion-yara' + global: True + advanced: True + helpLink: sigma.html autoUpdateEnabled: description: 'Set to true to enable automatic Internet-connected updates of the Yara rulesets. If this is an Airgap system, this setting will be overridden and set to false.' global: True From 8cc4d2668ed5f47e1e984c2e2920d80fe33e6ef9 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 16 Apr 2024 12:52:14 -0400 Subject: [PATCH 2/5] Move compile_yara --- .../files/bin => strelka/compile_yara}/compile_yara.py | 5 +++++ salt/strelka/config.sls | 8 ++++++++ 2 files changed, 13 insertions(+) rename salt/{soc/files/bin => strelka/compile_yara}/compile_yara.py (58%) diff --git a/salt/soc/files/bin/compile_yara.py b/salt/strelka/compile_yara/compile_yara.py similarity index 58% rename from salt/soc/files/bin/compile_yara.py rename to salt/strelka/compile_yara/compile_yara.py index 43c8b1a09..b840dcab6 100644 --- a/salt/soc/files/bin/compile_yara.py +++ b/salt/strelka/compile_yara/compile_yara.py @@ -1,3 +1,8 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + import os import yara import glob diff --git a/salt/strelka/config.sls b/salt/strelka/config.sls index 929bef113..f6c66d8ff 100644 --- a/salt/strelka/config.sls +++ b/salt/strelka/config.sls @@ -14,6 +14,14 @@ strelkaconfdir: - group: 939 - makedirs: True +strelkacompileyara: + file.managed: + - name: /opt/so/conf/strelka/compile_yara.py + - source: salt://strelka/compile_yara/compile_yara.py + - user: 939 + - group: 939 + - makedirs: True + strelkarulesdir: file.directory: - name: /opt/so/conf/strelka/rules From ff284761913ed4881c9d98ba68e2022ebcdbb1e6 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Tue, 16 Apr 2024 13:10:17 -0400 Subject: [PATCH 3/5] Fix compile_yara path --- salt/soc/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index c39b7bd90..6a1376478 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1314,7 +1314,7 @@ soc: - securityonion-yara autoUpdateEnabled: true communityRulesImportFrequencySeconds: 28800 - compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py + compileYaraPythonScriptPath: /opt/sensoroni/yara/compile_yara.py denyRegex: '' reposFolder: /opt/sensoroni/yara/repos rulesRepos: From 6c6647629ca042b56ac989ee71fe7bfe10756a83 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Thu, 18 Apr 2024 11:32:17 -0400 Subject: [PATCH 4/5] Refactor yara for compilation --- salt/strelka/backend/enabled.sls | 2 +- salt/strelka/compile_yara/compile_yara.py | 62 ++++++++++++++++++++--- salt/strelka/config.sls | 2 +- salt/strelka/defaults.yaml | 2 +- 4 files changed, 57 insertions(+), 11 deletions(-) diff --git a/salt/strelka/backend/enabled.sls b/salt/strelka/backend/enabled.sls index 0df764a6e..a626924b1 100644 --- a/salt/strelka/backend/enabled.sls +++ b/salt/strelka/backend/enabled.sls @@ -17,7 +17,7 @@ strelka_backend: - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-strelka-backend:{{ GLOBALS.so_version }} - binds: - /opt/so/conf/strelka/backend/:/etc/strelka/:ro - - /opt/so/conf/strelka/rules/:/etc/yara/:ro + - /opt/so/conf/strelka/rules/compiled/:/etc/yara/:ro {% if DOCKER.containers['so-strelka-backend'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-strelka-backend'].custom_bind_mounts %} - {{ BIND }} diff --git a/salt/strelka/compile_yara/compile_yara.py b/salt/strelka/compile_yara/compile_yara.py index b840dcab6..2138c73d1 100644 --- a/salt/strelka/compile_yara/compile_yara.py +++ b/salt/strelka/compile_yara/compile_yara.py @@ -6,14 +6,60 @@ import os import yara import glob -import sys +import json +from concurrent.futures import ThreadPoolExecutor -def compile_yara_rules(rules_dir: str) -> None: - compiled_rules_path: str = os.path.join(rules_dir, "rules.yar.compiled") - rule_files: list[str] = glob.glob(os.path.join(rules_dir, '**/*.yar'), recursive=True) +def check_syntax(rule_file): + try: + # Testing if compilation throws a syntax error, don't save the result + yara.compile(filepath=rule_file) + return (True, rule_file, None) + except yara.SyntaxError as e: + # Return the error message for logging purposes + return (False, rule_file, str(e)) - if rule_files: - rules: yara.Rules = yara.compile(filepaths={os.path.basename(f): f for f in rule_files}) - rules.save(compiled_rules_path) +def compile_yara_rules(rules_dir): + compiled_dir = os.path.join(rules_dir, "compiled") + compiled_rules_path = os.path.join(compiled_dir, "rules.compiled") + rule_files = glob.glob(os.path.join(rules_dir, '**/*.yar'), recursive=True) + files_to_compile = {} + removed_count = 0 + success_count = 0 + + # Use ThreadPoolExecutor to parallelize syntax checks + with ThreadPoolExecutor() as executor: + results = executor.map(check_syntax, rule_files) + + # Collect yara files and prepare for batch compilation + for success, rule_file, error_message in results: + if success: + files_to_compile[os.path.basename(rule_file)] = rule_file + success_count += 1 + else: + # Extract just the UUID from the rule file name + rule_id = os.path.splitext(os.path.basename(rule_file))[0] + log_entry = { + "event.module": "soc", + "event.dataset": "soc.detections", + "log.level": "error", + "error.message": error_message, + "detection_type": "yara", + "rule.uuid": rule_id, + "error.type": "runtime_status" + } + with open('/opt/sensoroni/logs/detections_runtime-status_yara.log', 'a') as log_file: + json.dump(log_entry, log_file) + log_file.write('\n') # Ensure new entries start on new lines + os.remove(rule_file) + removed_count += 1 -compile_yara_rules(sys.argv[1]) + # Compile all remaining valid rules into a single file + if files_to_compile: + compiled_rules = yara.compile(filepaths=files_to_compile) + compiled_rules.save(compiled_rules_path) + print(f"All remaining rules compiled and saved into {compiled_rules_path}") + + # Print summary of compilation results + print(f"Summary: {success_count} rules compiled successfully, {removed_count} rules removed due to errors.") + +compile_yara_rules("/opt/sensoroni/yara/rules/") \ No newline at end of file diff --git a/salt/strelka/config.sls b/salt/strelka/config.sls index f6c66d8ff..90bba58a7 100644 --- a/salt/strelka/config.sls +++ b/salt/strelka/config.sls @@ -9,7 +9,7 @@ # Strelka config strelkaconfdir: file.directory: - - name: /opt/so/conf/strelka + - name: /opt/so/conf/strelka/rules/compiled/ - user: 939 - group: 939 - makedirs: True diff --git a/salt/strelka/defaults.yaml b/salt/strelka/defaults.yaml index da259fa14..f91ad8691 100644 --- a/salt/strelka/defaults.yaml +++ b/salt/strelka/defaults.yaml @@ -563,7 +563,7 @@ strelka: options: location: '/etc/yara/' compiled: - enabled: False + enabled: True filename: "rules.compiled" store_offset: True offset_meta_key: "StrelkaHexDump" From a237ef5d96568d0293f950c41c1d6347e8a948ad Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Fri, 19 Apr 2024 16:33:35 -0400 Subject: [PATCH 5/5] Update default queries --- salt/soc/defaults.yaml | 7 ++----- salt/strelka/compile_yara/compile_yara.py | 1 + 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 6a1376478..a4e7cbf4d 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -2128,14 +2128,11 @@ soc: query: "so_detection.isEnabled:false | groupby so_detection.language | groupby so_detection.ruleset so_detection.severity" description: Show all disabled Detections - name: "Detection Type - Suricata (NIDS)" - query: "so_detection.language:suricata | groupby so_detection.ruleset so_detection.isEnabled" + query: "so_detection.language:suricata | groupby so_detection.ruleset so_detection.isEnabled | groupby so_detection.category" description: Show all NIDS Detections, which are run with Suricata - name: "Detection Type - Sigma (Elastalert) - All" - query: "so_detection.language:sigma | groupby so_detection.ruleset so_detection.isEnabled" + query: "so_detection.language:sigma | groupby so_detection.ruleset so_detection.isEnabled | groupby so_detection.category | groupby so_detection.product" description: Show all Sigma Detections, which are run with Elastalert - - name: "Detection Type - Sigma (Elastalert) - Windows" - query: 'so_detection.language:sigma AND so_detection.content: "*product: windows*" | groupby so_detection.ruleset so_detection.isEnabled' - description: Show all Sigma Detections with a logsource of Windows - name: "Detection Type - YARA (Strelka)" query: "so_detection.language:yara | groupby so_detection.ruleset so_detection.isEnabled" description: Show all YARA detections, which are used by Strelka diff --git a/salt/strelka/compile_yara/compile_yara.py b/salt/strelka/compile_yara/compile_yara.py index 2138c73d1..6d88fbbde 100644 --- a/salt/strelka/compile_yara/compile_yara.py +++ b/salt/strelka/compile_yara/compile_yara.py @@ -43,6 +43,7 @@ def compile_yara_rules(rules_dir): "event.dataset": "soc.detections", "log.level": "error", "error.message": error_message, + "error.analysis": "syntax error", "detection_type": "yara", "rule.uuid": rule_id, "error.type": "runtime_status"