From 5aa611302a7cdf3a1f6159758710dd7ab20141f3 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 6 May 2024 19:08:01 +0000 Subject: [PATCH 1/4] Handle YARA rules for distributed deployments --- salt/allowed_states.map.jinja | 3 +++ salt/strelka/config.sls | 9 +++++++++ salt/top.sls | 3 +++ 3 files changed, 15 insertions(+) diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 7fbf4ff14..109e244d7 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -65,6 +65,7 @@ 'registry', 'manager', 'nginx', + 'strelka.manager', 'soc', 'kratos', 'influxdb', @@ -91,6 +92,7 @@ 'nginx', 'telegraf', 'influxdb', + 'strelka.manager', 'soc', 'kratos', 'elasticfleet', @@ -111,6 +113,7 @@ 'nginx', 'telegraf', 'influxdb', + 'strelka.manager', 'soc', 'kratos', 'elastic-fleet-package-registry', diff --git a/salt/strelka/config.sls b/salt/strelka/config.sls index 90bba58a7..c65f9c2cb 100644 --- a/salt/strelka/config.sls +++ b/salt/strelka/config.sls @@ -29,6 +29,15 @@ strelkarulesdir: - group: 939 - makedirs: True +{%- if grains.role in ['so-sensor', 'so-heavynode'] %} +strelkasensorrules: + file.managed: + - name: /opt/so/conf/strelka/rules/compiled/rules.compiled + - source: salt://strelka/rules/compiled/rules.compiled + - user: 939 + - group: 939 +{%- endif %} + strelkareposdir: file.directory: - name: /opt/so/conf/strelka/repos diff --git a/salt/top.sls b/salt/top.sls index d4852aa4d..e4eaab786 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -87,6 +87,7 @@ base: - registry - nginx - influxdb + - strelka.manager - soc - kratos - firewall @@ -161,6 +162,7 @@ base: - registry - nginx - influxdb + - strelka.manager - soc - kratos - firewall @@ -210,6 +212,7 @@ base: - manager - nginx - influxdb + - strelka.manager - soc - kratos - sensoroni From 445fb316342089293bc45efe3c6e24e006e9413a Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 6 May 2024 19:09:37 +0000 Subject: [PATCH 2/4] Add manager SLS --- salt/strelka/compile_yara.py | 67 ++++++++++++++++++++++++++++++++++++ salt/strelka/manager.sls | 45 ++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 salt/strelka/compile_yara.py create mode 100644 salt/strelka/manager.sls diff --git a/salt/strelka/compile_yara.py b/salt/strelka/compile_yara.py new file mode 100644 index 000000000..dc77980d2 --- /dev/null +++ b/salt/strelka/compile_yara.py @@ -0,0 +1,67 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +import os +import yara +import glob +import json +from concurrent.futures import ThreadPoolExecutor + +def check_syntax(rule_file): + try: + # Testing if compilation throws a syntax error, don't save the result + yara.compile(filepath=rule_file) + return (True, rule_file, None) + except yara.SyntaxError as e: + # Return the error message for logging purposes + return (False, rule_file, str(e)) + +def compile_yara_rules(rules_dir): + compiled_dir = os.path.join(rules_dir, "compiled") + compiled_rules_path = [ os.path.join(compiled_dir, "rules.compiled"), "/opt/so/saltstack/default/salt/strelka/rules/compiled/rules.compiled" ] + rule_files = glob.glob(os.path.join(rules_dir, '**/*.yar'), recursive=True) + files_to_compile = {} + removed_count = 0 + success_count = 0 + + # Use ThreadPoolExecutor to parallelize syntax checks + with ThreadPoolExecutor() as executor: + results = executor.map(check_syntax, rule_files) + + # Collect yara files and prepare for batch compilation + for success, rule_file, error_message in results: + if success: + files_to_compile[os.path.basename(rule_file)] = rule_file + success_count += 1 + else: + # Extract just the UUID from the rule file name + rule_id = os.path.splitext(os.path.basename(rule_file))[0] + log_entry = { + "event_module": "soc", + "event_dataset": "soc.detections", + "log.level": "error", + "error_message": error_message, + "error_analysis": "Syntax Error", + "detection_type": "YARA", + "rule_uuid": rule_id, + "error_type": "runtime_status" + } + with open('/opt/sensoroni/logs/detections_runtime-status_yara.log', 'a') as log_file: + json.dump(log_entry, log_file) + log_file.write('\n') # Ensure new entries start on new lines + os.remove(rule_file) + removed_count += 1 + + # Compile all remaining valid rules into a single file + if files_to_compile: + compiled_rules = yara.compile(filepaths=files_to_compile) + for path in compiled_rules_path: + compiled_rules.save(path) + print(f"All remaining rules compiled and saved into {path}") + + # Print summary of compilation results + print(f"Summary: {success_count} rules compiled successfully, {removed_count} rules removed due to errors.") + +compile_yara_rules("/opt/sensoroni/yara/rules/") diff --git a/salt/strelka/manager.sls b/salt/strelka/manager.sls new file mode 100644 index 000000000..1c56a18fd --- /dev/null +++ b/salt/strelka/manager.sls @@ -0,0 +1,45 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls in allowed_states %} + +# Strelka config +strelkaconfdir: + file.directory: + - name: /opt/so/conf/strelka/rules/compiled/ + - user: 939 + - group: 939 + - makedirs: True + +strelkacompileyara: + file.managed: + - name: /opt/so/conf/strelka/compile_yara.py + - source: salt://strelka/compile_yara/compile_yara.py + - user: 939 + - group: 939 + - makedirs: True + +strelkarulesdir: + file.directory: + - name: /opt/so/conf/strelka/rules + - user: 939 + - group: 939 + - makedirs: True + +strelkareposdir: + file.directory: + - name: /opt/so/conf/strelka/repos + - user: 939 + - group: 939 + - makedirs: True + +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + +{% endif %} From d2fa77ae1074accd831129fc96c0a62d9a5d0cf1 Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 6 May 2024 19:10:41 +0000 Subject: [PATCH 3/4] Update compile script --- salt/strelka/compile_yara.py | 67 ----------------------- salt/strelka/compile_yara/compile_yara.py | 9 +-- 2 files changed, 5 insertions(+), 71 deletions(-) delete mode 100644 salt/strelka/compile_yara.py diff --git a/salt/strelka/compile_yara.py b/salt/strelka/compile_yara.py deleted file mode 100644 index dc77980d2..000000000 --- a/salt/strelka/compile_yara.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -import os -import yara -import glob -import json -from concurrent.futures import ThreadPoolExecutor - -def check_syntax(rule_file): - try: - # Testing if compilation throws a syntax error, don't save the result - yara.compile(filepath=rule_file) - return (True, rule_file, None) - except yara.SyntaxError as e: - # Return the error message for logging purposes - return (False, rule_file, str(e)) - -def compile_yara_rules(rules_dir): - compiled_dir = os.path.join(rules_dir, "compiled") - compiled_rules_path = [ os.path.join(compiled_dir, "rules.compiled"), "/opt/so/saltstack/default/salt/strelka/rules/compiled/rules.compiled" ] - rule_files = glob.glob(os.path.join(rules_dir, '**/*.yar'), recursive=True) - files_to_compile = {} - removed_count = 0 - success_count = 0 - - # Use ThreadPoolExecutor to parallelize syntax checks - with ThreadPoolExecutor() as executor: - results = executor.map(check_syntax, rule_files) - - # Collect yara files and prepare for batch compilation - for success, rule_file, error_message in results: - if success: - files_to_compile[os.path.basename(rule_file)] = rule_file - success_count += 1 - else: - # Extract just the UUID from the rule file name - rule_id = os.path.splitext(os.path.basename(rule_file))[0] - log_entry = { - "event_module": "soc", - "event_dataset": "soc.detections", - "log.level": "error", - "error_message": error_message, - "error_analysis": "Syntax Error", - "detection_type": "YARA", - "rule_uuid": rule_id, - "error_type": "runtime_status" - } - with open('/opt/sensoroni/logs/detections_runtime-status_yara.log', 'a') as log_file: - json.dump(log_entry, log_file) - log_file.write('\n') # Ensure new entries start on new lines - os.remove(rule_file) - removed_count += 1 - - # Compile all remaining valid rules into a single file - if files_to_compile: - compiled_rules = yara.compile(filepaths=files_to_compile) - for path in compiled_rules_path: - compiled_rules.save(path) - print(f"All remaining rules compiled and saved into {path}") - - # Print summary of compilation results - print(f"Summary: {success_count} rules compiled successfully, {removed_count} rules removed due to errors.") - -compile_yara_rules("/opt/sensoroni/yara/rules/") diff --git a/salt/strelka/compile_yara/compile_yara.py b/salt/strelka/compile_yara/compile_yara.py index ece3c6a9e..dc77980d2 100644 --- a/salt/strelka/compile_yara/compile_yara.py +++ b/salt/strelka/compile_yara/compile_yara.py @@ -20,7 +20,7 @@ def check_syntax(rule_file): def compile_yara_rules(rules_dir): compiled_dir = os.path.join(rules_dir, "compiled") - compiled_rules_path = os.path.join(compiled_dir, "rules.compiled") + compiled_rules_path = [ os.path.join(compiled_dir, "rules.compiled"), "/opt/so/saltstack/default/salt/strelka/rules/compiled/rules.compiled" ] rule_files = glob.glob(os.path.join(rules_dir, '**/*.yar'), recursive=True) files_to_compile = {} removed_count = 0 @@ -57,10 +57,11 @@ def compile_yara_rules(rules_dir): # Compile all remaining valid rules into a single file if files_to_compile: compiled_rules = yara.compile(filepaths=files_to_compile) - compiled_rules.save(compiled_rules_path) - print(f"All remaining rules compiled and saved into {compiled_rules_path}") + for path in compiled_rules_path: + compiled_rules.save(path) + print(f"All remaining rules compiled and saved into {path}") # Print summary of compilation results print(f"Summary: {success_count} rules compiled successfully, {removed_count} rules removed due to errors.") -compile_yara_rules("/opt/sensoroni/yara/rules/") \ No newline at end of file +compile_yara_rules("/opt/sensoroni/yara/rules/") From 5056ec526bb4c032b82df102e5955a445d8e6cee Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 6 May 2024 19:27:38 +0000 Subject: [PATCH 4/4] Add compiled directory --- salt/strelka/rules/compiled/DO.NOT.TOUCH | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 salt/strelka/rules/compiled/DO.NOT.TOUCH diff --git a/salt/strelka/rules/compiled/DO.NOT.TOUCH b/salt/strelka/rules/compiled/DO.NOT.TOUCH new file mode 100644 index 000000000..e69de29bb