Refactor yara for compilation

salt/strelka/backend/enabled.sls

@@ -17,7 +17,7 @@ strelka_backend:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-strelka-backend:{{ GLOBALS.so_version }}
     - binds:
       - /opt/so/conf/strelka/backend/:/etc/strelka/:ro
-      - /opt/so/conf/strelka/rules/:/etc/yara/:ro
+      - /opt/so/conf/strelka/rules/compiled/:/etc/yara/:ro
       {% if DOCKER.containers['so-strelka-backend'].custom_bind_mounts %}
         {% for BIND in DOCKER.containers['so-strelka-backend'].custom_bind_mounts %}
       - {{ BIND }}

salt/strelka/compile_yara/compile_yara.py

@@ -6,14 +6,60 @@
 import os
 import yara
 import glob
-import sys
+import json
+from concurrent.futures import ThreadPoolExecutor
 
-def compile_yara_rules(rules_dir: str) -> None:
-		compiled_rules_path: str = os.path.join(rules_dir, "rules.yar.compiled")
-		rule_files: list[str] = glob.glob(os.path.join(rules_dir, '**/*.yar'), recursive=True)
+def check_syntax(rule_file):
+    try:
+        # Testing if compilation throws a syntax error, don't save the result
+        yara.compile(filepath=rule_file)
+        return (True, rule_file, None)
+    except yara.SyntaxError as e:
+        # Return the error message for logging purposes
+        return (False, rule_file, str(e))
 
-		if rule_files:
-				rules: yara.Rules = yara.compile(filepaths={os.path.basename(f): f for f in rule_files})
-				rules.save(compiled_rules_path)
+def compile_yara_rules(rules_dir):
+    compiled_dir = os.path.join(rules_dir, "compiled")
+    compiled_rules_path = os.path.join(compiled_dir, "rules.compiled")
+    rule_files = glob.glob(os.path.join(rules_dir, '**/*.yar'), recursive=True)
+    files_to_compile = {}
+    removed_count = 0
+    success_count = 0
     
-compile_yara_rules(sys.argv[1])
+    # Use ThreadPoolExecutor to parallelize syntax checks
+    with ThreadPoolExecutor() as executor:
+        results = executor.map(check_syntax, rule_files)
+    
+    # Collect yara files and prepare for batch compilation
+    for success, rule_file, error_message in results:
+        if success:
+            files_to_compile[os.path.basename(rule_file)] = rule_file
+            success_count += 1
+        else:
+            # Extract just the UUID from the rule file name
+            rule_id = os.path.splitext(os.path.basename(rule_file))[0]
+            log_entry = {
+                "event.module": "soc",
+                "event.dataset": "soc.detections",
+                "log.level": "error",
+                "error.message": error_message,
+                "detection_type": "yara",
+                "rule.uuid": rule_id,
+                "error.type": "runtime_status"
+            }
+            with open('/opt/sensoroni/logs/detections_runtime-status_yara.log', 'a') as log_file:
+                json.dump(log_entry, log_file)
+                log_file.write('\n')  # Ensure new entries start on new lines
+            os.remove(rule_file)
+            removed_count += 1
+
+    # Compile all remaining valid rules into a single file
+    if files_to_compile:
+        compiled_rules = yara.compile(filepaths=files_to_compile)
+        compiled_rules.save(compiled_rules_path)
+        print(f"All remaining rules compiled and saved into {compiled_rules_path}")
+
+    # Print summary of compilation results
+    print(f"Summary: {success_count} rules compiled successfully, {removed_count} rules removed due to errors.")
+
+compile_yara_rules("/opt/sensoroni/yara/rules/")

salt/strelka/config.sls

@@ -9,7 +9,7 @@
 # Strelka config
 strelkaconfdir:
   file.directory:
-    - name: /opt/so/conf/strelka
+    - name: /opt/so/conf/strelka/rules/compiled/
     - user: 939
     - group: 939
     - makedirs: True

salt/strelka/defaults.yaml

@@ -563,7 +563,7 @@ strelka:
               options:
                 location: '/etc/yara/'
                 compiled:
-                  enabled: False
+                  enabled: True
                   filename: "rules.compiled"
                 store_offset: True
                 offset_meta_key: "StrelkaHexDump"