Merge pull request #12805 from Security-Onion-Solutions/2.4/detectiondefaults

Strelka fixes and more

salt/soc/defaults.yaml

@@ -1310,9 +1310,11 @@ soc:
             - rbac/users_roles
         strelkaengine:
           allowRegex: ''
+          autoEnabledYaraRules:
+            - securityonion-yara
           autoUpdateEnabled: true
           communityRulesImportFrequencySeconds: 28800
-          compileYaraPythonScriptPath: /opt/so/conf/strelka/compile_yara.py
+          compileYaraPythonScriptPath: /opt/sensoroni/yara/compile_yara.py
           denyRegex: ''
           reposFolder: /opt/sensoroni/yara/repos
           rulesRepos:
@@ -2126,14 +2128,11 @@ soc:
               query: "so_detection.isEnabled:false | groupby so_detection.language | groupby so_detection.ruleset so_detection.severity"
               description: Show all disabled Detections
             - name: "Detection Type - Suricata (NIDS)"
-              query: "so_detection.language:suricata | groupby so_detection.ruleset so_detection.isEnabled"
+              query: "so_detection.language:suricata | groupby so_detection.ruleset so_detection.isEnabled | groupby so_detection.category"
               description: Show all NIDS Detections, which are run with Suricata
             - name: "Detection Type - Sigma (Elastalert) - All"
-              query: "so_detection.language:sigma | groupby so_detection.ruleset so_detection.isEnabled"
+              query: "so_detection.language:sigma | groupby so_detection.ruleset so_detection.isEnabled | groupby so_detection.category | groupby so_detection.product"
               description: Show all Sigma Detections, which are run with Elastalert
-            - name: "Detection Type - Sigma (Elastalert) - Windows"
-              query: 'so_detection.language:sigma AND so_detection.content: "*product: windows*" | groupby so_detection.ruleset so_detection.isEnabled'
-              description: Show all Sigma Detections with a logsource of Windows
             - name: "Detection Type - YARA (Strelka)"
               query: "so_detection.language:yara | groupby so_detection.ruleset so_detection.isEnabled"
               description: Show all YARA detections, which are used by Strelka

salt/soc/files/bin/compile_yara.py

@@ -1,14 +0,0 @@
-import os
-import yara
-import glob
-import sys
-
-def compile_yara_rules(rules_dir: str) -> None:
-		compiled_rules_path: str = os.path.join(rules_dir, "rules.yar.compiled")
-		rule_files: list[str] = glob.glob(os.path.join(rules_dir, '**/*.yar'), recursive=True)
-
-		if rule_files:
-				rules: yara.Rules = yara.compile(filepaths={os.path.basename(f): f for f in rule_files})
-				rules.save(compiled_rules_path)
-
-compile_yara_rules(sys.argv[1])

salt/soc/soc_soc.yaml

@@ -186,6 +186,11 @@ soc:
             global: True
             advanced: True
             helpLink: yara.html
+          autoEnabledYaraRules:
+            description: 'Yara rules to automatically enable on initial import. Format is $Ruleset - for example, for the default shipped ruleset: securityonion-yara'
+            global: True
+            advanced: True
+            helpLink: sigma.html
           autoUpdateEnabled:
             description: 'Set to true to enable automatic Internet-connected updates of the Yara rulesets. If this is an Airgap system, this setting will be overridden and set to false.'
             global: True

salt/strelka/backend/enabled.sls

@@ -17,7 +17,7 @@ strelka_backend:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-strelka-backend:{{ GLOBALS.so_version }}
     - binds:
       - /opt/so/conf/strelka/backend/:/etc/strelka/:ro
-      - /opt/so/conf/strelka/rules/:/etc/yara/:ro
+      - /opt/so/conf/strelka/rules/compiled/:/etc/yara/:ro
       {% if DOCKER.containers['so-strelka-backend'].custom_bind_mounts %}
         {% for BIND in DOCKER.containers['so-strelka-backend'].custom_bind_mounts %}
       - {{ BIND }}

salt/strelka/compile_yara/compile_yara.py

@@ -0,0 +1,66 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+import os
+import yara
+import glob
+import json
+from concurrent.futures import ThreadPoolExecutor
+
+def check_syntax(rule_file):
+    try:
+        # Testing if compilation throws a syntax error, don't save the result
+        yara.compile(filepath=rule_file)
+        return (True, rule_file, None)
+    except yara.SyntaxError as e:
+        # Return the error message for logging purposes
+        return (False, rule_file, str(e))
+
+def compile_yara_rules(rules_dir):
+    compiled_dir = os.path.join(rules_dir, "compiled")
+    compiled_rules_path = os.path.join(compiled_dir, "rules.compiled")
+    rule_files = glob.glob(os.path.join(rules_dir, '**/*.yar'), recursive=True)
+    files_to_compile = {}
+    removed_count = 0
+    success_count = 0
+    
+    # Use ThreadPoolExecutor to parallelize syntax checks
+    with ThreadPoolExecutor() as executor:
+        results = executor.map(check_syntax, rule_files)
+    
+    # Collect yara files and prepare for batch compilation
+    for success, rule_file, error_message in results:
+        if success:
+            files_to_compile[os.path.basename(rule_file)] = rule_file
+            success_count += 1
+        else:
+            # Extract just the UUID from the rule file name
+            rule_id = os.path.splitext(os.path.basename(rule_file))[0]
+            log_entry = {
+                "event.module": "soc",
+                "event.dataset": "soc.detections",
+                "log.level": "error",
+                "error.message": error_message,
+                "error.analysis": "syntax error",
+                "detection_type": "yara",
+                "rule.uuid": rule_id,
+                "error.type": "runtime_status"
+            }
+            with open('/opt/sensoroni/logs/detections_runtime-status_yara.log', 'a') as log_file:
+                json.dump(log_entry, log_file)
+                log_file.write('\n')  # Ensure new entries start on new lines
+            os.remove(rule_file)
+            removed_count += 1
+
+    # Compile all remaining valid rules into a single file
+    if files_to_compile:
+        compiled_rules = yara.compile(filepaths=files_to_compile)
+        compiled_rules.save(compiled_rules_path)
+        print(f"All remaining rules compiled and saved into {compiled_rules_path}")
+
+    # Print summary of compilation results
+    print(f"Summary: {success_count} rules compiled successfully, {removed_count} rules removed due to errors.")
+
+compile_yara_rules("/opt/sensoroni/yara/rules/")

salt/strelka/config.sls

@@ -9,7 +9,15 @@
 # Strelka config
 strelkaconfdir:
   file.directory:
-    - name: /opt/so/conf/strelka
+    - name: /opt/so/conf/strelka/rules/compiled/
+    - user: 939
+    - group: 939
+    - makedirs: True
+
+strelkacompileyara:
+  file.managed:
+    - name: /opt/so/conf/strelka/compile_yara.py
+    - source: salt://strelka/compile_yara/compile_yara.py
     - user: 939
     - group: 939
     - makedirs: True

salt/strelka/defaults.yaml

@@ -563,7 +563,7 @@ strelka:
               options:
                 location: '/etc/yara/'
                 compiled:
-                  enabled: False
+                  enabled: True
                   filename: "rules.compiled"
                 store_offset: True
                 offset_meta_key: "StrelkaHexDump"