Merge pull request #5537 from Security-Onion-Solutions/delta

Add improved ignore functionality for YARA rules used by Strelka and add default ignored rules that break compilation

salt/common/tools/sbin/so-yara-update

@@ -1,5 +1,4 @@
 #!/bin/bash
-
 # Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
@@ -20,13 +19,8 @@ echo "Starting to check for yara rule updates at $(date)..."
 
 output_dir="/opt/so/saltstack/default/salt/strelka/rules"
 mkdir -p $output_dir
-
 repos="$output_dir/repos.txt"
-ignorefile="$output_dir/ignore.txt"
-
-deletecounter=0
 newcounter=0
-updatecounter=0
 
 {% if ISAIRGAP is sameas true %}
 
@@ -35,58 +29,21 @@ echo "Airgap mode enabled."
 clone_dir="/nsm/repo/rules/strelka"
 repo_name="signature-base"
 mkdir -p /opt/so/saltstack/default/salt/strelka/rules/signature-base
-
+# Ensure a copy of the license is available for the rules
 [ -f $clone_dir/LICENSE ] && cp $clone_dir/$repo_name/LICENSE $output_dir/$repo_name
 
 # Copy over rules
 for i in $(find $clone_dir/yara -name "*.yar*"); do
   rule_name=$(echo $i | awk -F '/' '{print $NF}')
-  repo_sum=$(sha256sum $i | awk '{print $1}')
+  echo "Adding rule: $rule_name..."
-
+  cp $i $output_dir/$repo_name
-  # Check rules against those in ignore list -- don't copy if ignored.
+  ((newcounter++))
-  if ! grep -iq $rule_name $ignorefile; then
-    existing_rules=$(find $output_dir/$repo_name/ -name $rule_name | wc -l)
-
-    # For existing rules, check to see if they need to be updated, by comparing checksums
-    if [ $existing_rules -gt 0 ];then
-      local_sum=$(sha256sum $output_dir/$repo_name/$rule_name | awk '{print $1}')
-      if [ "$repo_sum" != "$local_sum" ]; then
-        echo "Checksums do not match!"
-        echo "Updating $rule_name..."
-        cp $i $output_dir/$repo_name;
-        ((updatecounter++))
-      fi
-    else
-      # If rule doesn't exist already, we'll add it
-      echo "Adding new rule: $rule_name..."
-      cp $i $output_dir/$repo_name
-      ((newcounter++))
-    fi
-  fi;
-done
-
-# Check to see if we have any old rules that need to be removed
-for i in $(find $output_dir/$repo_name -name "*.yar*" | awk -F '/' '{print $NF}'); do
-  is_repo_rule=$(find $clone_dir -name "$i" | wc -l)
-  if [ $is_repo_rule -eq 0 ]; then
-    echo "Could not find $i in source $repo_name repo...removing from $output_dir/$repo_name..."
-    rm $output_dir/$repo_name/$i
-    ((deletecounter++))
-  fi
 done
 
 echo "Done!"
 
 if [ "$newcounter" -gt 0 ];then
-  echo "$newcounter new rules added."
+  echo "$newcounter rules added."
-fi
-
-if [ "$updatecounter" -gt 0 ];then
-  echo "$updatecounter rules updated."
-fi
-
-if [ "$deletecounter" -gt 0 ];then
-  echo "$deletecounter rules removed because they were deprecated or don't exist in the source repo."
 fi
 
 {% else %}
@@ -99,67 +56,30 @@ if [ "$gh_status" == "200" ]  || [ "$gh_status" == "301" ]; then
     if ! $(echo "$repo" | grep -qE '^#'); then
       # Remove old repo if existing bc of previous error condition or unexpected disruption
       repo_name=`echo $repo | awk -F '/' '{print $NF}'`
-      [ -d $repo_name ] && rm -rf $repo_name
+      [ -d $output_dir/$repo_name ] && rm -rf $output_dir/$repo_name
 
       # Clone repo and make appropriate directories for rules
-
       git clone $repo $clone_dir/$repo_name
       echo "Analyzing rules from $clone_dir/$repo_name..."
       mkdir -p $output_dir/$repo_name
+      # Ensure a copy of the license is available for the rules
       [ -f $clone_dir/$repo_name/LICENSE ] && cp $clone_dir/$repo_name/LICENSE $output_dir/$repo_name
 
       # Copy over rules
       for i in $(find $clone_dir/$repo_name -name "*.yar*"); do
         rule_name=$(echo $i | awk -F '/' '{print $NF}')
-        repo_sum=$(sha256sum $i | awk '{print $1}')
+        echo "Adding rule: $rule_name..."
-
+        cp $i $output_dir/$repo_name
-        # Check rules against those in ignore list -- don't copy if ignored.
+        ((newcounter++))
-        if ! grep -iq $rule_name $ignorefile; then
+     done
-          existing_rules=$(find $output_dir/$repo_name/ -name $rule_name | wc -l)
+     rm -rf $clone_dir/$repo_name
-
+    fi
-          # For existing rules, check to see if they need to be updated, by comparing checksums
+    done < $repos
-          if [ $existing_rules -gt 0 ];then
-            local_sum=$(sha256sum $output_dir/$repo_name/$rule_name | awk '{print $1}')
-            if [ "$repo_sum" != "$local_sum" ]; then
-              echo "Checksums do not match!"
-              echo "Updating $rule_name..."
-              cp $i $output_dir/$repo_name;
-              ((updatecounter++))
-            fi
-          else
-            # If rule doesn't exist already, we'll add it
-            echo "Adding new rule: $rule_name..."
-            cp $i $output_dir/$repo_name
-            ((newcounter++))
-          fi
-        fi;
-        done
-
-        # Check to see if we have any old rules that need to be removed
-        for i in $(find $output_dir/$repo_name -name "*.yar*" | awk -F '/' '{print $NF}'); do
-          is_repo_rule=$(find $clone_dir/$repo_name -name "$i" | wc -l)
-          if [ $is_repo_rule -eq 0 ]; then
-            echo "Could not find $i in source $repo_name repo...removing from $output_dir/$repo_name..."
-            rm $output_dir/$repo_name/$i
-            ((deletecounter++))
-          fi
-      done
-      rm -rf $clone_dir/$repo_name
-  fi
-  done < $repos
 
   echo "Done!"
   
   if [ "$newcounter" -gt 0 ];then
-    echo "$newcounter new rules added."
+    echo "$newcounter rules added."
-  fi
-
-  if [ "$updatecounter" -gt 0 ];then
-    echo "$updatecounter rules updated."
-  fi
-
-  if [ "$deletecounter" -gt 0 ];then
-    echo "$deletecounter rules removed because they were deprecated or don't exist in the source repo."
   fi
   
 else

salt/strelka/defaults.yaml

@@ -0,0 +1,9 @@
+strelka:
+  ignore:
+    - generic_anomalies.yar
+    - general_cloaking.yar
+    - thor_inverse_matches.yar
+    - yara_mixed_ext_vars.yar
+    - gen_susp_js_obfuscatorio.yar
+    - apt_flame2_orchestrator.yar
+    - apt_tetris.yar

salt/strelka/init.sls

@@ -21,6 +21,8 @@
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
 {% set STRELKA_RULES = salt['pillar.get']('strelka:rules', '1') %}
 {% set ENGINE = salt['pillar.get']('global:mdengine', '') %}
+{% import_yaml 'strelka/defaults.yaml' as strelka_config with context %}
+{% set IGNORELIST = salt['pillar.get']('strelka:ignore', strelka_config.strelka.ignore, merge=True, merge_nested_lists=True) %}
 
 # Strelka config
 strelkaconfdir:
@@ -54,6 +56,17 @@ strelkarules:
     - source: salt://strelka/rules
     - user: 939
     - group: 939
+    - clean: True
+    - exclude_pat:
+      {% for IGNOREDRULE in IGNORELIST %}
+      - {{ IGNOREDRULE }}
+      {% endfor %}
+
+      {% for IGNOREDRULE in IGNORELIST %}
+remove_rule_{{ IGNOREDRULE }}:
+  file.absent:
+    - name: /opt/so/conf/strelka/rules/signature-base/{{ IGNOREDRULE }}
+      {% endfor %}
 
 {% if grains['role'] in ['so-eval','so-managersearch', 'so-manager', 'so-standalone', 'so-import'] %}
 strelkarepos: