fixed detection lack when tab and enter control character in event record#395 (#396)

* fixed no detected bug when enter and tab control character in record data #395

* added remove \r \n \t character in utils.rs
* added call of utils.rs function in selectionnodes.rs

* added tests #395

* changed space control character function args #395

* fixed test due to function args changes #395

* changed replace method using regex #395

* changed regex by record_data_filter.txt #395

* added record_data_filter.txt #395

* fixed test #395

* added record_data_filter

- add Properties regex
- add ScriptBlockText regex
- add Payload regex

config/regex/record_data_filter.txt

@@ -0,0 +1,9 @@
+keyname,regex,replaced_str
+AccessMask,"[\r\n\t]+",
+Accesses,"[\r\n\t]+",
+AuditPolicyChanges,"[\r\n\t]+",
+SidHistory,"[\r\n\t]+",
+AccessList,"[\r\n\t]+",
+Properties,"[\r\n\t]+",
+ScriptBlockText,"[\r\n\t]+",
+Payload,"[\r\n\t]+",

src/detections/print.rs

@@ -2,6 +2,8 @@ extern crate lazy_static;
 use crate::detections::configs;
 use crate::detections::utils;
 use crate::detections::utils::get_serde_number_to_string;
+use crate::filter::DataFilterRule;
+use crate::filter::FILTER_REGEX;
 use chrono::{DateTime, Local, TimeZone, Utc};
 use hashbrown::HashMap;
 use lazy_static::lazy_static;
@@ -128,6 +130,7 @@ impl Message {
     fn parse_message(&mut self, event_record: &Value, output: String) -> String {
         let mut return_message: String = output;
         let mut hash_map: HashMap<String, String> = HashMap::new();
+        let mut output_filter: Option<&DataFilterRule> = None;
         for caps in ALIASREGEX.captures_iter(&return_message) {
             let full_target_str = &caps[0];
             let target_length = full_target_str.chars().count() - 2; // The meaning of 2 is two percent
@@ -141,15 +144,20 @@ impl Message {
                 let split: Vec<&str> = array_str.split(".").collect();
                 let mut is_exist_event_key = false;
                 let mut tmp_event_record: &Value = event_record.into();
-                for s in split {
+                for s in &split {
                     if let Some(record) = tmp_event_record.get(s) {
                         is_exist_event_key = true;
                         tmp_event_record = record;
+                        output_filter = FILTER_REGEX.get(&s.to_string());
                     }
                 }
                 if is_exist_event_key {
-                    let hash_value = get_serde_number_to_string(tmp_event_record);
+                    let mut hash_value = get_serde_number_to_string(tmp_event_record);
                     if hash_value.is_some() {
+                        if output_filter.is_some() {
+                            hash_value =
+                                utils::replace_target_character(hash_value.as_ref(), output_filter);
+                        }
                         hash_map.insert(full_target_str.to_string(), hash_value.unwrap());
                     }
                 }

src/detections/rule/selectionnodes.rs

@@ -1,4 +1,5 @@
 use crate::detections::{detection::EvtxRecordInfo, utils};
+use crate::filter::FILTER_REGEX;
 use mopa::mopafy;
 use std::{sync::Arc, vec};
 use yaml_rust::Yaml;
@@ -314,6 +315,9 @@ impl SelectionNode for LeafSelectionNode {
                     ]
                 }
         */
+
+        let filter_rule = FILTER_REGEX.get(self.get_key());
+
         if self.get_key() == "EventData" {
             let values =
                 utils::get_event_value(&"Event.EventData.Data".to_string(), &event_record.record);
@@ -329,11 +333,15 @@ impl SelectionNode for LeafSelectionNode {
             let eventdata_data = values.unwrap();
             if eventdata_data.is_boolean() || eventdata_data.is_i64() || eventdata_data.is_string()
             {
+                let replaced_str = utils::replace_target_character(
+                    event_record.get_value(self.get_key()),
+                    filter_rule,
+                );
                 return self
                     .matcher
                     .as_ref()
                     .unwrap()
-                    .is_match(event_record.get_value(self.get_key()), event_record);
+                    .is_match(replaced_str.as_ref(), event_record);
             }
             // 配列の場合は配列の要素のどれか一つでもルールに合致すれば条件に一致したことにする。
             if eventdata_data.is_array() {
@@ -342,12 +350,15 @@ impl SelectionNode for LeafSelectionNode {
                     .unwrap()
                     .iter()
                     .any(|ary_element| {
-                        let aryelement_val = utils::value_to_string(ary_element);
+                        let replaced_str = utils::replace_target_character(
+                            utils::value_to_string(ary_element).as_ref(),
+                            filter_rule,
+                        );
                         return self
                             .matcher
                             .as_ref()
                             .unwrap()
-                            .is_match(aryelement_val.as_ref(), event_record);
+                            .is_match(replaced_str.as_ref(), event_record);
                     });
             } else {
                 return self
@@ -358,12 +369,14 @@ impl SelectionNode for LeafSelectionNode {
             }
         }
 
-        let event_value = self.get_event_value(&event_record);
+        let replaced_str =
+            utils::replace_target_character(self.get_event_value(&event_record), filter_rule);
+
         return self
             .matcher
             .as_ref()
             .unwrap()
-            .is_match(event_value, event_record);
+            .is_match(replaced_str.as_ref(), event_record);
     }
 
     fn init(&mut self) -> Result<(), Vec<String>> {

src/detections/utils.rs

@@ -3,6 +3,7 @@ extern crate csv;
 extern crate regex;
 
 use crate::detections::configs;
+use crate::filter::DataFilterRule;
 
 use tokio::runtime::Builder;
 use tokio::runtime::Runtime;
@@ -39,6 +40,28 @@ pub fn check_regex(string: &str, regex_list: &Vec<Regex>) -> bool {
     return false;
 }
 
+/// replace string from all defined regex in input to replace_str
+pub fn replace_target_character<'a>(
+    input_str: Option<&'a String>,
+    replace_rule: Option<&'a DataFilterRule>,
+) -> Option<String> {
+    if input_str.is_none() {
+        return None;
+    }
+    if replace_rule.is_none() {
+        return Some(input_str.unwrap().to_string());
+    }
+
+    let replace_regex_rule = &replace_rule.unwrap().regex_rule;
+    let replace_str = &replace_rule.unwrap().replace_str;
+
+    return Some(
+        replace_regex_rule
+            .replace_all(input_str.unwrap(), replace_str)
+            .to_string(),
+    );
+}
+
 pub fn check_allowlist(target: &str, regexes: &Vec<Regex>) -> bool {
     for regex in regexes {
         if regex.is_match(target) {
@@ -238,6 +261,7 @@ pub fn create_rec_info(data: Value, path: String, keys: &Vec<String>) -> EvtxRec
 #[cfg(test)]
 mod tests {
     use crate::detections::utils;
+    use crate::filter::DataFilterRule;
     use regex::Regex;
     use serde_json::Value;
 
@@ -326,4 +350,31 @@ mod tests {
 
         assert!(utils::get_serde_number_to_string(&event_record["Event"]["EventData"]).is_none());
     }
+
+    #[test]
+    /// 指定された文字から指定されたregexぉ実行する関数が動作するかのテスト
+    fn test_remove_space_control() {
+        let test_filter_rule = DataFilterRule {
+            regex_rule: Regex::new(r"[\r\n\t]+").unwrap(),
+            replace_str: "".to_string(),
+        };
+        let none_test_str: Option<&String> = None;
+
+        assert_eq!(
+            utils::replace_target_character(none_test_str, None).is_none(),
+            true
+        );
+
+        assert_eq!(
+            utils::replace_target_character(none_test_str, Some(&test_filter_rule)).is_none(),
+            true
+        );
+
+        let tmp = "h\ra\ny\ta\tb\nu\r\nsa".to_string();
+        let test_str: Option<&String> = Some(&tmp);
+        assert_eq!(
+            utils::replace_target_character(test_str, Some(&test_filter_rule)).unwrap(),
+            "hayabusa"
+        );
+    }
 }

src/filter.rs

@@ -2,6 +2,8 @@ use crate::detections::configs;
 use crate::detections::print::AlertMessage;
 use crate::detections::print::ERROR_LOG_STACK;
 use crate::detections::print::QUIET_ERRORS_FLAG;
+use crate::detections::utils;
+use hashbrown::HashMap;
 use hashbrown::HashSet;
 use lazy_static::lazy_static;
 use regex::Regex;
@@ -12,6 +14,78 @@ use std::io::{BufRead, BufReader};
 lazy_static! {
     static ref IDS_REGEX: Regex =
         Regex::new(r"^[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12}$").unwrap();
+    pub static ref FILTER_REGEX: HashMap<String, DataFilterRule> = load_record_filters();
+}
+
+#[derive(Debug)]
+pub struct DataFilterRule {
+    pub regex_rule: Regex,
+    pub replace_str: String,
+}
+
+fn load_record_filters() -> HashMap<String, DataFilterRule> {
+    let file_path = "config/regex/record_data_filter.txt";
+    let read_result = utils::read_csv(file_path);
+    let mut ret = HashMap::new();
+    if read_result.is_err() {
+        if configs::CONFIG.read().unwrap().args.is_present("verbose") {
+            AlertMessage::warn(
+                &mut BufWriter::new(std::io::stderr().lock()),
+                &format!("{} does not exist", file_path),
+            )
+            .ok();
+        }
+        if !*QUIET_ERRORS_FLAG {
+            ERROR_LOG_STACK
+                .lock()
+                .unwrap()
+                .push(format!("{} does not exist", file_path));
+        }
+        return HashMap::default();
+    }
+    read_result.unwrap().into_iter().for_each(|line| {
+        if line.len() != 3 {
+            return;
+        }
+
+        let empty = &"".to_string();
+        let key = line.get(0).unwrap_or(empty).trim();
+        let regex_str = line.get(1).unwrap_or(empty).trim();
+        let replaced_str = line.get(2).unwrap_or(empty).trim();
+        if key.len() == 0 || regex_str.len() == 0 {
+            return;
+        }
+
+        let regex_rule: Option<Regex> = match Regex::new(regex_str) {
+            Ok(regex) => Some(regex),
+            Err(_err) => {
+                let errmsg = format!("failed to read regex filter in record_data_filter.txt");
+                if configs::CONFIG.read().unwrap().args.is_present("verbose") {
+                    AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &errmsg)
+                        .ok();
+                }
+                if !*QUIET_ERRORS_FLAG {
+                    ERROR_LOG_STACK
+                        .lock()
+                        .unwrap()
+                        .push(format!("[ERROR] {}", errmsg));
+                }
+                None
+            }
+        };
+
+        if regex_rule.is_none() {
+            return;
+        }
+        ret.insert(
+            key.to_string(),
+            DataFilterRule {
+                regex_rule: regex_rule.unwrap(),
+                replace_str: replaced_str.to_string(),
+            },
+        );
+    });
+    return ret;
 }
 
 #[derive(Clone, Debug)]