separate excluded and noisy rules count (#559)

* changed ignored rules display separately  exclude rules and noisy rules.

* fixed tests #556

* cargo fmt

* updated changelog #556

* change order

* sorted output order #556

* cargo fmt

* screenshot update

Co-authored-by: Tanaka Zakku <71482215+YamatoSecurity@users.noreply.github.com>

CHANGELOG-Japanese.md

@@ -17,6 +17,7 @@
 - evtxファイルのファイルサイズの合計を出力するようにした。(#540) (@hitenkoku)
 - ロゴの色を変更した (#537) (@hitenkoku)
 - Channelの列にchannel_abbrevations.txtに記載されていないチャンネルも表示するようにした。(#553) (@hitenkoku)
+- `Ignored rules`として集計されていた`Exclude rules`、`Noisy rules`、`Deprecated rules`に分けて表示するようにした。 (#556) (@hitenkoku)
 
 **バグ修正:**
 

CHANGELOG.md

@@ -17,6 +17,7 @@
 - Display total evtx file size. (#540) (@hitenkoku)
 - Changed logo color. (#537) (@hitenkoku)
 - Display the original `Channel` name when not specified in `channel_abbrevations.txt`. (#553) (@hitenkoku)
+- Display separately `Ignored rules` to `Exclude rules`, `Noisy rules`, and `Deprecated rules`. (#556) (@hitenkoku)
 
 **Bug Fixes:**
 

src/detections/detection.rs

@@ -129,7 +129,9 @@ impl Detection {
             Detection::print_rule_load_info(
                 &rulefile_loader.rulecounter,
                 &parseerror_count,
-                &rulefile_loader.ignorerule_count,
+                &rulefile_loader.exclude_rule_count,
+                &rulefile_loader.noisy_rule_count,
+                &rulefile_loader.deprecate_rule_count,
             );
         }
         ret
@@ -354,22 +356,26 @@ impl Detection {
     pub fn print_rule_load_info(
         rc: &HashMap<String, u128>,
         parseerror_count: &u128,
-        ignore_count: &u128,
+        exclude_count: &u128,
+        noisy_count: &u128,
+        deprecate_count: &u128,
     ) {
         if *STATISTICS_FLAG {
             return;
         }
-        let mut total = parseerror_count + ignore_count;
-        rc.into_iter().for_each(|(key, value)| {
-            println!("{} rules: {}", key, value);
-            total += value;
-        });
-        println!("Ignored rules: {}", ignore_count);
+        println!("Deprecated rules: {}", deprecate_count);
+        println!("Excluded rules: {}", exclude_count);
+        println!("Noisy rules: {}", noisy_count);
         println!("Rule parsing errors: {}", parseerror_count);
-        println!(
-            "Total enabled detection rules: {}",
-            total - ignore_count - parseerror_count
-        );
+        println!();
+        let mut sorted_rc: Vec<(&String, &u128)> = rc.iter().collect();
+        sorted_rc.sort_by(|a, b| a.0.cmp(b.0));
+        let mut enable_total = 0;
+        sorted_rc.into_iter().for_each(|(key, value)| {
+            println!("{} rules: {}", key, value);
+            enable_total += value;
+        });
+        println!("Total enabled detection rules: {}", enable_total);
         println!();
     }
 }

src/filter.rs

@@ -2,7 +2,7 @@ use crate::detections::configs;
 use crate::detections::print::AlertMessage;
 use crate::detections::print::ERROR_LOG_STACK;
 use crate::detections::print::QUIET_ERRORS_FLAG;
-use hashbrown::HashSet;
+use hashbrown::HashMap;
 use regex::Regex;
 use std::fs::File;
 use std::io::{BufRead, BufReader};
@@ -15,13 +15,13 @@ pub struct DataFilterRule {
 
 #[derive(Clone, Debug)]
 pub struct RuleExclude {
-    pub no_use_rule: HashSet<String>,
+    pub no_use_rule: HashMap<String, String>,
 }
 
 impl RuleExclude {
     pub fn default() -> RuleExclude {
         RuleExclude {
-            no_use_rule: HashSet::new(),
+            no_use_rule: HashMap::new(),
         }
     }
 }
@@ -73,7 +73,7 @@ impl RuleExclude {
                 // 空行は無視する。IDの検証
                 continue;
             }
-            self.no_use_rule.insert(v);
+            self.no_use_rule.insert(v, filename.to_owned());
         }
     }
 }

src/yaml.rs

@@ -18,7 +18,9 @@ use yaml_rust::YamlLoader;
 pub struct ParseYaml {
     pub files: Vec<(String, yaml_rust::Yaml)>,
     pub rulecounter: HashMap<String, u128>,
-    pub ignorerule_count: u128,
+    pub exclude_rule_count: u128,
+    pub noisy_rule_count: u128,
+    pub deprecate_rule_count: u128,
     pub errorrule_count: u128,
 }
 
@@ -33,7 +35,9 @@ impl ParseYaml {
         ParseYaml {
             files: Vec::new(),
             rulecounter: HashMap::new(),
-            ignorerule_count: 0,
+            exclude_rule_count: 0,
+            noisy_rule_count: 0,
+            deprecate_rule_count: 0,
             errorrule_count: 0,
         }
     }
@@ -222,8 +226,12 @@ impl ParseYaml {
                         .get(&rule_id.unwrap_or("").to_string())
                     {
                         None => (),
-                        Some(_) => {
-                            self.ignorerule_count += 1;
+                        Some(v) => {
+                            if v.contains("exclude_rule") {
+                                self.exclude_rule_count += 1;
+                            } else {
+                                self.noisy_rule_count += 1;
+                            }
                             return Option::None;
                         }
                     }
@@ -259,9 +267,9 @@ impl ParseYaml {
                     .args
                     .is_present("enable-deprecated-rules")
                 {
-                    let rule_status = &yaml_doc["status"].as_str();
-                    if rule_status.is_some() && rule_status.unwrap() == "deprecated" {
-                        self.ignorerule_count += 1;
+                    let rule_status = &yaml_doc["status"].as_str().unwrap_or_default();
+                    if *rule_status == "deprecated" {
+                        self.deprecate_rule_count += 1;
                         return Option::None;
                     }
                 }
@@ -282,7 +290,7 @@ mod tests {
     use crate::filter;
     use crate::yaml;
     use crate::yaml::RuleExclude;
-    use hashbrown::HashSet;
+    use hashbrown::HashMap;
     use std::path::Path;
     use yaml_rust::YamlLoader;
 
@@ -306,7 +314,7 @@ mod tests {
 
         let mut yaml = yaml::ParseYaml::new();
         let exclude_ids = RuleExclude {
-            no_use_rule: HashSet::new(),
+            no_use_rule: HashMap::new(),
         };
         let _ = &yaml.read_dir("test_files/rules/yaml/", &String::default(), &exclude_ids);
         assert_ne!(yaml.files.len(), 0);
@@ -392,7 +400,16 @@ mod tests {
         let mut yaml = yaml::ParseYaml::new();
         let path = Path::new("test_files/rules/yaml");
         yaml.read_dir(path, "", &filter::exclude_ids()).unwrap();
-        assert_eq!(yaml.ignorerule_count, 10);
+        assert_eq!(yaml.exclude_rule_count, 5);
+    }
+    #[test]
+    fn test_all_noisy_rules_file() {
+        AlertMessage::create_error_log(ERROR_LOG_PATH.to_string());
+
+        let mut yaml = yaml::ParseYaml::new();
+        let path = Path::new("test_files/rules/yaml");
+        yaml.read_dir(path, "", &filter::exclude_ids()).unwrap();
+        assert_eq!(yaml.noisy_rule_count, 5);
     }
     #[test]
     fn test_none_exclude_rules_file() {
@@ -402,7 +419,7 @@ mod tests {
         let path = Path::new("test_files/rules/yaml");
         let exclude_ids = RuleExclude::default();
         yaml.read_dir(path, "", &exclude_ids).unwrap();
-        assert_eq!(yaml.ignorerule_count, 0);
+        assert_eq!(yaml.exclude_rule_count, 0);
     }
     #[test]
     fn test_exclude_deprecated_rules_file() {
@@ -410,6 +427,6 @@ mod tests {
         let path = Path::new("test_files/rules/deprecated");
         let exclude_ids = RuleExclude::default();
         yaml.read_dir(path, "", &exclude_ids).unwrap();
-        assert_eq!(yaml.ignorerule_count, 1);
+        assert_eq!(yaml.deprecate_rule_count, 1);
     }
 }