Merge pull request #248 from Yamato-Security/feature/fill_no_use_rules

feature : exclude rules by their ID in case of duplicates, etc...

config/exclude-rules.txt

@@ -1,5 +1,5 @@
-c92f1896-d1d2-43c3-92d5-7a5b35c217bb # rules/sigma/other/win_exchange_cve_2021_42321.yml (rule parse error)
-83809e84-4475-4b69-bc3e-4aad8568612f # rules/sigma/builtin/win_exchange_transportagent.yml (rule parse error)
-7b449a5e-1db5-4dd0-a2dc-4e3a67282538 # replaced by hayabusa rule 
-c265cf08-3f99-46c1-8d59-328247057d57 # replaced by hayabusa rule
-66b6be3d-55d0-4f47-9855-d69df21740ea # replaced by hayabusa rule
+4fe151c2-ecf9-4fae-95ae-b88ec9c2fca6
+c92f1896-d1d2-43c3-92d5-7a5b35c217bb
+7b449a5e-1db5-4dd0-a2dc-4e3a67282538
+c265cf08-3f99-46c1-8d59-328247057d57
+66b6be3d-55d0-4f47-9855-d69df21740ea

config/noisy-rules.txt

@@ -1,5 +1,5 @@
-0f06a3a5-6a09-413f-8743-e6cf35561297 # sysmon_wmi_event_subscription.yml
-b0d77106-7bb0-41fe-bd94-d1752164d066 # win_rare_schtasks_creations.yml
-66bfef30-22a5-4fcd-ad44-8d81e60922ae # win_rare_service_installs.yml
-e98374a6-e2d9-4076-9b5c-11bdb2569995 # win_susp_failed_logons_single_source.yml
-6309ffc4-8fa2-47cf-96b8-a2f72e58e538 # win_susp_failed_logons_single_source2.yml
+0f06a3a5-6a09-413f-8743-e6cf35561297
+b0d77106-7bb0-41fe-bd94-d1752164d066
+66bfef30-22a5-4fcd-ad44-8d81e60922ae
+e98374a6-e2d9-4076-9b5c-11bdb2569995
+6309ffc4-8fa2-47cf-96b8-a2f72e58e538

rules/sigma/builtin/win_hidden_user_creation.yml

@@ -1,4 +1,3 @@
-
 title: Hidden Local User Creation
 author: Christian Burkard
 date: 2021/05/03

rules/sigma/builtin/win_user_added_to_local_administrators.yml

@@ -1,4 +1,3 @@
-
 title: User Added to Local Administrators
 author: Florian Roth
 date: 2017/03/14

rules/sigma/builtin/win_user_creation.yml

@@ -1,4 +1,3 @@
-
 title: Local User Creation
 author: Patrick Bareiss
 date: 2019/04/18

rules/sigma/other/win_exchange_cve_2021_42321.yml

@@ -1,4 +1,3 @@
-
 title: Possible Exploitation of Exchange RCE CVE-2021-42321
 author: Florian Roth, @testanull
 date: 2021/11/18

src/detections/configs.rs

@@ -60,6 +60,7 @@ fn build_app<'a>() -> ArgMatches<'a> {
     -u --utc 'Output time in UTC format (default: local time)'
     -d --directory=[DIRECTORY] 'Directory of multiple .evtx files'
     -s --statistics 'Prints statistics of event IDs'
+    -n --show-noisyalerts 'do not exclude noisy rules'
     -t --threadnum=[NUM] 'Thread number (default: optimal number for performance)'
     --contributors 'Prints the list of contributors'";
     App::new(&program)

src/detections/detection.rs

@@ -11,6 +11,7 @@ use crate::detections::print::MESSAGES;
 use crate::detections::rule;
 use crate::detections::rule::RuleNode;
 use crate::detections::utils::get_serde_number_to_string;
+use crate::fillter;
 use crate::yaml::ParseYaml;
 
 use std::sync::Arc;
@@ -51,10 +52,15 @@ impl Detection {
     }
 
     // ルールファイルをパースします。
-    pub fn parse_rule_files(level: String, rulespath: Option<&str>) -> Vec<RuleNode> {
+    pub fn parse_rule_files(
+        level: String,
+        rulespath: Option<&str>,
+        fill_ids: &fillter::RuleFill,
+    ) -> Vec<RuleNode> {
         // ルールファイルのパースを実行
         let mut rulefile_loader = ParseYaml::new();
-        let result_readdir = rulefile_loader.read_dir(rulespath.unwrap_or(DIRPATH_RULES), &level);
+        let result_readdir =
+            rulefile_loader.read_dir(rulespath.unwrap_or(DIRPATH_RULES), &level, fill_ids);
         if result_readdir.is_err() {
             AlertMessage::alert(
                 &mut std::io::stderr().lock(),
@@ -269,6 +275,7 @@ impl Detection {
 fn test_parse_rule_files() {
     let level = "informational";
     let opt_rule_path = Some("./test_files/rules/level_yaml");
-    let cole = Detection::parse_rule_files(level.to_owned(), opt_rule_path);
+    let cole =
+        Detection::parse_rule_files(level.to_owned(), opt_rule_path, &fillter::exclude_ids());
     assert_eq!(5, cole.len());
 }

src/fillter.rs

@@ -0,0 +1,36 @@
+use crate::detections::configs;
+use std::collections::HashSet;
+use std::fs;
+
+#[derive(Clone, Debug)]
+pub struct RuleFill {
+    pub no_use_rule: HashSet<String>,
+}
+
+pub fn exclude_ids() -> RuleFill {
+    let mut ids = String::from_utf8(fs::read("config/exclude-rules.txt").unwrap()).unwrap();
+    if !configs::CONFIG
+        .read()
+        .unwrap()
+        .args
+        .is_present("show-noisyalerts")
+    {
+        ids += "\n"; // 改行を入れないとexclude-rulesの一番最後の行とnoisy-rules.txtの一番最後の行が一行にまとめられてしまう。
+        ids += &String::from_utf8(fs::read("config/noisy-rules.txt").unwrap()).unwrap();
+    }
+
+    let mut fill_ids = RuleFill {
+        no_use_rule: HashSet::new(),
+    };
+
+    for v in ids.split_whitespace() {
+        let v = v.to_string();
+        if v.is_empty() {
+            // 空行は無視する。
+            continue;
+        }
+        fill_ids.no_use_rule.insert(v);
+    }
+
+    return fill_ids;
+}

src/lib.rs

@@ -1,5 +1,6 @@
 pub mod afterfact;
 pub mod detections;
+pub mod fillter;
 pub mod notify;
 pub mod omikuji;
 pub mod timeline;

src/main.rs

@@ -7,6 +7,7 @@ use evtx::{EvtxParser, ParserSettings};
 use hayabusa::detections::detection;
 use hayabusa::detections::detection::EvtxRecordInfo;
 use hayabusa::detections::print::AlertMessage;
+use hayabusa::fillter;
 use hayabusa::omikuji::Omikuji;
 use hayabusa::{afterfact::after_fact, detections::utils};
 use hayabusa::{detections::configs, timeline::timeline::Timeline};
@@ -128,9 +129,11 @@ fn analysis_files(evtx_files: Vec<PathBuf>) {
         .unwrap_or("informational")
         .to_uppercase();
     println!("Analyzing event files: {:?}", evtx_files.len());
+
     let rule_files = detection::Detection::parse_rule_files(
         level,
         configs::CONFIG.read().unwrap().args.value_of("rules"),
+        &fillter::exclude_ids(),
     );
     let mut pb = ProgressBar::new(evtx_files.len() as u64);
     let mut detection = detection::Detection::new(rule_files);

src/yaml.rs

@@ -3,6 +3,7 @@ extern crate yaml_rust;
 
 use crate::detections::configs;
 use crate::detections::print::AlertMessage;
+use crate::fillter::RuleFill;
 use std::collections::HashMap;
 use std::ffi::OsStr;
 use std::fs;
@@ -42,13 +43,18 @@ impl ParseYaml {
         Ok(file_content)
     }
 
-    pub fn read_dir<P: AsRef<Path>>(&mut self, path: P, level: &str) -> io::Result<String> {
+    pub fn read_dir<P: AsRef<Path>>(
+        &mut self,
+        path: P,
+        level: &str,
+        fill_ids: &RuleFill,
+    ) -> io::Result<String> {
         let mut entries = fs::read_dir(path)?;
         let yaml_docs = entries.try_fold(vec![], |mut ret, entry| {
             let entry = entry?;
             // フォルダは再帰的に呼び出す。
             if entry.file_type()?.is_dir() {
-                self.read_dir(entry.path(), level)?;
+                self.read_dir(entry.path(), level, fill_ids)?;
                 return io::Result::Ok(ret);
             }
             // ファイル以外は無視
@@ -115,6 +121,7 @@ impl ParseYaml {
                         .unwrap_or(&0)
                         + 1,
                 );
+
                 if configs::CONFIG.read().unwrap().args.is_present("verbose") {
                     println!("Loaded yml file path: {}", filepath);
                 }
@@ -130,6 +137,18 @@ impl ParseYaml {
                     return Option::None;
                 }
 
+                //除外されたルールは無視する
+                let rule_id = &yaml_doc["id"].as_str();
+                if rule_id.is_some() {
+                    match fill_ids.no_use_rule.get(&rule_id.unwrap_or("").to_string()) {
+                        None => (),
+                        Some(_) => {
+                            self.ignorerule_count += 1;
+                            return Option::None;
+                        }
+                    }
+                }
+
                 return Option::Some((filepath, yaml_doc));
             })
             .collect();
@@ -141,14 +160,24 @@ impl ParseYaml {
 #[cfg(test)]
 mod tests {
 
+    use crate::fillter;
     use crate::yaml;
+    use crate::yaml::RuleFill;
+    use std::collections::HashSet;
     use std::path::Path;
     use yaml_rust::YamlLoader;
 
     #[test]
     fn test_read_dir_yaml() {
         let mut yaml = yaml::ParseYaml::new();
-        &yaml.read_dir("test_files/rules/yaml/".to_string(), &"".to_owned());
+        let fill_ids = RuleFill {
+            no_use_rule: HashSet::new(),
+        };
+        let _ = &yaml.read_dir(
+            "test_files/rules/yaml/".to_string(),
+            &"".to_owned(),
+            &fill_ids,
+        );
         assert_ne!(yaml.files.len(), 0);
     }
 
@@ -183,7 +212,8 @@ mod tests {
     fn test_default_level_read_yaml() {
         let mut yaml = yaml::ParseYaml::new();
         let path = Path::new("test_files/rules/level_yaml");
-        yaml.read_dir(path.to_path_buf(), &"").unwrap();
+        yaml.read_dir(path.to_path_buf(), &"", &fillter::exclude_ids())
+            .unwrap();
         assert_eq!(yaml.files.len(), 5);
     }
 
@@ -191,36 +221,52 @@ mod tests {
     fn test_info_level_read_yaml() {
         let mut yaml = yaml::ParseYaml::new();
         let path = Path::new("test_files/rules/level_yaml");
-        yaml.read_dir(path.to_path_buf(), &"informational").unwrap();
+        yaml.read_dir(
+            path.to_path_buf(),
+            &"informational",
+            &fillter::exclude_ids(),
+        )
+        .unwrap();
         assert_eq!(yaml.files.len(), 5);
     }
     #[test]
     fn test_low_level_read_yaml() {
         let mut yaml = yaml::ParseYaml::new();
         let path = Path::new("test_files/rules/level_yaml");
-
-        yaml.read_dir(path.to_path_buf(), &"LOW").unwrap();
+        yaml.read_dir(path.to_path_buf(), &"LOW", &fillter::exclude_ids())
+            .unwrap();
         assert_eq!(yaml.files.len(), 4);
     }
     #[test]
     fn test_medium_level_read_yaml() {
         let mut yaml = yaml::ParseYaml::new();
         let path = Path::new("test_files/rules/level_yaml");
-        yaml.read_dir(path.to_path_buf(), &"MEDIUM").unwrap();
+        yaml.read_dir(path.to_path_buf(), &"MEDIUM", &fillter::exclude_ids())
+            .unwrap();
         assert_eq!(yaml.files.len(), 3);
     }
     #[test]
     fn test_high_level_read_yaml() {
         let mut yaml = yaml::ParseYaml::new();
         let path = Path::new("test_files/rules/level_yaml");
-        yaml.read_dir(path.to_path_buf(), &"HIGH").unwrap();
+        yaml.read_dir(path.to_path_buf(), &"HIGH", &fillter::exclude_ids())
+            .unwrap();
         assert_eq!(yaml.files.len(), 2);
     }
     #[test]
     fn test_critical_level_read_yaml() {
         let mut yaml = yaml::ParseYaml::new();
         let path = Path::new("test_files/rules/level_yaml");
-        yaml.read_dir(path.to_path_buf(), &"CRITICAL").unwrap();
+        yaml.read_dir(path.to_path_buf(), &"CRITICAL", &fillter::exclude_ids())
+            .unwrap();
         assert_eq!(yaml.files.len(), 1);
     }
+    #[test]
+    fn test_exclude_rules_file() {
+        let mut yaml = yaml::ParseYaml::new();
+        let path = Path::new("test_files/rules/yaml");
+        yaml.read_dir(path.to_path_buf(), &"", &fillter::exclude_ids())
+            .unwrap();
+        assert_eq!(yaml.ignorerule_count, 10);
+    }
 }

test_files/rules/yaml/1.yml

@@ -15,5 +15,4 @@ falsepositives:
 level: medium
 output: 'CommandLine=%CommandLine%¥nParentImage=%ParentImage%'
 creation_date: 2020/11/8
-updated_date: 2020/11/8
-
+updated_date: 2020/11/8

test_files/rules/yaml/exclude1.yml

@@ -0,0 +1,19 @@
+title: Sysmon Check command lines
+id : 4fe151c2-ecf9-4fae-95ae-b88ec9c2fca6
+description: hogehoge
+enabled: true
+author: Yea
+logsource: 
+    product: windows
+detection:
+    selection:
+        EventLog: Sysmon
+        EventID: 1
+        CommandLine: '*'
+    condition: selection
+falsepositives:
+    - unknown
+level: medium
+output: 'CommandLine=%CommandLine%¥nParentImage=%ParentImage%'
+creation_date: 2020/11/8
+updated_date: 2020/11/8

test_files/rules/yaml/exclude2.yml

@@ -0,0 +1,21 @@
+title: Possible Exploitation of Exchange RCE CVE-2021-42321
+author: Florian Roth, @testanull
+date: 2021/11/18
+description: Detects log entries that appear in exploitation attempts against MS Exchange
+  RCE CVE-2021-42321
+detection:
+  condition: 'Cmdlet failed. Cmdlet Get-App, '
+falsepositives:
+- Unknown, please report false positives via https://github.com/SigmaHQ/sigma/issues
+id: c92f1896-d1d2-43c3-92d5-7a5b35c217bb
+level: critical
+logsource:
+  product: windows
+  service: msexchange-management
+references:
+- https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-42321
+status: experimental
+tags:
+- attack.lateral_movement
+- attack.t1210
+ruletype: SIGMA

test_files/rules/yaml/exclude3.yml

@@ -0,0 +1,28 @@
+title: Hidden Local User Creation
+author: Christian Burkard
+date: 2021/05/03
+description: Detects the creation of a local hidden user account which should not
+  happen for event ID 4720.
+detection:
+  SELECTION_1:
+    EventID: 4720
+  SELECTION_2:
+    TargetUserName: '*$'
+  condition: (SELECTION_1 and SELECTION_2)
+falsepositives:
+- unknown
+fields:
+- EventCode
+- AccountName
+id: 7b449a5e-1db5-4dd0-a2dc-4e3a67282538
+level: high
+logsource:
+  product: windows
+  service: security
+references:
+- https://twitter.com/SBousseaden/status/1387743867663958021
+status: experimental
+tags:
+- attack.persistence
+- attack.t1136.001
+ruletype: SIGMA

test_files/rules/yaml/exclude4.yml

@@ -0,0 +1,30 @@
+title: User Added to Local Administrators
+author: Florian Roth
+date: 2017/03/14
+description: This rule triggers on user accounts that are added to the local Administrators
+  group, which could be legitimate activity or a sign of privilege escalation activity
+detection:
+  SELECTION_1:
+    EventID: 4732
+  SELECTION_2:
+    TargetUserName: Administr*
+  SELECTION_3:
+    TargetSid: S-1-5-32-544
+  SELECTION_4:
+    SubjectUserName: '*$'
+  condition: ((SELECTION_1 and (SELECTION_2 or SELECTION_3)) and  not (SELECTION_4))
+falsepositives:
+- Legitimate administrative activity
+id: c265cf08-3f99-46c1-8d59-328247057d57
+level: medium
+logsource:
+  product: windows
+  service: security
+modified: 2021/07/07
+status: stable
+tags:
+- attack.privilege_escalation
+- attack.t1078
+- attack.persistence
+- attack.t1098
+ruletype: SIGMA

test_files/rules/yaml/exclude5.yml

@@ -0,0 +1,31 @@
+title: Local User Creation
+author: Patrick Bareiss
+date: 2019/04/18
+description: Detects local user creation on windows servers, which shouldn't happen
+  in an Active Directory environment. Apply this Sigma Use Case on your windows server
+  logs and not on your DC logs.
+detection:
+  SELECTION_1:
+    EventID: 4720
+  condition: SELECTION_1
+falsepositives:
+- Domain Controller Logs
+- Local accounts managed by privileged account management tools
+fields:
+- EventCode
+- AccountName
+- AccountDomain
+id: 66b6be3d-55d0-4f47-9855-d69df21740ea
+level: low
+logsource:
+  product: windows
+  service: security
+modified: 2020/08/23
+references:
+- https://patrick-bareiss.com/detecting-local-user-creation-in-ad-with-sigma/
+status: experimental
+tags:
+- attack.persistence
+- attack.t1136
+- attack.t1136.001
+ruletype: SIGMA

test_files/rules/yaml/noisy1.yml

@@ -0,0 +1,25 @@
+title: WMI Event Subscription
+author: Tom Ueltschi (@c_APT_ure)
+date: 2019/01/12
+description: Detects creation of WMI event subscription persistence method
+detection:
+  SELECTION_1:
+    EventID: 19
+  SELECTION_2:
+    EventID: 20
+  SELECTION_3:
+    EventID: 21
+  condition: (SELECTION_1 or SELECTION_2 or SELECTION_3)
+falsepositives:
+- exclude legitimate (vetted) use of WMI event subscription in your network
+id: 0f06a3a5-6a09-413f-8743-e6cf35561297
+level: high
+logsource:
+  category: wmi_event
+  product: windows
+status: experimental
+tags:
+- attack.t1084
+- attack.persistence
+- attack.t1546.003
+ruletype: SIGMA

test_files/rules/yaml/noisy2.yml

@@ -0,0 +1,31 @@
+title: Rare Schtasks Creations
+author: Florian Roth
+date: 2017/03/23
+description: Detects rare scheduled tasks creations that only appear a few times per
+  time frame and could reveal password dumpers, backdoor installs or other types of
+  malicious code
+detection:
+  SELECTION_1:
+    EventID: 4698
+  condition: SELECTION_1 | count() by TaskName < 5
+falsepositives:
+- Software installation
+- Software updates
+id: b0d77106-7bb0-41fe-bd94-d1752164d066
+level: low
+logsource:
+  definition: The Advanced Audit Policy setting Object Access > Audit Other Object
+    Access Events has to be configured to allow this detection (not in the baseline
+    recommendations by Microsoft). We also recommend extracting the Command field
+    from the embedded XML in the event data.
+  product: windows
+  service: security
+status: experimental
+tags:
+- attack.execution
+- attack.privilege_escalation
+- attack.persistence
+- attack.t1053
+- car.2013-08-001
+- attack.t1053.005
+ruletype: SIGMA

test_files/rules/yaml/noisy3.yml

@@ -0,0 +1,26 @@
+title: Rare Service Installs
+author: Florian Roth
+date: 2017/03/08
+description: Detects rare service installs that only appear a few times per time frame
+  and could reveal password dumpers, backdoor installs or other types of malicious
+  services
+detection:
+  SELECTION_1:
+    EventID: 7045
+  condition: SELECTION_1 | count() by ServiceFileName < 5
+falsepositives:
+- Software installation
+- Software updates
+id: 66bfef30-22a5-4fcd-ad44-8d81e60922ae
+level: low
+logsource:
+  product: windows
+  service: system
+status: experimental
+tags:
+- attack.persistence
+- attack.privilege_escalation
+- attack.t1050
+- car.2013-09-005
+- attack.t1543.003
+ruletype: SIGMA

test_files/rules/yaml/noisy4.yml

@@ -0,0 +1,33 @@
+title: Failed Logins with Different Accounts from Single Source System
+author: Florian Roth
+date: 2017/01/10
+description: Detects suspicious failed logins with different user accounts from a
+  single source system
+detection:
+  SELECTION_1:
+    EventID: 529
+  SELECTION_2:
+    EventID: 4625
+  SELECTION_3:
+    TargetUserName: '*'
+  SELECTION_4:
+    WorkstationName: '*'
+  condition: ((SELECTION_1 or SELECTION_2) and SELECTION_3 and SELECTION_4) | count(TargetUserName)
+    by WorkstationName > 3
+falsepositives:
+- Terminal servers
+- Jump servers
+- Other multiuser systems like Citrix server farms
+- Workstations with frequently changing users
+id: e98374a6-e2d9-4076-9b5c-11bdb2569995
+level: medium
+logsource:
+  product: windows
+  service: security
+modified: 2021/09/21
+status: experimental
+tags:
+- attack.persistence
+- attack.privilege_escalation
+- attack.t1078
+ruletype: SIGMA

test_files/rules/yaml/noisy5.yml

@@ -0,0 +1,34 @@
+title: Failed Logins with Different Accounts from Single Source System
+author: Florian Roth
+date: 2017/01/10
+description: Detects suspicious failed logins with different user accounts from a
+  single source system
+detection:
+  SELECTION_1:
+    EventID: 4776
+  SELECTION_2:
+    TargetUserName: '*'
+  SELECTION_3:
+    Workstation: '*'
+  condition: (SELECTION_1 and SELECTION_2 and SELECTION_3) | count(TargetUserName)
+    by Workstation > 3
+falsepositives:
+- Terminal servers
+- Jump servers
+- Other multiuser systems like Citrix server farms
+- Workstations with frequently changing users
+id: 6309ffc4-8fa2-47cf-96b8-a2f72e58e538
+level: medium
+logsource:
+  product: windows
+  service: security
+modified: 2021/09/21
+related:
+- id: e98374a6-e2d9-4076-9b5c-11bdb2569995
+  type: derived
+status: experimental
+tags:
+- attack.persistence
+- attack.privilege_escalation
+- attack.t1078
+ruletype: SIGMA