From 5a9d33c56578d3e6e88d41fc44ebab863eff319f Mon Sep 17 00:00:00 2001 From: DustInDark Date: Tue, 21 Jun 2022 15:25:20 +0900 Subject: [PATCH 1/6] fixed test --- test_files/rules/yaml/exclude1.yml | 4 ++-- test_files/rules/yaml/exclude2.yml | 10 ++-------- test_files/rules/yaml/exclude3.yml | 10 ++-------- test_files/rules/yaml/exclude4.yml | 14 ++------------ test_files/rules/yaml/exclude5.yml | 12 ++---------- test_files/rules/yaml/noisy1.yml | 6 ++---- test_files/rules/yaml/noisy2.yml | 24 +++--------------------- test_files/rules/yaml/noisy3.yml | 17 ++--------------- test_files/rules/yaml/noisy4.yml | 16 ++-------------- test_files/rules/yaml/noisy5.yml | 19 ++----------------- 10 files changed, 21 insertions(+), 111 deletions(-) diff --git a/test_files/rules/yaml/exclude1.yml b/test_files/rules/yaml/exclude1.yml index 76e3e73d..7fd19c8d 100644 --- a/test_files/rules/yaml/exclude1.yml +++ b/test_files/rules/yaml/exclude1.yml @@ -1,5 +1,5 @@ -title: Sysmon Check command lines -id : 4fe151c2-ecf9-4fae-95ae-b88ec9c2fca6 +title: Excluded Rule Test 1 +id : 00000000-0000-0000-0000-000000000000 description: hogehoge enabled: true author: Yea diff --git a/test_files/rules/yaml/exclude2.yml b/test_files/rules/yaml/exclude2.yml index e17e37cf..89214921 100644 --- a/test_files/rules/yaml/exclude2.yml +++ b/test_files/rules/yaml/exclude2.yml @@ -1,13 +1,10 @@ -title: Possible Exploitation of Exchange RCE CVE-2021-42321 -author: Florian Roth, @testanull +title: Excluded Rule 2 date: 2021/11/18 -description: Detects log entries that appear in exploitation attempts against MS Exchange - RCE CVE-2021-42321 detection: condition: 'Cmdlet failed. Cmdlet Get-App, ' falsepositives: - Unknown, please report false positives via https://github.com/SigmaHQ/sigma/issues -id: c92f1896-d1d2-43c3-92d5-7a5b35c217bb +id: 00000000-0000-0000-0000-000000000000 level: critical logsource: product: windows @@ -15,7 +12,4 @@ logsource: references: - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-42321 status: experimental -tags: -- attack.lateral_movement -- attack.t1210 ruletype: SIGMA diff --git a/test_files/rules/yaml/exclude3.yml b/test_files/rules/yaml/exclude3.yml index 45f43c4a..e5b79e6d 100644 --- a/test_files/rules/yaml/exclude3.yml +++ b/test_files/rules/yaml/exclude3.yml @@ -1,8 +1,5 @@ -title: Hidden Local User Creation -author: Christian Burkard +title: Excluded Rule 3 date: 2021/05/03 -description: Detects the creation of a local hidden user account which should not - happen for event ID 4720. detection: SELECTION_1: EventID: 4720 @@ -14,7 +11,7 @@ falsepositives: fields: - EventCode - AccountName -id: 7b449a5e-1db5-4dd0-a2dc-4e3a67282538 +id: 00000000-0000-0000-0000-000000000000 level: high logsource: product: windows @@ -22,7 +19,4 @@ logsource: references: - https://twitter.com/SBousseaden/status/1387743867663958021 status: experimental -tags: -- attack.persistence -- attack.t1136.001 ruletype: SIGMA diff --git a/test_files/rules/yaml/exclude4.yml b/test_files/rules/yaml/exclude4.yml index 06b76c48..95fe7061 100644 --- a/test_files/rules/yaml/exclude4.yml +++ b/test_files/rules/yaml/exclude4.yml @@ -1,8 +1,5 @@ -title: User Added to Local Administrators -author: Florian Roth +title: Excluded Rule 4 date: 2017/03/14 -description: This rule triggers on user accounts that are added to the local Administrators - group, which could be legitimate activity or a sign of privilege escalation activity detection: SELECTION_1: EventID: 4732 @@ -13,18 +10,11 @@ detection: SELECTION_4: SubjectUserName: '*$' condition: ((SELECTION_1 and (SELECTION_2 or SELECTION_3)) and not (SELECTION_4)) -falsepositives: -- Legitimate administrative activity -id: c265cf08-3f99-46c1-8d59-328247057d57 +id: 00000000-0000-0000-0000-000000000000 level: medium logsource: product: windows service: security modified: 2021/07/07 status: stable -tags: -- attack.privilege_escalation -- attack.t1078 -- attack.persistence -- attack.t1098 ruletype: SIGMA diff --git a/test_files/rules/yaml/exclude5.yml b/test_files/rules/yaml/exclude5.yml index 27ec53cc..b54b5eab 100644 --- a/test_files/rules/yaml/exclude5.yml +++ b/test_files/rules/yaml/exclude5.yml @@ -1,9 +1,5 @@ -title: Local User Creation -author: Patrick Bareiss +title: Excluded Rule 5 date: 2019/04/18 -description: Detects local user creation on windows servers, which shouldn't happen - in an Active Directory environment. Apply this Sigma Use Case on your windows server - logs and not on your DC logs. detection: SELECTION_1: EventID: 4720 @@ -15,7 +11,7 @@ fields: - EventCode - AccountName - AccountDomain -id: 66b6be3d-55d0-4f47-9855-d69df21740ea +id: 00000000-0000-0000-0000-000000000000 level: low logsource: product: windows @@ -24,8 +20,4 @@ modified: 2020/08/23 references: - https://patrick-bareiss.com/detecting-local-user-creation-in-ad-with-sigma/ status: experimental -tags: -- attack.persistence -- attack.t1136 -- attack.t1136.001 ruletype: SIGMA diff --git a/test_files/rules/yaml/noisy1.yml b/test_files/rules/yaml/noisy1.yml index 6ea217b6..eab1c29a 100644 --- a/test_files/rules/yaml/noisy1.yml +++ b/test_files/rules/yaml/noisy1.yml @@ -1,7 +1,5 @@ -title: WMI Event Subscription -author: Tom Ueltschi (@c_APT_ure) +title: Noisy Rule Test1 date: 2019/01/12 -description: Detects creation of WMI event subscription persistence method detection: SELECTION_1: EventID: 19 @@ -12,7 +10,7 @@ detection: condition: (SELECTION_1 or SELECTION_2 or SELECTION_3) falsepositives: - exclude legitimate (vetted) use of WMI event subscription in your network -id: 0f06a3a5-6a09-413f-8743-e6cf35561297 +id: 0090ea60-f4a2-43a8-8657-3a9a4ddcf547 level: high logsource: category: wmi_event diff --git a/test_files/rules/yaml/noisy2.yml b/test_files/rules/yaml/noisy2.yml index 2296fba4..20b18825 100644 --- a/test_files/rules/yaml/noisy2.yml +++ b/test_files/rules/yaml/noisy2.yml @@ -1,9 +1,6 @@ -title: Rare Schtasks Creations -author: Florian Roth +title: Noisy Rule Test2 date: 2017/03/23 -description: Detects rare scheduled tasks creations that only appear a few times per - time frame and could reveal password dumpers, backdoor installs or other types of - malicious code +description: excluded rule detection: SELECTION_1: EventID: 4698 @@ -11,21 +8,6 @@ detection: falsepositives: - Software installation - Software updates -id: b0d77106-7bb0-41fe-bd94-d1752164d066 +id: 8b8db936-172e-4bb7-9f84-ccc954d51d93 level: low -logsource: - definition: The Advanced Audit Policy setting Object Access > Audit Other Object - Access Events has to be configured to allow this detection (not in the baseline - recommendations by Microsoft). We also recommend extracting the Command field - from the embedded XML in the event data. - product: windows - service: security -status: experimental -tags: -- attack.execution -- attack.privilege_escalation -- attack.persistence -- attack.t1053 -- car.2013-08-001 -- attack.t1053.005 ruletype: SIGMA diff --git a/test_files/rules/yaml/noisy3.yml b/test_files/rules/yaml/noisy3.yml index 7e2071a0..8b4f209d 100644 --- a/test_files/rules/yaml/noisy3.yml +++ b/test_files/rules/yaml/noisy3.yml @@ -1,26 +1,13 @@ -title: Rare Service Installs -author: Florian Roth +title: Noisy Rule Test 3 date: 2017/03/08 -description: Detects rare service installs that only appear a few times per time frame - and could reveal password dumpers, backdoor installs or other types of malicious - services detection: SELECTION_1: EventID: 7045 condition: SELECTION_1 | count() by ServiceFileName < 5 -falsepositives: -- Software installation -- Software updates -id: 66bfef30-22a5-4fcd-ad44-8d81e60922ae +id: 1703ba97-b2c2-4071-a241-a16d017d25d3 level: low logsource: product: windows service: system status: experimental -tags: -- attack.persistence -- attack.privilege_escalation -- attack.t1050 -- car.2013-09-005 -- attack.t1543.003 ruletype: SIGMA diff --git a/test_files/rules/yaml/noisy4.yml b/test_files/rules/yaml/noisy4.yml index 39bbd1a3..5157c38a 100644 --- a/test_files/rules/yaml/noisy4.yml +++ b/test_files/rules/yaml/noisy4.yml @@ -1,8 +1,5 @@ -title: Failed Logins with Different Accounts from Single Source System -author: Florian Roth +title: Noisy Rule Test 4 date: 2017/01/10 -description: Detects suspicious failed logins with different user accounts from a - single source system detection: SELECTION_1: EventID: 529 @@ -14,20 +11,11 @@ detection: WorkstationName: '*' condition: ((SELECTION_1 or SELECTION_2) and SELECTION_3 and SELECTION_4) | count(TargetUserName) by WorkstationName > 3 -falsepositives: -- Terminal servers -- Jump servers -- Other multiuser systems like Citrix server farms -- Workstations with frequently changing users -id: e98374a6-e2d9-4076-9b5c-11bdb2569995 +id: 9f5663ce-6205-4753-b486-fb8498d1fae5 level: medium logsource: product: windows service: security modified: 2021/09/21 status: experimental -tags: -- attack.persistence -- attack.privilege_escalation -- attack.t1078 ruletype: SIGMA diff --git a/test_files/rules/yaml/noisy5.yml b/test_files/rules/yaml/noisy5.yml index ddfc134a..7a4b62d2 100644 --- a/test_files/rules/yaml/noisy5.yml +++ b/test_files/rules/yaml/noisy5.yml @@ -1,8 +1,5 @@ -title: Failed Logins with Different Accounts from Single Source System -author: Florian Roth +title: Noisy Rule Test 5 date: 2017/01/10 -description: Detects suspicious failed logins with different user accounts from a - single source system detection: SELECTION_1: EventID: 4776 @@ -12,23 +9,11 @@ detection: Workstation: '*' condition: (SELECTION_1 and SELECTION_2 and SELECTION_3) | count(TargetUserName) by Workstation > 3 -falsepositives: -- Terminal servers -- Jump servers -- Other multiuser systems like Citrix server farms -- Workstations with frequently changing users -id: 6309ffc4-8fa2-47cf-96b8-a2f72e58e538 +id: 3546ce10-19b4-4c4c-9658-f4f3b5d27ae9 level: medium logsource: product: windows service: security modified: 2021/09/21 -related: -- id: e98374a6-e2d9-4076-9b5c-11bdb2569995 - type: derived status: experimental -tags: -- attack.persistence -- attack.privilege_escalation -- attack.t1078 ruletype: SIGMA From 0f129a3a8c9849fdba6570c12f154e0c9f9828f6 Mon Sep 17 00:00:00 2001 From: DustInDark Date: Wed, 22 Jun 2022 01:22:22 +0900 Subject: [PATCH 2/6] refactering. moved update_rules option to src/options/update_rules.rs --- src/main.rs | 241 +-------------------------------------------- src/options/mod.rs | 1 + 2 files changed, 4 insertions(+), 238 deletions(-) diff --git a/src/main.rs b/src/main.rs index bf6a4797..2e46c9fc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,9 +7,8 @@ extern crate serde_derive; extern crate static_vcruntime; use bytesize::ByteSize; -use chrono::{DateTime, Datelike, Local, TimeZone}; +use chrono::{DateTime, Datelike, Local}; use evtx::{EvtxParser, ParserSettings}; -use git2::Repository; use hashbrown::{HashMap, HashSet}; use hayabusa::detections::configs::{load_pivot_keywords, TargetEventTime, TARGET_EXTENSIONS}; use hayabusa::detections::detection::{self, EvtxRecordInfo}; @@ -21,23 +20,19 @@ use hayabusa::detections::print::{ }; use hayabusa::detections::rule::{get_detection_keys, RuleNode}; use hayabusa::omikuji::Omikuji; -use hayabusa::options::level_tuning::LevelTuning; -use hayabusa::yaml::ParseYaml; +use hayabusa::options::{level_tuning::LevelTuning, update_rules::UpdateRules}; use hayabusa::{afterfact::after_fact, detections::utils}; use hayabusa::{detections::configs, timeline::timelines::Timeline}; use hayabusa::{detections::utils::write_color_buffer, filter}; use hhmmss::Hhmmss; use pbr::ProgressBar; use serde_json::Value; -use std::cmp::Ordering; use std::ffi::{OsStr, OsString}; use std::fmt::Display; use std::fmt::Write as _; -use std::fs::create_dir; use std::io::{BufWriter, Write}; use std::path::Path; use std::sync::Arc; -use std::time::SystemTime; use std::{ env, fs::{self, File}, @@ -113,7 +108,7 @@ impl App { } if configs::CONFIG.read().unwrap().args.update_rules { - match self.update_rules() { + match UpdateRules::update_rules() { Ok(output) => { if output != "You currently have the latest rules." { write_color_buffer( @@ -691,220 +686,6 @@ impl App { } } - /// update rules(hayabusa-rules subrepository) - fn update_rules(&self) -> Result { - let mut result; - let mut prev_modified_time: SystemTime = SystemTime::UNIX_EPOCH; - let mut prev_modified_rules: HashSet = HashSet::default(); - let hayabusa_repo = Repository::open(Path::new(".")); - let hayabusa_rule_repo = Repository::open(Path::new("rules")); - if hayabusa_repo.is_err() && hayabusa_rule_repo.is_err() { - write_color_buffer( - BufferWriter::stdout(ColorChoice::Always), - None, - "Attempting to git clone the hayabusa-rules repository into the rules folder.", - ) - .ok(); - // execution git clone of hayabusa-rules repository when failed open hayabusa repository. - result = self.clone_rules(); - } else if hayabusa_rule_repo.is_ok() { - // case of exist hayabusa-rules repository - self._repo_main_reset_hard(hayabusa_rule_repo.as_ref().unwrap())?; - // case of failed fetching origin/main, git clone is not executed so network error has occurred possibly. - prev_modified_rules = self.get_updated_rules("rules", &prev_modified_time); - prev_modified_time = fs::metadata("rules").unwrap().modified().unwrap(); - result = self.pull_repository(&hayabusa_rule_repo.unwrap()); - } else { - // case of no exist hayabusa-rules repository in rules. - // execute update because submodule information exists if hayabusa repository exists submodule information. - - prev_modified_time = fs::metadata("rules").unwrap().modified().unwrap(); - let rules_path = Path::new("rules"); - if !rules_path.exists() { - create_dir(rules_path).ok(); - } - let hayabusa_repo = hayabusa_repo.unwrap(); - let submodules = hayabusa_repo.submodules()?; - let mut is_success_submodule_update = true; - // submodule rules erase path is hard coding to avoid unintentional remove folder. - fs::remove_dir_all(".git/.submodule/rules").ok(); - for mut submodule in submodules { - submodule.update(true, None)?; - let submodule_repo = submodule.open()?; - if let Err(e) = self.pull_repository(&submodule_repo) { - AlertMessage::alert(&format!("Failed submodule update. {}", e)).ok(); - is_success_submodule_update = false; - } - } - if is_success_submodule_update { - result = Ok("Successed submodule update".to_string()); - } else { - result = Err(git2::Error::from_str(&String::default())); - } - } - if result.is_ok() { - let updated_modified_rules = self.get_updated_rules("rules", &prev_modified_time); - result = - self.print_diff_modified_rule_dates(prev_modified_rules, updated_modified_rules); - } - result - } - - /// hard reset in main branch - fn _repo_main_reset_hard(&self, input_repo: &Repository) -> Result<(), git2::Error> { - let branch = input_repo - .find_branch("main", git2::BranchType::Local) - .unwrap(); - let local_head = branch.get().target().unwrap(); - let object = input_repo.find_object(local_head, None).unwrap(); - match input_repo.reset(&object, git2::ResetType::Hard, None) { - Ok(()) => Ok(()), - _ => Err(git2::Error::from_str("Failed reset main branch in rules")), - } - } - - /// Pull(fetch and fast-forward merge) repositoryto input_repo. - fn pull_repository(&self, input_repo: &Repository) -> Result { - match input_repo - .find_remote("origin")? - .fetch(&["main"], None, None) - .map_err(|e| { - AlertMessage::alert(&format!("Failed git fetch to rules folder. {}", e)).ok(); - }) { - Ok(it) => it, - Err(_err) => return Err(git2::Error::from_str(&String::default())), - }; - let fetch_head = input_repo.find_reference("FETCH_HEAD")?; - let fetch_commit = input_repo.reference_to_annotated_commit(&fetch_head)?; - let analysis = input_repo.merge_analysis(&[&fetch_commit])?; - if analysis.0.is_up_to_date() { - Ok("Already up to date".to_string()) - } else if analysis.0.is_fast_forward() { - let mut reference = input_repo.find_reference("refs/heads/main")?; - reference.set_target(fetch_commit.id(), "Fast-Forward")?; - input_repo.set_head("refs/heads/main")?; - input_repo.checkout_head(Some(git2::build::CheckoutBuilder::default().force()))?; - Ok("Finished fast forward merge.".to_string()) - } else if analysis.0.is_normal() { - AlertMessage::alert( - "update-rules option is git Fast-Forward merge only. please check your rules folder." - , - ).ok(); - Err(git2::Error::from_str(&String::default())) - } else { - Err(git2::Error::from_str(&String::default())) - } - } - - /// git clone でhauyabusa-rules レポジトリをrulesフォルダにgit cloneする関数 - fn clone_rules(&self) -> Result { - match Repository::clone( - "https://github.com/Yamato-Security/hayabusa-rules.git", - "rules", - ) { - Ok(_repo) => { - println!("Finished cloning the hayabusa-rules repository."); - Ok("Finished clone".to_string()) - } - Err(e) => { - AlertMessage::alert( - &format!( - "Failed to git clone into the rules folder. Please rename your rules folder name. {}", - e - ), - ) - .ok(); - Err(git2::Error::from_str(&String::default())) - } - } - } - - /// Create rules folder files Hashset. Format is "[rule title in yaml]|[filepath]|[filemodified date]|[rule type in yaml]" - fn get_updated_rules( - &self, - rule_folder_path: &str, - target_date: &SystemTime, - ) -> HashSet { - let mut rulefile_loader = ParseYaml::new(); - // level in read_dir is hard code to check all rules. - rulefile_loader - .read_dir( - rule_folder_path, - "INFORMATIONAL", - &filter::RuleExclude::default(), - ) - .ok(); - - let hash_set_keys: HashSet = rulefile_loader - .files - .into_iter() - .filter_map(|(filepath, yaml)| { - let file_modified_date = fs::metadata(&filepath).unwrap().modified().unwrap(); - - if file_modified_date.cmp(target_date).is_gt() { - let yaml_date = yaml["date"].as_str().unwrap_or("-"); - return Option::Some(format!( - "{}|{}|{}|{}", - yaml["title"].as_str().unwrap_or(&String::default()), - yaml["modified"].as_str().unwrap_or(yaml_date), - &filepath, - yaml["ruletype"].as_str().unwrap_or("Other") - )); - } - Option::None - }) - .collect(); - hash_set_keys - } - - /// print updated rule files. - fn print_diff_modified_rule_dates( - &self, - prev_sets: HashSet, - updated_sets: HashSet, - ) -> Result { - let diff = updated_sets.difference(&prev_sets); - let mut update_count_by_rule_type: HashMap = HashMap::new(); - let mut latest_update_date = Local.timestamp(0, 0); - for diff_key in diff { - let tmp: Vec<&str> = diff_key.split('|').collect(); - let file_modified_date = fs::metadata(&tmp[2]).unwrap().modified().unwrap(); - - let dt_local: DateTime = file_modified_date.into(); - - if latest_update_date.cmp(&dt_local) == Ordering::Less { - latest_update_date = dt_local; - } - *update_count_by_rule_type - .entry(tmp[3].to_string()) - .or_insert(0b0) += 1; - write_color_buffer( - BufferWriter::stdout(ColorChoice::Always), - None, - &format!( - "[Updated] {} (Modified: {} | Path: {})", - tmp[0], tmp[1], tmp[2] - ), - ) - .ok(); - } - println!(); - for (key, value) in &update_count_by_rule_type { - println!("Updated {} rules: {}", key, value); - } - if !&update_count_by_rule_type.is_empty() { - Ok("Rule updated".to_string()) - } else { - write_color_buffer( - BufferWriter::stdout(ColorChoice::Always), - None, - "You currently have the latest rules.", - ) - .ok(); - Ok("You currently have the latest rules.".to_string()) - } - } - /// check architecture fn is_matched_architecture_and_binary(&self) -> bool { if cfg!(target_os = "windows") { @@ -942,20 +723,4 @@ mod tests { assert_eq!(is_contains, &true); }) } - - #[test] - fn test_get_updated_rules() { - let app = App::new(); - - let prev_modified_time: SystemTime = SystemTime::UNIX_EPOCH; - - let prev_modified_rules = - app.get_updated_rules("test_files/rules/level_yaml", &prev_modified_time); - assert_eq!(prev_modified_rules.len(), 5); - - let target_time: SystemTime = SystemTime::now(); - let prev_modified_rules2 = - app.get_updated_rules("test_files/rules/level_yaml", &target_time); - assert_eq!(prev_modified_rules2.len(), 0); - } } diff --git a/src/options/mod.rs b/src/options/mod.rs index 1f3c32b6..6841494b 100644 --- a/src/options/mod.rs +++ b/src/options/mod.rs @@ -1 +1,2 @@ pub mod level_tuning; +pub mod update_rules; From 94006a46777b27b206d07b26b333af6bd099e05d Mon Sep 17 00:00:00 2001 From: DustInDark Date: Wed, 22 Jun 2022 01:23:59 +0900 Subject: [PATCH 3/6] moved update_rules to options folder #597 --- src/options/update_rules.rs | 253 ++++++++++++++++++++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 src/options/update_rules.rs diff --git a/src/options/update_rules.rs b/src/options/update_rules.rs new file mode 100644 index 00000000..cd05a4aa --- /dev/null +++ b/src/options/update_rules.rs @@ -0,0 +1,253 @@ +use crate::detections::print::AlertMessage; +use crate::detections::utils::write_color_buffer; +use crate::filter; +use crate::yaml::ParseYaml; +use chrono::{DateTime, Local, TimeZone}; +use git2::Repository; +use std::fs::{self}; +use std::path::Path; + +use hashbrown::{HashMap, HashSet}; +use std::cmp::Ordering; + +use std::time::SystemTime; + +use std::fs::create_dir; + +use termcolor::{BufferWriter, ColorChoice}; + +pub struct UpdateRules {} + +impl UpdateRules { + /// update rules(hayabusa-rules subrepository) + pub fn update_rules() -> Result { + let mut result; + let mut prev_modified_time: SystemTime = SystemTime::UNIX_EPOCH; + let mut prev_modified_rules: HashSet = HashSet::default(); + let hayabusa_repo = Repository::open(Path::new(".")); + let hayabusa_rule_repo = Repository::open(Path::new("rules")); + if hayabusa_repo.is_err() && hayabusa_rule_repo.is_err() { + write_color_buffer( + BufferWriter::stdout(ColorChoice::Always), + None, + "Attempting to git clone the hayabusa-rules repository into the rules folder.", + ) + .ok(); + // execution git clone of hayabusa-rules repository when failed open hayabusa repository. + result = UpdateRules::clone_rules(); + } else if hayabusa_rule_repo.is_ok() { + // case of exist hayabusa-rules repository + UpdateRules::_repo_main_reset_hard(hayabusa_rule_repo.as_ref().unwrap())?; + // case of failed fetching origin/main, git clone is not executed so network error has occurred possibly. + prev_modified_rules = UpdateRules::get_updated_rules("rules", &prev_modified_time); + prev_modified_time = fs::metadata("rules").unwrap().modified().unwrap(); + result = UpdateRules::pull_repository(&hayabusa_rule_repo.unwrap()); + } else { + // case of no exist hayabusa-rules repository in rules. + // execute update because submodule information exists if hayabusa repository exists submodule information. + + prev_modified_time = fs::metadata("rules").unwrap().modified().unwrap(); + let rules_path = Path::new("rules"); + if !rules_path.exists() { + create_dir(rules_path).ok(); + } + let hayabusa_repo = hayabusa_repo.unwrap(); + let submodules = hayabusa_repo.submodules()?; + let mut is_success_submodule_update = true; + // submodule rules erase path is hard coding to avoid unintentional remove folder. + fs::remove_dir_all(".git/.submodule/rules").ok(); + for mut submodule in submodules { + submodule.update(true, None)?; + let submodule_repo = submodule.open()?; + if let Err(e) = UpdateRules::pull_repository(&submodule_repo) { + AlertMessage::alert(&format!("Failed submodule update. {}", e)).ok(); + is_success_submodule_update = false; + } + } + if is_success_submodule_update { + result = Ok("Successed submodule update".to_string()); + } else { + result = Err(git2::Error::from_str(&String::default())); + } + } + if result.is_ok() { + let updated_modified_rules = + UpdateRules::get_updated_rules("rules", &prev_modified_time); + result = UpdateRules::print_diff_modified_rule_dates( + prev_modified_rules, + updated_modified_rules, + ); + } + result + } + + /// hard reset in main branch + fn _repo_main_reset_hard(input_repo: &Repository) -> Result<(), git2::Error> { + let branch = input_repo + .find_branch("main", git2::BranchType::Local) + .unwrap(); + let local_head = branch.get().target().unwrap(); + let object = input_repo.find_object(local_head, None).unwrap(); + match input_repo.reset(&object, git2::ResetType::Hard, None) { + Ok(()) => Ok(()), + _ => Err(git2::Error::from_str("Failed reset main branch in rules")), + } + } + + /// Pull(fetch and fast-forward merge) repositoryto input_repo. + fn pull_repository(input_repo: &Repository) -> Result { + match input_repo + .find_remote("origin")? + .fetch(&["main"], None, None) + .map_err(|e| { + AlertMessage::alert(&format!("Failed git fetch to rules folder. {}", e)).ok(); + }) { + Ok(it) => it, + Err(_err) => return Err(git2::Error::from_str(&String::default())), + }; + let fetch_head = input_repo.find_reference("FETCH_HEAD")?; + let fetch_commit = input_repo.reference_to_annotated_commit(&fetch_head)?; + let analysis = input_repo.merge_analysis(&[&fetch_commit])?; + if analysis.0.is_up_to_date() { + Ok("Already up to date".to_string()) + } else if analysis.0.is_fast_forward() { + let mut reference = input_repo.find_reference("refs/heads/main")?; + reference.set_target(fetch_commit.id(), "Fast-Forward")?; + input_repo.set_head("refs/heads/main")?; + input_repo.checkout_head(Some(git2::build::CheckoutBuilder::default().force()))?; + Ok("Finished fast forward merge.".to_string()) + } else if analysis.0.is_normal() { + AlertMessage::alert( + "update-rules option is git Fast-Forward merge only. please check your rules folder." + , + ).ok(); + Err(git2::Error::from_str(&String::default())) + } else { + Err(git2::Error::from_str(&String::default())) + } + } + + /// git clone でhauyabusa-rules レポジトリをrulesフォルダにgit cloneする関数 + fn clone_rules() -> Result { + match Repository::clone( + "https://github.com/Yamato-Security/hayabusa-rules.git", + "rules", + ) { + Ok(_repo) => { + println!("Finished cloning the hayabusa-rules repository."); + Ok("Finished clone".to_string()) + } + Err(e) => { + AlertMessage::alert( + &format!( + "Failed to git clone into the rules folder. Please rename your rules folder name. {}", + e + ), + ) + .ok(); + Err(git2::Error::from_str(&String::default())) + } + } + } + + /// Create rules folder files Hashset. Format is "[rule title in yaml]|[filepath]|[filemodified date]|[rule type in yaml]" + fn get_updated_rules(rule_folder_path: &str, target_date: &SystemTime) -> HashSet { + let mut rulefile_loader = ParseYaml::new(); + // level in read_dir is hard code to check all rules. + rulefile_loader + .read_dir( + rule_folder_path, + "INFORMATIONAL", + &filter::RuleExclude::default(), + ) + .ok(); + + let hash_set_keys: HashSet = rulefile_loader + .files + .into_iter() + .filter_map(|(filepath, yaml)| { + let file_modified_date = fs::metadata(&filepath).unwrap().modified().unwrap(); + + if file_modified_date.cmp(target_date).is_gt() { + let yaml_date = yaml["date"].as_str().unwrap_or("-"); + return Option::Some(format!( + "{}|{}|{}|{}", + yaml["title"].as_str().unwrap_or(&String::default()), + yaml["modified"].as_str().unwrap_or(yaml_date), + &filepath, + yaml["ruletype"].as_str().unwrap_or("Other") + )); + } + Option::None + }) + .collect(); + hash_set_keys + } + + /// print updated rule files. + fn print_diff_modified_rule_dates( + prev_sets: HashSet, + updated_sets: HashSet, + ) -> Result { + let diff = updated_sets.difference(&prev_sets); + let mut update_count_by_rule_type: HashMap = HashMap::new(); + let mut latest_update_date = Local.timestamp(0, 0); + for diff_key in diff { + let tmp: Vec<&str> = diff_key.split('|').collect(); + let file_modified_date = fs::metadata(&tmp[2]).unwrap().modified().unwrap(); + + let dt_local: DateTime = file_modified_date.into(); + + if latest_update_date.cmp(&dt_local) == Ordering::Less { + latest_update_date = dt_local; + } + *update_count_by_rule_type + .entry(tmp[3].to_string()) + .or_insert(0b0) += 1; + write_color_buffer( + BufferWriter::stdout(ColorChoice::Always), + None, + &format!( + "[Updated] {} (Modified: {} | Path: {})", + tmp[0], tmp[1], tmp[2] + ), + ) + .ok(); + } + println!(); + for (key, value) in &update_count_by_rule_type { + println!("Updated {} rules: {}", key, value); + } + if !&update_count_by_rule_type.is_empty() { + Ok("Rule updated".to_string()) + } else { + write_color_buffer( + BufferWriter::stdout(ColorChoice::Always), + None, + "You currently have the latest rules.", + ) + .ok(); + Ok("You currently have the latest rules.".to_string()) + } + } +} + +#[cfg(test)] +mod tests { + use crate::options::update_rules::UpdateRules; + use std::time::SystemTime; + + #[test] + fn test_get_updated_rules() { + let prev_modified_time: SystemTime = SystemTime::UNIX_EPOCH; + + let prev_modified_rules = + UpdateRules::get_updated_rules("test_files/rules/level_yaml", &prev_modified_time); + assert_eq!(prev_modified_rules.len(), 5); + + let target_time: SystemTime = SystemTime::now(); + let prev_modified_rules2 = + UpdateRules::get_updated_rules("test_files/rules/level_yaml", &target_time); + assert_eq!(prev_modified_rules2.len(), 0); + } +} From c8c97c046aa6af94d5b787a46a7dde4f27f67b6e Mon Sep 17 00:00:00 2001 From: DustInDark Date: Wed, 22 Jun 2022 01:25:14 +0900 Subject: [PATCH 4/6] removed unused import --- src/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 2e46c9fc..46c7bec2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -706,7 +706,6 @@ impl App { #[cfg(test)] mod tests { use crate::App; - use std::time::SystemTime; #[test] fn test_collect_evtxfiles() { From 9ff2fd19ca4718814083feb3ded97fda8522766d Mon Sep 17 00:00:00 2001 From: DustInDark Date: Sun, 26 Jun 2022 01:02:54 +0900 Subject: [PATCH 5/6] changed write_color_buffer 1st arg is borrow --- src/options/update_rules.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/options/update_rules.rs b/src/options/update_rules.rs index cd05a4aa..258e02cd 100644 --- a/src/options/update_rules.rs +++ b/src/options/update_rules.rs @@ -28,7 +28,7 @@ impl UpdateRules { let hayabusa_rule_repo = Repository::open(Path::new("rules")); if hayabusa_repo.is_err() && hayabusa_rule_repo.is_err() { write_color_buffer( - BufferWriter::stdout(ColorChoice::Always), + &BufferWriter::stdout(ColorChoice::Always), None, "Attempting to git clone the hayabusa-rules repository into the rules folder.", ) @@ -205,7 +205,7 @@ impl UpdateRules { .entry(tmp[3].to_string()) .or_insert(0b0) += 1; write_color_buffer( - BufferWriter::stdout(ColorChoice::Always), + &BufferWriter::stdout(ColorChoice::Always), None, &format!( "[Updated] {} (Modified: {} | Path: {})", @@ -222,7 +222,7 @@ impl UpdateRules { Ok("Rule updated".to_string()) } else { write_color_buffer( - BufferWriter::stdout(ColorChoice::Always), + &BufferWriter::stdout(ColorChoice::Always), None, "You currently have the latest rules.", ) From f8adc21ba8b7505881b86db29d2dca191bd1f8bd Mon Sep 17 00:00:00 2001 From: DustInDark Date: Wed, 29 Jun 2022 00:54:47 +0900 Subject: [PATCH 6/6] adjusted #592 change in update_rules #597 --- src/options/update_rules.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/options/update_rules.rs b/src/options/update_rules.rs index 258e02cd..764a2a8a 100644 --- a/src/options/update_rules.rs +++ b/src/options/update_rules.rs @@ -31,6 +31,7 @@ impl UpdateRules { &BufferWriter::stdout(ColorChoice::Always), None, "Attempting to git clone the hayabusa-rules repository into the rules folder.", + true, ) .ok(); // execution git clone of hayabusa-rules repository when failed open hayabusa repository. @@ -211,6 +212,7 @@ impl UpdateRules { "[Updated] {} (Modified: {} | Path: {})", tmp[0], tmp[1], tmp[2] ), + true, ) .ok(); } @@ -225,6 +227,7 @@ impl UpdateRules { &BufferWriter::stdout(ColorChoice::Always), None, "You currently have the latest rules.", + true, ) .ok(); Ok("You currently have the latest rules.".to_string())