From fa86a9a0279bd4f245b2c4dbe96985ef8b37bd8b Mon Sep 17 00:00:00 2001 From: DustInDark Date: Tue, 29 Mar 2022 13:09:54 +0900 Subject: [PATCH] Fearture/ added output update result#410 (#452) * add git2 crate #391 * added Update option #391 * updated readme #391 * fixed cargo.lock * fixed option if-statement #391 * changed utc short option and rule-update short option #391 * updated readme * updated readme * fixed -u long option & version number update #391 * added fast-forwarding rules repository #391 * updated command line option #391 * moved output logo prev update rule * fixed readme #391 * removed recursive option in readme * changed rules update from clone and pull to submodule update #391 * fixed document * changed unnecessary clone recursively to clone only * English message update. * cargo fmt * English message update. ( 4657c35e5cab754334423be2381d1f5a3c0532b4 cherry-pick) * added create rules folder when rules folder is not exist * fixed gitmodules github-rules url from ssh to https * added output of updated file #420 * fixed error #410 * changed update rule list seq * added test * fixed output #410 * fixed output and fixed output date field when modified field is lacked #410 * fixed compile error * fixed output - added enter after Latest rule update output - added output when no exist new rule - fixed Latest rule update date format - changed output from 'Latest rule update' to 'Latest rules update' * fixed compile error * changed modified date source from rules folder to each yml rule file * formatting use chrono in main.rs * merge develop clippy ci * fixed output when no update rule #410 - removed Latest rule update - no output "Rules update successfully" when No rule changed * Change English Co-authored-by: Tanaka Zakku <71482215+YamatoSecurity@users.noreply.github.com> --- src/main.rs | 166 +++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 139 insertions(+), 27 deletions(-) diff --git a/src/main.rs b/src/main.rs index 06d0fb01..3461b1d9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,10 +5,10 @@ extern crate serde_derive; #[cfg(target_os = "windows")] extern crate static_vcruntime; -use chrono::Datelike; -use chrono::{DateTime, Local}; +use chrono::{DateTime, Datelike, Local, TimeZone}; use evtx::{EvtxParser, ParserSettings}; use git2::Repository; +use hashbrown::{HashMap, HashSet}; use hayabusa::detections::detection::{self, EvtxRecordInfo}; use hayabusa::detections::print::{ AlertMessage, ERROR_LOG_PATH, ERROR_LOG_STACK, QUIET_ERRORS_FLAG, STATISTICS_FLAG, @@ -16,18 +16,20 @@ use hayabusa::detections::print::{ use hayabusa::detections::rule::{get_detection_keys, RuleNode}; use hayabusa::filter; use hayabusa::omikuji::Omikuji; +use hayabusa::yaml::ParseYaml; use hayabusa::{afterfact::after_fact, detections::utils}; use hayabusa::{detections::configs, timeline::timelines::Timeline}; use hhmmss::Hhmmss; use pbr::ProgressBar; use serde_json::Value; -use std::collections::{HashMap, HashSet}; +use std::cmp::Ordering; use std::ffi::OsStr; use std::fmt::Display; use std::fs::create_dir; use std::io::BufWriter; use std::path::Path; use std::sync::Arc; +use std::time::SystemTime; use std::{ fs::{self, File}, path::PathBuf, @@ -86,7 +88,11 @@ impl App { .is_present("update-rules") { match self.update_rules() { - Ok(_ok) => println!("Rules updated successfully."), + Ok(output) => { + if output != "You currently have the latest rules." { + println!("Rules updated successfully."); + } + } Err(e) => { AlertMessage::alert( &mut BufWriter::new(std::io::stderr().lock()), @@ -95,6 +101,7 @@ impl App { .ok(); } } + println!(); return; } if !Path::new("./config").exists() { @@ -506,22 +513,29 @@ impl App { } /// update rules(hayabusa-rules subrepository) - fn update_rules(&self) -> Result<(), git2::Error> { + fn update_rules(&self) -> Result { + let mut result; + let mut prev_modified_time: SystemTime = SystemTime::UNIX_EPOCH; + let mut prev_modified_rules: HashSet = HashSet::default(); let hayabusa_repo = Repository::open(Path::new(".")); - let hayabusa_rule_repo = Repository::open(Path::new("./rules")); + let hayabusa_rule_repo = Repository::open(Path::new("rules")); if hayabusa_repo.is_err() && hayabusa_rule_repo.is_err() { println!( "Attempting to git clone the hayabusa-rules repository into the rules folder." ); // レポジトリが開けなかった段階でhayabusa rulesのgit cloneを実施する - self.clone_rules() + result = self.clone_rules(); } else if hayabusa_rule_repo.is_ok() { // rulesのrepositoryが確認できる場合 // origin/mainのfetchができなくなるケースはネットワークなどのケースが考えられるため、git cloneは実施しない - self.pull_repository(hayabusa_rule_repo.unwrap()) + prev_modified_rules = self.get_updated_rules("rules", &prev_modified_time); + prev_modified_time = fs::metadata("rules").unwrap().modified().unwrap(); + result = self.pull_repository(hayabusa_rule_repo.unwrap()); } else { - //hayabusa repositoryがあればsubmodule情報もあると思われるのでupdate - let rules_path = Path::new("./rules"); + // hayabusa-rulesのrepositoryがrulesに存在しない場合 + // hayabusa repositoryがあればsubmodule情報もあると思われるのでupdate + prev_modified_time = fs::metadata("rules").unwrap().modified().unwrap(); + let rules_path = Path::new("rules"); if !rules_path.exists() { create_dir(rules_path).ok(); } @@ -533,28 +547,31 @@ impl App { for mut submodule in submodules { submodule.update(true, None)?; let submodule_repo = submodule.open()?; - match self.pull_repository(submodule_repo) { - Ok(it) => it, - Err(e) => { - AlertMessage::alert( - &mut BufWriter::new(std::io::stderr().lock()), - &format!("Failed submodule update. {}", e), - ) - .ok(); - is_success_submodule_update = false; - } + if let Err(e) = self.pull_repository(submodule_repo) { + AlertMessage::alert( + &mut BufWriter::new(std::io::stderr().lock()), + &format!("Failed submodule update. {}", e), + ) + .ok(); + is_success_submodule_update = false; } } if is_success_submodule_update { - Ok(()) + result = Ok("Successed submodule update".to_string()); } else { - Err(git2::Error::from_str(&String::default())) + result = Err(git2::Error::from_str(&String::default())); } } + if result.is_ok() { + let updated_modified_rules = self.get_updated_rules("rules", &prev_modified_time); + result = + self.print_diff_modified_rule_dates(prev_modified_rules, updated_modified_rules); + } + result } /// Pull(fetch and fast-forward merge) repositoryto input_repo. - fn pull_repository(&self, input_repo: Repository) -> Result<(), git2::Error> { + fn pull_repository(&self, input_repo: Repository) -> Result { match input_repo .find_remote("origin")? .fetch(&["main"], None, None) @@ -572,13 +589,13 @@ impl App { let fetch_commit = input_repo.reference_to_annotated_commit(&fetch_head)?; let analysis = input_repo.merge_analysis(&[&fetch_commit])?; if analysis.0.is_up_to_date() { - Ok(()) + Ok("Already up to date".to_string()) } else if analysis.0.is_fast_forward() { let mut reference = input_repo.find_reference("refs/heads/main")?; reference.set_target(fetch_commit.id(), "Fast-Forward")?; input_repo.set_head("refs/heads/main")?; input_repo.checkout_head(Some(git2::build::CheckoutBuilder::default().force()))?; - Ok(()) + Ok("Finished fast forward merge.".to_string()) } else if analysis.0.is_normal() { AlertMessage::alert( &mut BufWriter::new(std::io::stderr().lock()), @@ -592,14 +609,14 @@ impl App { } /// git clone でhauyabusa-rules レポジトリをrulesフォルダにgit cloneする関数 - fn clone_rules(&self) -> Result<(), git2::Error> { + fn clone_rules(&self) -> Result { match Repository::clone( "https://github.com/Yamato-Security/hayabusa-rules.git", "rules", ) { Ok(_repo) => { println!("Finished cloning the hayabusa-rules repository."); - Ok(()) + Ok("Finished clone".to_string()) } Err(e) => { AlertMessage::alert( @@ -614,11 +631,90 @@ impl App { } } } + + /// Create rules folder files Hashset. Format is "[rule title in yaml]|[filepath]|[filemodified date]|[rule type in yaml]" + fn get_updated_rules( + &self, + rule_folder_path: &str, + target_date: &SystemTime, + ) -> HashSet { + let mut rulefile_loader = ParseYaml::new(); + // level in read_dir is hard code to check all rules. + rulefile_loader + .read_dir( + rule_folder_path, + "INFORMATIONAL", + &filter::RuleExclude { + no_use_rule: HashSet::new(), + }, + ) + .ok(); + + let hash_set_keys: HashSet = rulefile_loader + .files + .into_iter() + .filter_map(|(filepath, yaml)| { + let file_modified_date = fs::metadata(&filepath).unwrap().modified().unwrap(); + + if file_modified_date.cmp(target_date).is_gt() { + let yaml_date = yaml["date"].as_str().unwrap_or("-"); + return Option::Some(format!( + "{}|{}|{}|{}", + yaml["title"].as_str().unwrap_or(&String::default()), + yaml["modified"].as_str().unwrap_or(yaml_date), + &filepath, + yaml["ruletype"].as_str().unwrap_or("Other") + )); + } + Option::None + }) + .collect(); + hash_set_keys + } + + /// print updated rule files. + fn print_diff_modified_rule_dates( + &self, + prev_sets: HashSet, + updated_sets: HashSet, + ) -> Result { + let diff = updated_sets.difference(&prev_sets); + let mut update_count_by_rule_type: HashMap = HashMap::new(); + let mut latest_update_date = Local.timestamp(0, 0); + for diff_key in diff { + let tmp: Vec<&str> = diff_key.split('|').collect(); + let file_modified_date = fs::metadata(&tmp[2]).unwrap().modified().unwrap(); + + let dt_local: DateTime = file_modified_date.into(); + + if latest_update_date.cmp(&dt_local) == Ordering::Less { + latest_update_date = dt_local; + } + *update_count_by_rule_type + .entry(tmp[3].to_string()) + .or_insert(0b0) += 1; + println!( + "[Updated] {} (Modified: {} | Path: {})", + tmp[0], tmp[1], tmp[2] + ); + } + println!(); + for (key, value) in &update_count_by_rule_type { + println!("Updated {} rules: {}", key, value); + } + if !&update_count_by_rule_type.is_empty() { + Ok("Rule updated".to_string()) + } else { + println!("You currently have the latest rules."); + Ok("You currently have the latest rules.".to_string()) + } + } } #[cfg(test)] mod tests { use crate::App; + use std::time::SystemTime; #[test] fn test_collect_evtxfiles() { @@ -635,4 +731,20 @@ mod tests { assert_eq!(is_contains, &true); }) } + + #[test] + fn test_get_updated_rules() { + let app = App::new(); + + let prev_modified_time: SystemTime = SystemTime::UNIX_EPOCH; + + let prev_modified_rules = + app.get_updated_rules("test_files/rules/level_yaml", &prev_modified_time); + assert_eq!(prev_modified_rules.len(), 5); + + let target_time: SystemTime = SystemTime::now(); + let prev_modified_rules2 = + app.get_updated_rules("test_files/rules/level_yaml", &target_time); + assert_eq!(prev_modified_rules2.len(), 0); + } }