diff --git a/contributors.txt b/contributors.txt index 8631af4c..95d6f486 100644 --- a/contributors.txt +++ b/contributors.txt @@ -1,10 +1,9 @@ Hayabusa was possible thanks to the following people (in alphabetical order): Akira Nishikawa (@nishikawaakira): Previous lead developer, core hayabusa rule support, etc... -DustInDark(@hitenkoku): Core developer, project management, sigma count implementation, rule creation, countless feature additions and fixes, etc… Garigariganzy (@garigariganzy31): Developer, event ID statistics implementation, etc... ItiB (@itiB_S144) : Core developer, sigmac hayabusa backend, rule creation, etc... -James Takai / hachiyone(@hach1yon): Current lead developer, tokio multi-threading, sigma aggregation logic, sigmac backend, rule creation, etc… +James Takai / hachiyone(@hach1yon): Current lead developer, tokio multi-threading, sigma aggregation logic, sigmac backend, rule creation, sigma count implementation etc… Kazuminn (@k47_um1n): Developer Yusuke Matsui (@apt773): AD hacking working group leader, rule testing, documentation, research, support, etc... Zach Mathis (@yamatosecurity, Yamato Security Founder): Project leader, tool and concept design, rule creation and tuning, etc… @@ -17,7 +16,6 @@ Nishikawa Akira (@nishikawaakira): Lead Developer kazuminn (@k47_um1n): Core Developer itiB (@itiB_S144): Core Developer James Takai / hachiyone (@hach1yon): Core Developer -DustInDark (@hitenkoku): Core Developer garigariganzy (@garigariganzy31): Developer 7itoh (@yNitocrypto22): Developer dai (@__da13__): Developer diff --git a/src/afterfact.rs b/src/afterfact.rs index 97b1c85f..a19a01b7 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -36,8 +36,8 @@ pub struct DisplayFormat<'a> { pub fn after_fact() { let fn_emit_csv_err = |err: Box| { AlertMessage::alert( - &mut std::io::stderr().lock(), - format!("Failed to write CSV. {}", err), + &mut BufWriter::new(std::io::stderr().lock()), + &format!("Failed to write CSV. {}", err), ) .ok(); process::exit(1); @@ -51,8 +51,8 @@ pub fn after_fact() { Ok(file) => Box::new(BufWriter::new(file)), Err(err) => { AlertMessage::alert( - &mut std::io::stderr().lock(), - format!("Failed to open file. {}", err), + &mut BufWriter::new(std::io::stderr().lock()), + &format!("Failed to open file. {}", err), ) .ok(); process::exit(1); diff --git a/src/detections/configs.rs b/src/detections/configs.rs index 06f60e3d..7f5c37cd 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -5,6 +5,7 @@ use clap::{App, AppSettings, ArgMatches}; use hashbrown::HashMap; use hashbrown::HashSet; use lazy_static::lazy_static; +use std::io::BufWriter; use std::sync::RwLock; lazy_static! { pub static ref CONFIG: RwLock = RwLock::new(ConfigReader::new()); @@ -68,6 +69,7 @@ fn build_app<'a>() -> ArgMatches<'a> { -t --thread-number=[NUMBER] 'Thread number (default: optimal number for performance)' -s --statistics 'Prints statistics of event IDs' -q --quiet 'Quiet mode. Do not display the launch banner' + -Q --quiet-errors 'Quiet errors mode. Do not save error logs.' --contributors 'Prints the list of contributors'"; App::new(&program) .about("Hayabusa: Aiming to be the world's greatest Windows event log analysis tool!") @@ -139,8 +141,8 @@ impl TargetEventTime { Ok(dt) => Some(dt.with_timezone(&Utc)), Err(err) => { AlertMessage::alert( - &mut std::io::stderr().lock(), - format!("starttimeline field: {}", err), + &mut BufWriter::new(std::io::stderr().lock()), + &format!("start-timeline field: {}", err), ) .ok(); None @@ -156,8 +158,8 @@ impl TargetEventTime { Ok(dt) => Some(dt.with_timezone(&Utc)), Err(err) => { AlertMessage::alert( - &mut std::io::stderr().lock(), - format!("endtimeline field: {}", err), + &mut BufWriter::new(std::io::stderr().lock()), + &format!("end-timeline field: {}", err), ) .ok(); None diff --git a/src/detections/detection.rs b/src/detections/detection.rs index 547df80a..a2685b9d 100644 --- a/src/detections/detection.rs +++ b/src/detections/detection.rs @@ -1,7 +1,10 @@ extern crate csv; +use crate::detections::configs; use crate::detections::print::AlertMessage; +use crate::detections::print::ERROR_LOG_STACK; use crate::detections::print::MESSAGES; +use crate::detections::print::QUIET_ERRORS_FLAG; use crate::detections::rule; use crate::detections::rule::AggResult; use crate::detections::rule::RuleNode; @@ -11,9 +14,9 @@ use crate::yaml::ParseYaml; use hashbrown; use serde_json::Value; use std::collections::HashMap; -use tokio::{runtime::Runtime, spawn, task::JoinHandle}; - +use std::io::BufWriter; use std::sync::Arc; +use tokio::{runtime::Runtime, spawn, task::JoinHandle}; const DIRPATH_RULES: &str = "rules"; @@ -57,11 +60,16 @@ impl Detection { let result_readdir = rulefile_loader.read_dir(rulespath.unwrap_or(DIRPATH_RULES), &level, exclude_ids); if result_readdir.is_err() { - AlertMessage::alert( - &mut std::io::stderr().lock(), - format!("{}", result_readdir.unwrap_err()), - ) - .ok(); + let errmsg = format!("{}", result_readdir.unwrap_err()); + if configs::CONFIG.read().unwrap().args.is_present("verbose") { + AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &errmsg).ok(); + } + if !*QUIET_ERRORS_FLAG { + ERROR_LOG_STACK + .lock() + .unwrap() + .push(format!("[ERROR] {}", errmsg)); + } return vec![]; } let mut parseerror_count = rulefile_loader.errorrule_count; @@ -75,10 +83,10 @@ impl Detection { err_msgs_result.err().iter().for_each(|err_msgs| { let errmsg_body = format!("Failed to parse rule file. (FilePath : {})", rule.rulepath); - AlertMessage::warn(&mut std::io::stdout().lock(), errmsg_body).ok(); + AlertMessage::warn(&mut std::io::stdout().lock(), &errmsg_body).ok(); err_msgs.iter().for_each(|err_msg| { - AlertMessage::warn(&mut std::io::stdout().lock(), err_msg.to_string()).ok(); + AlertMessage::warn(&mut std::io::stdout().lock(), err_msg).ok(); }); parseerror_count += 1; println!(""); // 一行開けるためのprintln diff --git a/src/detections/print.rs b/src/detections/print.rs index 594aa504..380c2858 100644 --- a/src/detections/print.rs +++ b/src/detections/print.rs @@ -2,13 +2,18 @@ extern crate lazy_static; use crate::detections::configs; use crate::detections::utils; use crate::detections::utils::get_serde_number_to_string; -use chrono::{DateTime, TimeZone, Utc}; +use chrono::{DateTime, Local, TimeZone, Utc}; use lazy_static::lazy_static; use regex::Regex; use serde_json::Value; use std::collections::BTreeMap; use std::collections::HashMap; +use std::env; +use std::fs::create_dir; +use std::fs::File; +use std::io::BufWriter; use std::io::{self, Write}; +use std::path::Path; use std::sync::Mutex; #[derive(Debug)] @@ -32,6 +37,16 @@ pub struct AlertMessage {} lazy_static! { pub static ref MESSAGES: Mutex = Mutex::new(Message::new()); pub static ref ALIASREGEX: Regex = Regex::new(r"%[a-zA-Z0-9-_]+%").unwrap(); + pub static ref ERROR_LOG_PATH: String = format!( + "./logs/errorlog-{}.log", + Local::now().format("%Y%m%d_%H%M%S") + ); + pub static ref QUIET_ERRORS_FLAG: bool = configs::CONFIG + .read() + .unwrap() + .args + .is_present("quiet-errors"); + pub static ref ERROR_LOG_STACK: Mutex> = Mutex::new(Vec::new()); } impl Message { @@ -180,10 +195,48 @@ impl Message { } impl AlertMessage { - pub fn alert(w: &mut W, contents: String) -> io::Result<()> { + ///対象のディレクトリが存在することを確認後、最初の定型文を追加して、ファイルのbufwriterを返す関数 + pub fn create_error_log(path_str: String) { + if *QUIET_ERRORS_FLAG { + return; + } + let path = Path::new(&path_str); + if !path.parent().unwrap().exists() { + create_dir(path.parent().unwrap()).ok(); + } + let mut error_log_writer = BufWriter::new(File::create(path).unwrap()); + error_log_writer + .write( + format!( + "user input: {:?}\n", + format_args!( + "{}", + env::args() + .map(|arg| arg) + .collect::>() + .join(" ") + ) + ) + .as_bytes(), + ) + .unwrap(); + for error_log in ERROR_LOG_STACK.lock().unwrap().iter() { + writeln!(error_log_writer, "{}", error_log).ok(); + } + println!( + "Errors were generated. Please check {} for details.", + ERROR_LOG_PATH.to_string() + ); + println!(""); + } + + /// ERRORメッセージを表示する関数 + pub fn alert(w: &mut W, contents: &String) -> io::Result<()> { writeln!(w, "[ERROR] {}", contents) } - pub fn warn(w: &mut W, contents: String) -> io::Result<()> { + + /// WARNメッセージを表示する関数 + pub fn warn(w: &mut W, contents: &String) -> io::Result<()> { writeln!(w, "[WARN] {}", contents) } } @@ -192,6 +245,7 @@ impl AlertMessage { mod tests { use crate::detections::print::{AlertMessage, Message}; use serde_json::Value; + use std::io::BufWriter; #[test] fn test_create_and_append_message() { @@ -304,17 +358,21 @@ mod tests { #[test] fn test_error_message() { let input = "TEST!"; - let stdout = std::io::stdout(); - let mut stdout = stdout.lock(); - AlertMessage::alert(&mut stdout, input.to_string()).expect("[ERROR] TEST!"); + AlertMessage::alert( + &mut BufWriter::new(std::io::stdout().lock()), + &input.to_string(), + ) + .expect("[ERROR] TEST!"); } #[test] fn test_warn_message() { let input = "TESTWarn!"; - let stdout = std::io::stdout(); - let mut stdout = stdout.lock(); - AlertMessage::alert(&mut stdout, input.to_string()).expect("[WARN] TESTWarn!"); + AlertMessage::warn( + &mut BufWriter::new(std::io::stdout().lock()), + &input.to_string(), + ) + .expect("[WARN] TESTWarn!"); } #[test] diff --git a/src/detections/rule/count.rs b/src/detections/rule/count.rs index c42ddcc8..a2226c37 100644 --- a/src/detections/rule/count.rs +++ b/src/detections/rule/count.rs @@ -1,10 +1,14 @@ +use crate::detections::configs; use crate::detections::print::AlertMessage; +use crate::detections::print::ERROR_LOG_STACK; +use crate::detections::print::QUIET_ERRORS_FLAG; use crate::detections::rule::AggResult; use crate::detections::rule::Message; use crate::detections::rule::RuleNode; use chrono::{DateTime, TimeZone, Utc}; use hashbrown::HashMap; use serde_json::Value; +use std::io::BufWriter; use std::num::ParseIntError; use std::path::Path; @@ -65,30 +69,35 @@ fn get_alias_value_in_record( return Some(value.to_string().replace("\"", "")); } None => { - AlertMessage::alert( - &mut std::io::stderr().lock(), - match is_by_alias { - true => format!( - "count by clause alias value not found in count process. rule file:{} EventID:{}", - Path::new(&rule.rulepath) - .file_name() - .unwrap() - .to_str() - .unwrap(), - utils::get_event_value(&utils::get_event_id_key(), record).unwrap() - ), - false => format!( - "count field clause alias value not found in count process. rule file:{} EventID:{}", - Path::new(&rule.rulepath) - .file_name() - .unwrap() - .to_str() - .unwrap(), - utils::get_event_value(&utils::get_event_id_key(), record).unwrap() - ), - }, - ) - .ok(); + let errmsg = match is_by_alias { + true => format!( + "count by clause alias value not found in count process. rule file:{} EventID:{}", + Path::new(&rule.rulepath) + .file_name() + .unwrap() + .to_str() + .unwrap(), + utils::get_event_value(&utils::get_event_id_key(), record).unwrap() + ), + false => format!( + "count field clause alias value not found in count process. rule file:{} EventID:{}", + Path::new(&rule.rulepath) + .file_name() + .unwrap() + .to_str() + .unwrap(), + utils::get_event_value(&utils::get_event_id_key(), record).unwrap() + ), + }; + if configs::CONFIG.read().unwrap().args.is_present("verbose") { + AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &errmsg).ok(); + } + if !*QUIET_ERRORS_FLAG { + ERROR_LOG_STACK + .lock() + .unwrap() + .push(format!("[ERROR] {}", errmsg)); + } return None; } }; @@ -181,11 +190,16 @@ impl TimeFrameInfo { ttype = "d".to_owned(); tnum.retain(|c| c != 'd'); } else { - AlertMessage::alert( - &mut std::io::stderr().lock(), - format!("Timeframe is invalid. Input value:{}", value), - ) - .ok(); + let errmsg = format!("Timeframe is invalid. Input value:{}", value); + if configs::CONFIG.read().unwrap().args.is_present("verbose") { + AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &errmsg).ok(); + } + if !*QUIET_ERRORS_FLAG { + ERROR_LOG_STACK + .lock() + .unwrap() + .push(format!("[ERROR] {}", errmsg)); + } } return TimeFrameInfo { timetype: ttype, @@ -214,11 +228,16 @@ pub fn get_sec_timeframe(rule: &RuleNode) -> Option { } } Err(err) => { - AlertMessage::alert( - &mut std::io::stderr().lock(), - format!("Timeframe number is invalid. timeframe.{}", err), - ) - .ok(); + let errmsg = format!("Timeframe number is invalid. timeframe. {}", err); + if configs::CONFIG.read().unwrap().args.is_present("verbose") { + AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &errmsg).ok(); + } + if !*QUIET_ERRORS_FLAG { + ERROR_LOG_STACK + .lock() + .unwrap() + .push(format!("[ERROR] {}", errmsg.to_string())); + } return Option::None; } } diff --git a/src/main.rs b/src/main.rs index 4ac01d18..2910353a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,6 +6,9 @@ use chrono::{DateTime, Local}; use evtx::{EvtxParser, ParserSettings}; use hayabusa::detections::detection::{self, EvtxRecordInfo}; use hayabusa::detections::print::AlertMessage; +use hayabusa::detections::print::ERROR_LOG_PATH; +use hayabusa::detections::print::ERROR_LOG_STACK; +use hayabusa::detections::print::QUIET_ERRORS_FLAG; use hayabusa::detections::rule::{get_detection_keys, RuleNode}; use hayabusa::filter; use hayabusa::omikuji::Omikuji; @@ -16,6 +19,8 @@ use pbr::ProgressBar; use serde_json::Value; use std::collections::{HashMap, HashSet}; use std::fmt::Display; +use std::io::BufWriter; +use std::path::Path; use std::sync::Arc; use std::{ fs::{self, File}, @@ -66,11 +71,24 @@ impl App { ); return; } + if let Some(csv_path) = configs::CONFIG.read().unwrap().args.value_of("output") { + if Path::new(csv_path).exists() { + AlertMessage::alert( + &mut BufWriter::new(std::io::stderr().lock()), + &format!( + " The file {} already exists. Please specify a different filename.", + csv_path + ), + ) + .ok(); + return; + } + } if let Some(filepath) = configs::CONFIG.read().unwrap().args.value_of("filepath") { if !filepath.ends_with(".evtx") { AlertMessage::alert( - &mut std::io::stderr().lock(), - "--filepath only accepts .evtx files.".to_owned(), + &mut BufWriter::new(std::io::stderr().lock()), + &"--filepath only accepts .evtx files.".to_string(), ) .ok(); return; @@ -80,8 +98,8 @@ impl App { let evtx_files = self.collect_evtxfiles(&directory); if evtx_files.len() == 0 { AlertMessage::alert( - &mut std::io::stderr().lock(), - "No .evtx files were found.".to_owned(), + &mut BufWriter::new(std::io::stderr().lock()), + &"No .evtx files were found.".to_string(), ) .ok(); return; @@ -100,14 +118,26 @@ impl App { let analysis_duration = analysis_end_time.signed_duration_since(analysis_start_time); println!("Elapsed Time: {}", &analysis_duration.hhmmssxxx()); println!(""); + + // Qオプションを付けた場合もしくはパースのエラーがない場合はerrorのstackが9となるのでエラーログファイル自体が生成されない。 + if ERROR_LOG_STACK.lock().unwrap().len() > 0 { + AlertMessage::create_error_log(ERROR_LOG_PATH.to_string()); + } } fn collect_evtxfiles(&self, dirpath: &str) -> Vec { let entries = fs::read_dir(dirpath); if entries.is_err() { - let stderr = std::io::stderr(); - let mut stderr = stderr.lock(); - AlertMessage::alert(&mut stderr, format!("{}", entries.unwrap_err())).ok(); + let errmsg = format!("{}", entries.unwrap_err()); + if configs::CONFIG.read().unwrap().args.is_present("verbose") { + AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &errmsg).ok(); + } + if !*QUIET_ERRORS_FLAG { + ERROR_LOG_STACK + .lock() + .unwrap() + .push(format!("[ERROR] {}", errmsg)); + } return vec![]; } @@ -139,7 +169,11 @@ impl App { match fs::read_to_string("./contributors.txt") { Ok(contents) => println!("{}", contents), Err(err) => { - AlertMessage::alert(&mut std::io::stderr().lock(), format!("{}", err)).ok(); + AlertMessage::alert( + &mut BufWriter::new(std::io::stderr().lock()), + &format!("{}", err), + ) + .ok(); } } } @@ -207,7 +241,16 @@ impl App { evtx_filepath, record_result.unwrap_err() ); - AlertMessage::alert(&mut std::io::stderr().lock(), errmsg).ok(); + if configs::CONFIG.read().unwrap().args.is_present("verbose") { + AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &errmsg) + .ok(); + } + if !*QUIET_ERRORS_FLAG { + ERROR_LOG_STACK + .lock() + .unwrap() + .push(format!("[ERROR] {}", errmsg)); + } continue; } diff --git a/src/yaml.rs b/src/yaml.rs index 4411afb2..005ceedf 100644 --- a/src/yaml.rs +++ b/src/yaml.rs @@ -3,11 +3,14 @@ extern crate yaml_rust; use crate::detections::configs; use crate::detections::print::AlertMessage; +use crate::detections::print::ERROR_LOG_STACK; +use crate::detections::print::QUIET_ERRORS_FLAG; use crate::filter::RuleExclude; use std::collections::HashMap; use std::ffi::OsStr; use std::fs; use std::io; +use std::io::BufWriter; use std::io::{BufReader, Read}; use std::path::{Path, PathBuf}; use yaml_rust::Yaml; @@ -71,14 +74,20 @@ impl ParseYaml { // 個別のファイルの読み込みは即終了としない。 let read_content = self.read_file(path); if read_content.is_err() { - AlertMessage::warn( - &mut std::io::stdout().lock(), - format!( - "fail to read file: {}\n{} ", - entry.path().display(), - read_content.unwrap_err() - ), - )?; + let errmsg = format!( + "fail to read file: {}\n{} ", + entry.path().display(), + read_content.unwrap_err() + ); + if configs::CONFIG.read().unwrap().args.is_present("verbose") { + AlertMessage::warn(&mut BufWriter::new(std::io::stderr().lock()), &errmsg)?; + } + if !*QUIET_ERRORS_FLAG { + ERROR_LOG_STACK + .lock() + .unwrap() + .push(format!("[WARN] {}", errmsg)); + } self.errorrule_count += 1; return io::Result::Ok(ret); } @@ -86,14 +95,20 @@ impl ParseYaml { // ここも個別のファイルの読み込みは即終了としない。 let yaml_contents = YamlLoader::load_from_str(&read_content.unwrap()); if yaml_contents.is_err() { - AlertMessage::warn( - &mut std::io::stdout().lock(), - format!( - "Failed to parse yml: {}\n{} ", - entry.path().display(), - yaml_contents.unwrap_err() - ), - )?; + let errmsg = format!( + "Failed to parse yml: {}\n{} ", + entry.path().display(), + yaml_contents.unwrap_err() + ); + if configs::CONFIG.read().unwrap().args.is_present("verbose") { + AlertMessage::warn(&mut BufWriter::new(std::io::stderr().lock()), &errmsg)?; + } + if !*QUIET_ERRORS_FLAG { + ERROR_LOG_STACK + .lock() + .unwrap() + .push(format!("[WARN] {}", errmsg)); + } self.errorrule_count += 1; return io::Result::Ok(ret); } @@ -176,6 +191,8 @@ impl ParseYaml { #[cfg(test)] mod tests { + use crate::detections::print::AlertMessage; + use crate::detections::print::ERROR_LOG_PATH; use crate::filter; use crate::yaml; use crate::yaml::RuleExclude; @@ -185,6 +202,8 @@ mod tests { #[test] fn test_read_dir_yaml() { + AlertMessage::create_error_log(ERROR_LOG_PATH.to_string()); + let mut yaml = yaml::ParseYaml::new(); let exclude_ids = RuleExclude { no_use_rule: HashSet::new(), @@ -275,6 +294,8 @@ mod tests { } #[test] fn test_all_exclude_rules_file() { + AlertMessage::create_error_log(ERROR_LOG_PATH.to_string()); + let mut yaml = yaml::ParseYaml::new(); let path = Path::new("test_files/rules/yaml"); yaml.read_dir(path.to_path_buf(), &"", &filter::exclude_ids()) @@ -283,6 +304,8 @@ mod tests { } #[test] fn test_none_exclude_rules_file() { + AlertMessage::create_error_log(ERROR_LOG_PATH.to_string()); + let mut yaml = yaml::ParseYaml::new(); let path = Path::new("test_files/rules/yaml"); let exclude_ids = RuleExclude {