Merge pull request #316 from Yamato-Security/feature/output_error_log_file_and_options#301

fixed #301 #303 #309

contributors.txt

@@ -1,10 +1,9 @@
 Hayabusa was possible thanks to the following people (in alphabetical order):
 
 Akira Nishikawa (@nishikawaakira): Previous lead developer, core hayabusa rule support, etc...
-DustInDark(@hitenkoku): Core developer, project management, sigma count implementation, rule creation, countless feature additions and fixes, etc…
 Garigariganzy (@garigariganzy31): Developer, event ID statistics implementation, etc... 
 ItiB (@itiB_S144) :  Core developer, sigmac hayabusa backend, rule creation, etc...
-James Takai / hachiyone(@hach1yon): Current lead developer, tokio multi-threading, sigma aggregation logic, sigmac backend, rule creation, etc…
+James Takai / hachiyone(@hach1yon): Current lead developer, tokio multi-threading, sigma aggregation logic, sigmac backend, rule creation, sigma count implementation etc…
 Kazuminn (@k47_um1n): Developer
 Yusuke Matsui (@apt773): AD hacking working group leader, rule testing, documentation, research, support, etc...
 Zach Mathis (@yamatosecurity, Yamato Security Founder): Project leader, tool and concept design, rule creation and tuning, etc…
@@ -17,7 +16,6 @@ Nishikawa Akira (@nishikawaakira): Lead Developer
 kazuminn (@k47_um1n): Core Developer
 itiB (@itiB_S144): Core Developer
 James Takai / hachiyone (@hach1yon): Core Developer
-DustInDark (@hitenkoku): Core Developer
 garigariganzy (@garigariganzy31): Developer
 7itoh (@yNitocrypto22): Developer
 dai (@__da13__): Developer

src/afterfact.rs

@@ -36,8 +36,8 @@ pub struct DisplayFormat<'a> {
 pub fn after_fact() {
     let fn_emit_csv_err = |err: Box<dyn Error>| {
         AlertMessage::alert(
-            &mut std::io::stderr().lock(),
+            &mut BufWriter::new(std::io::stderr().lock()),
-            format!("Failed to write CSV. {}", err),
+            &format!("Failed to write CSV. {}", err),
         )
         .ok();
         process::exit(1);
@@ -51,8 +51,8 @@ pub fn after_fact() {
                 Ok(file) => Box::new(BufWriter::new(file)),
                 Err(err) => {
                     AlertMessage::alert(
-                        &mut std::io::stderr().lock(),
+                        &mut BufWriter::new(std::io::stderr().lock()),
-                        format!("Failed to open file. {}", err),
+                        &format!("Failed to open file. {}", err),
                     )
                     .ok();
                     process::exit(1);

src/detections/configs.rs

@@ -5,6 +5,7 @@ use clap::{App, AppSettings, ArgMatches};
 use hashbrown::HashMap;
 use hashbrown::HashSet;
 use lazy_static::lazy_static;
+use std::io::BufWriter;
 use std::sync::RwLock;
 lazy_static! {
     pub static ref CONFIG: RwLock<ConfigReader> = RwLock::new(ConfigReader::new());
@@ -68,6 +69,7 @@ fn build_app<'a>() -> ArgMatches<'a> {
     -t --thread-number=[NUMBER] 'Thread number (default: optimal number for performance)'
     -s --statistics 'Prints statistics of event IDs'
     -q --quiet 'Quiet mode. Do not display the launch banner'
+    -Q --quiet-errors 'Quiet errors mode. Do not save error logs.'
     --contributors 'Prints the list of contributors'";
     App::new(&program)
         .about("Hayabusa: Aiming to be the world's greatest Windows event log analysis tool!")
@@ -139,8 +141,8 @@ impl TargetEventTime {
                 Ok(dt) => Some(dt.with_timezone(&Utc)),
                 Err(err) => {
                     AlertMessage::alert(
-                        &mut std::io::stderr().lock(),
+                        &mut BufWriter::new(std::io::stderr().lock()),
-                        format!("starttimeline field: {}", err),
+                        &format!("start-timeline field: {}", err),
                     )
                     .ok();
                     None
@@ -156,8 +158,8 @@ impl TargetEventTime {
             Ok(dt) => Some(dt.with_timezone(&Utc)),
             Err(err) => {
                     AlertMessage::alert(
-                        &mut std::io::stderr().lock(),
+                        &mut BufWriter::new(std::io::stderr().lock()),
-                        format!("endtimeline field: {}", err),
+                        &format!("end-timeline field: {}", err),
                     )
                     .ok();
                     None

src/detections/detection.rs

@@ -1,7 +1,10 @@
 extern crate csv;
 
+use crate::detections::configs;
 use crate::detections::print::AlertMessage;
+use crate::detections::print::ERROR_LOG_STACK;
 use crate::detections::print::MESSAGES;
+use crate::detections::print::QUIET_ERRORS_FLAG;
 use crate::detections::rule;
 use crate::detections::rule::AggResult;
 use crate::detections::rule::RuleNode;
@@ -11,9 +14,9 @@ use crate::yaml::ParseYaml;
 use hashbrown;
 use serde_json::Value;
 use std::collections::HashMap;
-use tokio::{runtime::Runtime, spawn, task::JoinHandle};
+use std::io::BufWriter;
-
 use std::sync::Arc;
+use tokio::{runtime::Runtime, spawn, task::JoinHandle};
 
 const DIRPATH_RULES: &str = "rules";
 
@@ -57,11 +60,16 @@ impl Detection {
         let result_readdir =
             rulefile_loader.read_dir(rulespath.unwrap_or(DIRPATH_RULES), &level, exclude_ids);
         if result_readdir.is_err() {
-            AlertMessage::alert(
+            let errmsg = format!("{}", result_readdir.unwrap_err());
-                &mut std::io::stderr().lock(),
+            if configs::CONFIG.read().unwrap().args.is_present("verbose") {
-                format!("{}", result_readdir.unwrap_err()),
+                AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &errmsg).ok();
-            )
+            }
-            .ok();
+            if !*QUIET_ERRORS_FLAG {
+                ERROR_LOG_STACK
+                    .lock()
+                    .unwrap()
+                    .push(format!("[ERROR] {}", errmsg));
+            }
             return vec![];
         }
         let mut parseerror_count = rulefile_loader.errorrule_count;
@@ -75,10 +83,10 @@ impl Detection {
             err_msgs_result.err().iter().for_each(|err_msgs| {
                 let errmsg_body =
                     format!("Failed to parse rule file. (FilePath : {})", rule.rulepath);
-                AlertMessage::warn(&mut std::io::stdout().lock(), errmsg_body).ok();
+                AlertMessage::warn(&mut std::io::stdout().lock(), &errmsg_body).ok();
 
                 err_msgs.iter().for_each(|err_msg| {
-                    AlertMessage::warn(&mut std::io::stdout().lock(), err_msg.to_string()).ok();
+                    AlertMessage::warn(&mut std::io::stdout().lock(), err_msg).ok();
                 });
                 parseerror_count += 1;
                 println!(""); // 一行開けるためのprintln

src/detections/print.rs

@@ -2,13 +2,18 @@ extern crate lazy_static;
 use crate::detections::configs;
 use crate::detections::utils;
 use crate::detections::utils::get_serde_number_to_string;
-use chrono::{DateTime, TimeZone, Utc};
+use chrono::{DateTime, Local, TimeZone, Utc};
 use lazy_static::lazy_static;
 use regex::Regex;
 use serde_json::Value;
 use std::collections::BTreeMap;
 use std::collections::HashMap;
+use std::env;
+use std::fs::create_dir;
+use std::fs::File;
+use std::io::BufWriter;
 use std::io::{self, Write};
+use std::path::Path;
 use std::sync::Mutex;
 
 #[derive(Debug)]
@@ -32,6 +37,16 @@ pub struct AlertMessage {}
 lazy_static! {
     pub static ref MESSAGES: Mutex<Message> = Mutex::new(Message::new());
     pub static ref ALIASREGEX: Regex = Regex::new(r"%[a-zA-Z0-9-_]+%").unwrap();
+    pub static ref ERROR_LOG_PATH: String = format!(
+        "./logs/errorlog-{}.log",
+        Local::now().format("%Y%m%d_%H%M%S")
+    );
+    pub static ref QUIET_ERRORS_FLAG: bool = configs::CONFIG
+        .read()
+        .unwrap()
+        .args
+        .is_present("quiet-errors");
+    pub static ref ERROR_LOG_STACK: Mutex<Vec<String>> = Mutex::new(Vec::new());
 }
 
 impl Message {
@@ -180,10 +195,48 @@ impl Message {
 }
 
 impl AlertMessage {
-    pub fn alert<W: Write>(w: &mut W, contents: String) -> io::Result<()> {
+    ///対象のディレクトリが存在することを確認後、最初の定型文を追加して、ファイルのbufwriterを返す関数
+    pub fn create_error_log(path_str: String) {
+        if *QUIET_ERRORS_FLAG {
+            return;
+        }
+        let path = Path::new(&path_str);
+        if !path.parent().unwrap().exists() {
+            create_dir(path.parent().unwrap()).ok();
+        }
+        let mut error_log_writer = BufWriter::new(File::create(path).unwrap());
+        error_log_writer
+            .write(
+                format!(
+                    "user input: {:?}\n",
+                    format_args!(
+                        "{}",
+                        env::args()
+                            .map(|arg| arg)
+                            .collect::<Vec<String>>()
+                            .join(" ")
+                    )
+                )
+                .as_bytes(),
+            )
+            .unwrap();
+        for error_log in ERROR_LOG_STACK.lock().unwrap().iter() {
+            writeln!(error_log_writer, "{}", error_log).ok();
+        }
+        println!(
+            "Errors were generated. Please check {} for details.",
+            ERROR_LOG_PATH.to_string()
+        );
+        println!("");
+    }
+
+    /// ERRORメッセージを表示する関数
+    pub fn alert<W: Write>(w: &mut W, contents: &String) -> io::Result<()> {
         writeln!(w, "[ERROR] {}", contents)
     }
-    pub fn warn<W: Write>(w: &mut W, contents: String) -> io::Result<()> {
+
+    /// WARNメッセージを表示する関数
+    pub fn warn<W: Write>(w: &mut W, contents: &String) -> io::Result<()> {
         writeln!(w, "[WARN] {}", contents)
     }
 }
@@ -192,6 +245,7 @@ impl AlertMessage {
 mod tests {
     use crate::detections::print::{AlertMessage, Message};
     use serde_json::Value;
+    use std::io::BufWriter;
 
     #[test]
     fn test_create_and_append_message() {
@@ -304,17 +358,21 @@ mod tests {
     #[test]
     fn test_error_message() {
         let input = "TEST!";
-        let stdout = std::io::stdout();
+        AlertMessage::alert(
-        let mut stdout = stdout.lock();
+            &mut BufWriter::new(std::io::stdout().lock()),
-        AlertMessage::alert(&mut stdout, input.to_string()).expect("[ERROR] TEST!");
+            &input.to_string(),
+        )
+        .expect("[ERROR] TEST!");
     }
 
     #[test]
     fn test_warn_message() {
         let input = "TESTWarn!";
-        let stdout = std::io::stdout();
+        AlertMessage::warn(
-        let mut stdout = stdout.lock();
+            &mut BufWriter::new(std::io::stdout().lock()),
-        AlertMessage::alert(&mut stdout, input.to_string()).expect("[WARN] TESTWarn!");
+            &input.to_string(),
+        )
+        .expect("[WARN] TESTWarn!");
     }
 
     #[test]

src/detections/rule/count.rs

@@ -1,10 +1,14 @@
+use crate::detections::configs;
 use crate::detections::print::AlertMessage;
+use crate::detections::print::ERROR_LOG_STACK;
+use crate::detections::print::QUIET_ERRORS_FLAG;
 use crate::detections::rule::AggResult;
 use crate::detections::rule::Message;
 use crate::detections::rule::RuleNode;
 use chrono::{DateTime, TimeZone, Utc};
 use hashbrown::HashMap;
 use serde_json::Value;
+use std::io::BufWriter;
 use std::num::ParseIntError;
 use std::path::Path;
 
@@ -65,9 +69,7 @@ fn get_alias_value_in_record(
             return Some(value.to_string().replace("\"", ""));
         }
         None => {
-            AlertMessage::alert(
+            let errmsg = match is_by_alias {
-                &mut std::io::stderr().lock(),
-                match is_by_alias {
                 true => format!(
           "count by clause alias value not found in count process. rule file:{} EventID:{}",
           Path::new(&rule.rulepath)
@@ -86,9 +88,16 @@ fn get_alias_value_in_record(
             .unwrap(),
           utils::get_event_value(&utils::get_event_id_key(), record).unwrap()
         ),
-                },
+            };
-            )
+            if configs::CONFIG.read().unwrap().args.is_present("verbose") {
-            .ok();
+                AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &errmsg).ok();
+            }
+            if !*QUIET_ERRORS_FLAG {
+                ERROR_LOG_STACK
+                    .lock()
+                    .unwrap()
+                    .push(format!("[ERROR] {}", errmsg));
+            }
             return None;
         }
     };
@@ -181,11 +190,16 @@ impl TimeFrameInfo {
             ttype = "d".to_owned();
             tnum.retain(|c| c != 'd');
         } else {
-            AlertMessage::alert(
+            let errmsg = format!("Timeframe is invalid. Input value:{}", value);
-                &mut std::io::stderr().lock(),
+            if configs::CONFIG.read().unwrap().args.is_present("verbose") {
-                format!("Timeframe is invalid. Input value:{}", value),
+                AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &errmsg).ok();
-            )
+            }
-            .ok();
+            if !*QUIET_ERRORS_FLAG {
+                ERROR_LOG_STACK
+                    .lock()
+                    .unwrap()
+                    .push(format!("[ERROR] {}", errmsg));
+            }
         }
         return TimeFrameInfo {
             timetype: ttype,
@@ -214,11 +228,16 @@ pub fn get_sec_timeframe(rule: &RuleNode) -> Option<i64> {
             }
         }
         Err(err) => {
-            AlertMessage::alert(
+            let errmsg = format!("Timeframe number is invalid. timeframe. {}", err);
-                &mut std::io::stderr().lock(),
+            if configs::CONFIG.read().unwrap().args.is_present("verbose") {
-                format!("Timeframe number is invalid. timeframe.{}", err),
+                AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &errmsg).ok();
-            )
+            }
-            .ok();
+            if !*QUIET_ERRORS_FLAG {
+                ERROR_LOG_STACK
+                    .lock()
+                    .unwrap()
+                    .push(format!("[ERROR] {}", errmsg.to_string()));
+            }
             return Option::None;
         }
     }

src/main.rs

@@ -6,6 +6,9 @@ use chrono::{DateTime, Local};
 use evtx::{EvtxParser, ParserSettings};
 use hayabusa::detections::detection::{self, EvtxRecordInfo};
 use hayabusa::detections::print::AlertMessage;
+use hayabusa::detections::print::ERROR_LOG_PATH;
+use hayabusa::detections::print::ERROR_LOG_STACK;
+use hayabusa::detections::print::QUIET_ERRORS_FLAG;
 use hayabusa::detections::rule::{get_detection_keys, RuleNode};
 use hayabusa::filter;
 use hayabusa::omikuji::Omikuji;
@@ -16,6 +19,8 @@ use pbr::ProgressBar;
 use serde_json::Value;
 use std::collections::{HashMap, HashSet};
 use std::fmt::Display;
+use std::io::BufWriter;
+use std::path::Path;
 use std::sync::Arc;
 use std::{
     fs::{self, File},
@@ -66,11 +71,24 @@ impl App {
             );
             return;
         }
+        if let Some(csv_path) = configs::CONFIG.read().unwrap().args.value_of("output") {
+            if Path::new(csv_path).exists() {
+                AlertMessage::alert(
+                    &mut BufWriter::new(std::io::stderr().lock()),
+                    &format!(
+                        " The file {} already exists. Please specify a different filename.",
+                        csv_path
+                    ),
+                )
+                .ok();
+                return;
+            }
+        }
         if let Some(filepath) = configs::CONFIG.read().unwrap().args.value_of("filepath") {
             if !filepath.ends_with(".evtx") {
                 AlertMessage::alert(
-                    &mut std::io::stderr().lock(),
+                    &mut BufWriter::new(std::io::stderr().lock()),
-                    "--filepath only accepts .evtx files.".to_owned(),
+                    &"--filepath only accepts .evtx files.".to_string(),
                 )
                 .ok();
                 return;
@@ -80,8 +98,8 @@ impl App {
             let evtx_files = self.collect_evtxfiles(&directory);
             if evtx_files.len() == 0 {
                 AlertMessage::alert(
-                    &mut std::io::stderr().lock(),
+                    &mut BufWriter::new(std::io::stderr().lock()),
-                    "No .evtx files were found.".to_owned(),
+                    &"No .evtx files were found.".to_string(),
                 )
                 .ok();
                 return;
@@ -100,14 +118,26 @@ impl App {
         let analysis_duration = analysis_end_time.signed_duration_since(analysis_start_time);
         println!("Elapsed Time: {}", &analysis_duration.hhmmssxxx());
         println!("");
+
+        // Qオプションを付けた場合もしくはパースのエラーがない場合はerrorのstackが9となるのでエラーログファイル自体が生成されない。
+        if ERROR_LOG_STACK.lock().unwrap().len() > 0 {
+            AlertMessage::create_error_log(ERROR_LOG_PATH.to_string());
+        }
     }
 
     fn collect_evtxfiles(&self, dirpath: &str) -> Vec<PathBuf> {
         let entries = fs::read_dir(dirpath);
         if entries.is_err() {
-            let stderr = std::io::stderr();
+            let errmsg = format!("{}", entries.unwrap_err());
-            let mut stderr = stderr.lock();
+            if configs::CONFIG.read().unwrap().args.is_present("verbose") {
-            AlertMessage::alert(&mut stderr, format!("{}", entries.unwrap_err())).ok();
+                AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &errmsg).ok();
+            }
+            if !*QUIET_ERRORS_FLAG {
+                ERROR_LOG_STACK
+                    .lock()
+                    .unwrap()
+                    .push(format!("[ERROR] {}", errmsg));
+            }
             return vec![];
         }
 
@@ -139,7 +169,11 @@ impl App {
         match fs::read_to_string("./contributors.txt") {
             Ok(contents) => println!("{}", contents),
             Err(err) => {
-                AlertMessage::alert(&mut std::io::stderr().lock(), format!("{}", err)).ok();
+                AlertMessage::alert(
+                    &mut BufWriter::new(std::io::stderr().lock()),
+                    &format!("{}", err),
+                )
+                .ok();
             }
         }
     }
@@ -207,7 +241,16 @@ impl App {
                         evtx_filepath,
                         record_result.unwrap_err()
                     );
-                    AlertMessage::alert(&mut std::io::stderr().lock(), errmsg).ok();
+                    if configs::CONFIG.read().unwrap().args.is_present("verbose") {
+                        AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &errmsg)
+                            .ok();
+                    }
+                    if !*QUIET_ERRORS_FLAG {
+                        ERROR_LOG_STACK
+                            .lock()
+                            .unwrap()
+                            .push(format!("[ERROR] {}", errmsg));
+                    }
                     continue;
                 }
 

src/yaml.rs

@@ -3,11 +3,14 @@ extern crate yaml_rust;
 
 use crate::detections::configs;
 use crate::detections::print::AlertMessage;
+use crate::detections::print::ERROR_LOG_STACK;
+use crate::detections::print::QUIET_ERRORS_FLAG;
 use crate::filter::RuleExclude;
 use std::collections::HashMap;
 use std::ffi::OsStr;
 use std::fs;
 use std::io;
+use std::io::BufWriter;
 use std::io::{BufReader, Read};
 use std::path::{Path, PathBuf};
 use yaml_rust::Yaml;
@@ -71,14 +74,20 @@ impl ParseYaml {
             // 個別のファイルの読み込みは即終了としない。
             let read_content = self.read_file(path);
             if read_content.is_err() {
-                AlertMessage::warn(
+                let errmsg = format!(
-                    &mut std::io::stdout().lock(),
-                    format!(
                     "fail to read file: {}\n{} ",
                     entry.path().display(),
                     read_content.unwrap_err()
-                    ),
+                );
-                )?;
+                if configs::CONFIG.read().unwrap().args.is_present("verbose") {
+                    AlertMessage::warn(&mut BufWriter::new(std::io::stderr().lock()), &errmsg)?;
+                }
+                if !*QUIET_ERRORS_FLAG {
+                    ERROR_LOG_STACK
+                        .lock()
+                        .unwrap()
+                        .push(format!("[WARN] {}", errmsg));
+                }
                 self.errorrule_count += 1;
                 return io::Result::Ok(ret);
             }
@@ -86,14 +95,20 @@ impl ParseYaml {
             // ここも個別のファイルの読み込みは即終了としない。
             let yaml_contents = YamlLoader::load_from_str(&read_content.unwrap());
             if yaml_contents.is_err() {
-                AlertMessage::warn(
+                let errmsg = format!(
-                    &mut std::io::stdout().lock(),
-                    format!(
                     "Failed to parse yml: {}\n{} ",
                     entry.path().display(),
                     yaml_contents.unwrap_err()
-                    ),
+                );
-                )?;
+                if configs::CONFIG.read().unwrap().args.is_present("verbose") {
+                    AlertMessage::warn(&mut BufWriter::new(std::io::stderr().lock()), &errmsg)?;
+                }
+                if !*QUIET_ERRORS_FLAG {
+                    ERROR_LOG_STACK
+                        .lock()
+                        .unwrap()
+                        .push(format!("[WARN] {}", errmsg));
+                }
                 self.errorrule_count += 1;
                 return io::Result::Ok(ret);
             }
@@ -176,6 +191,8 @@ impl ParseYaml {
 #[cfg(test)]
 mod tests {
 
+    use crate::detections::print::AlertMessage;
+    use crate::detections::print::ERROR_LOG_PATH;
     use crate::filter;
     use crate::yaml;
     use crate::yaml::RuleExclude;
@@ -185,6 +202,8 @@ mod tests {
 
     #[test]
     fn test_read_dir_yaml() {
+        AlertMessage::create_error_log(ERROR_LOG_PATH.to_string());
+
         let mut yaml = yaml::ParseYaml::new();
         let exclude_ids = RuleExclude {
             no_use_rule: HashSet::new(),
@@ -275,6 +294,8 @@ mod tests {
     }
     #[test]
     fn test_all_exclude_rules_file() {
+        AlertMessage::create_error_log(ERROR_LOG_PATH.to_string());
+
         let mut yaml = yaml::ParseYaml::new();
         let path = Path::new("test_files/rules/yaml");
         yaml.read_dir(path.to_path_buf(), &"", &filter::exclude_ids())
@@ -283,6 +304,8 @@ mod tests {
     }
     #[test]
     fn test_none_exclude_rules_file() {
+        AlertMessage::create_error_log(ERROR_LOG_PATH.to_string());
+
         let mut yaml = yaml::ParseYaml::new();
         let path = Path::new("test_files/rules/yaml");
         let exclude_ids = RuleExclude {