CSEC_PUBLIC/hayabusa
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #316 from Yamato-Security/feature/output_error_log_file_and_options#301

Browse Source 
fixed #301 #303 #309
This commit is contained in:
Yamato Security
2021-12-22 16:08:13 +09:00
committed by GitHub
parent 72b6bceb36 3412434d99
commit 67f0ee007b
8 changed files with 239 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
contributors.txt
View File

@@ -1,10 +1,9 @@
Hayabusa was possible thanks to the following people (in alphabetical order):
Akira Nishikawa (@nishikawaakira): Previous lead developer, core hayabusa rule support, etc...
DustInDark(@hitenkoku): Core developer, project management, sigma count implementation, rule creation, countless feature additions and fixes, etc…
Garigariganzy (@garigariganzy31): Developer, event ID statistics implementation, etc...
ItiB (@itiB_S144) : Core developer, sigmac hayabusa backend, rule creation, etc...
James Takai / hachiyone(@hach1yon): Current lead developer, tokio multi-threading, sigma aggregation logic, sigmac backend, rule creation, etc…
James Takai / hachiyone(@hach1yon): Current lead developer, tokio multi-threading, sigma aggregation logic, sigmac backend, rule creation, sigma count implementation etc…
Kazuminn (@k47_um1n): Developer
Yusuke Matsui (@apt773): AD hacking working group leader, rule testing, documentation, research, support, etc...
Zach Mathis (@yamatosecurity, Yamato Security Founder): Project leader, tool and concept design, rule creation and tuning, etc…
@@ -17,7 +16,6 @@ Nishikawa Akira (@nishikawaakira): Lead Developer
kazuminn (@k47_um1n): Core Developer
itiB (@itiB_S144): Core Developer
James Takai / hachiyone (@hach1yon): Core Developer
DustInDark (@hitenkoku): Core Developer
garigariganzy (@garigariganzy31): Developer
7itoh (@yNitocrypto22): Developer
dai (@__da13__): Developer

8
src/afterfact.rs
View File

@@ -36,8 +36,8 @@ pub struct DisplayFormat<'a> {
pub fn after_fact() {
let fn_emit_csv_err = |err: Box<dyn Error>| {
AlertMessage::alert(
&mut std::io::stderr().lock(),
format!("Failed to write CSV. {}", err),
&mut BufWriter::new(std::io::stderr().lock()),
&format!("Failed to write CSV. {}", err),
)
.ok();
process::exit(1);
@@ -51,8 +51,8 @@ pub fn after_fact() {
Ok(file) => Box::new(BufWriter::new(file)),
Err(err) => {
AlertMessage::alert(
&mut std::io::stderr().lock(),
format!("Failed to open file. {}", err),
&mut BufWriter::new(std::io::stderr().lock()),
&format!("Failed to open file. {}", err),
)
.ok();
process::exit(1);

10
src/detections/configs.rs
View File

@@ -5,6 +5,7 @@ use clap::{App, AppSettings, ArgMatches};
use hashbrown::HashMap;
use hashbrown::HashSet;
use lazy_static::lazy_static;
use std::io::BufWriter;
use std::sync::RwLock;
lazy_static! {
pub static ref CONFIG: RwLock<ConfigReader> = RwLock::new(ConfigReader::new());
@@ -68,6 +69,7 @@ fn build_app<'a>() -> ArgMatches<'a> {
-t --thread-number=[NUMBER] 'Thread number (default: optimal number for performance)'
-s --statistics 'Prints statistics of event IDs'
-q --quiet 'Quiet mode. Do not display the launch banner'
-Q --quiet-errors 'Quiet errors mode. Do not save error logs.'
--contributors 'Prints the list of contributors'";
App::new(&program)
.about("Hayabusa: Aiming to be the world's greatest Windows event log analysis tool!")
@@ -139,8 +141,8 @@ impl TargetEventTime {
Ok(dt) => Some(dt.with_timezone(&Utc)),
Err(err) => {
AlertMessage::alert(
&mut std::io::stderr().lock(),
format!("starttimeline field: {}", err),
&mut BufWriter::new(std::io::stderr().lock()),
&format!("start-timeline field: {}", err),
)
.ok();
None
@@ -156,8 +158,8 @@ impl TargetEventTime {
Ok(dt) => Some(dt.with_timezone(&Utc)),
Err(err) => {
AlertMessage::alert(
&mut std::io::stderr().lock(),
format!("endtimeline field: {}", err),
&mut BufWriter::new(std::io::stderr().lock()),
&format!("end-timeline field: {}", err),
)
.ok();
None

26
src/detections/detection.rs
View File

@@ -1,7 +1,10 @@
extern crate csv;
use crate::detections::configs;
use crate::detections::print::AlertMessage;
use crate::detections::print::ERROR_LOG_STACK;
use crate::detections::print::MESSAGES;
use crate::detections::print::QUIET_ERRORS_FLAG;
use crate::detections::rule;
use crate::detections::rule::AggResult;
use crate::detections::rule::RuleNode;
@@ -11,9 +14,9 @@ use crate::yaml::ParseYaml;
use hashbrown;
use serde_json::Value;
use std::collections::HashMap;
use tokio::{runtime::Runtime, spawn, task::JoinHandle};
use std::io::BufWriter;
use std::sync::Arc;
use tokio::{runtime::Runtime, spawn, task::JoinHandle};
const DIRPATH_RULES: &str = "rules";
@@ -57,11 +60,16 @@ impl Detection {
let result_readdir =
rulefile_loader.read_dir(rulespath.unwrap_or(DIRPATH_RULES), &level, exclude_ids);
if result_readdir.is_err() {
AlertMessage::alert(
&mut std::io::stderr().lock(),
format!("{}", result_readdir.unwrap_err()),
)
.ok();
let errmsg = format!("{}", result_readdir.unwrap_err());
if configs::CONFIG.read().unwrap().args.is_present("verbose") {
AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &errmsg).ok();
}
if !*QUIET_ERRORS_FLAG {
ERROR_LOG_STACK
.lock()
.unwrap()
.push(format!("[ERROR] {}", errmsg));
}
return vec![];
}
let mut parseerror_count = rulefile_loader.errorrule_count;
@@ -75,10 +83,10 @@ impl Detection {
err_msgs_result.err().iter().for_each(|err_msgs| {
let errmsg_body =
format!("Failed to parse rule file. (FilePath : {})", rule.rulepath);
AlertMessage::warn(&mut std::io::stdout().lock(), errmsg_body).ok();
AlertMessage::warn(&mut std::io::stdout().lock(), &errmsg_body).ok();
err_msgs.iter().for_each(|err_msg| {
AlertMessage::warn(&mut std::io::stdout().lock(), err_msg.to_string()).ok();
AlertMessage::warn(&mut std::io::stdout().lock(), err_msg).ok();
});
parseerror_count += 1;
println!(""); // 一行開けるためのprintln

76
src/detections/print.rs
View File

@@ -2,13 +2,18 @@ extern crate lazy_static;
use crate::detections::configs;
use crate::detections::utils;
use crate::detections::utils::get_serde_number_to_string;
use chrono::{DateTime, TimeZone, Utc};
use chrono::{DateTime, Local, TimeZone, Utc};
use lazy_static::lazy_static;
use regex::Regex;
use serde_json::Value;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::env;
use std::fs::create_dir;
use std::fs::File;
use std::io::BufWriter;
use std::io::{self, Write};
use std::path::Path;
use std::sync::Mutex;
#[derive(Debug)]
@@ -32,6 +37,16 @@ pub struct AlertMessage {}
lazy_static! {
pub static ref MESSAGES: Mutex<Message> = Mutex::new(Message::new());
pub static ref ALIASREGEX: Regex = Regex::new(r"%[a-zA-Z0-9-_]+%").unwrap();
pub static ref ERROR_LOG_PATH: String = format!(
"./logs/errorlog-{}.log",
Local::now().format("%Y%m%d_%H%M%S")
);
pub static ref QUIET_ERRORS_FLAG: bool = configs::CONFIG
.read()
.unwrap()
.args
.is_present("quiet-errors");
pub static ref ERROR_LOG_STACK: Mutex<Vec<String>> = Mutex::new(Vec::new());
}
impl Message {
@@ -180,10 +195,48 @@ impl Message {
}
impl AlertMessage {
pub fn alert<W: Write>(w: &mut W, contents: String) -> io::Result<()> {
///対象のディレクトリが存在することを確認後、最初の定型文を追加して、ファイルのbufwriterを返す関数
pub fn create_error_log(path_str: String) {
if *QUIET_ERRORS_FLAG {
return;
}
let path = Path::new(&path_str);
if !path.parent().unwrap().exists() {
create_dir(path.parent().unwrap()).ok();
}
let mut error_log_writer = BufWriter::new(File::create(path).unwrap());
error_log_writer
.write(
format!(
"user input: {:?}\n",
format_args!(
"{}",
env::args()
.map(|arg| arg)
.collect::<Vec<String>>()
.join(" ")
)
)
.as_bytes(),
)
.unwrap();
for error_log in ERROR_LOG_STACK.lock().unwrap().iter() {
writeln!(error_log_writer, "{}", error_log).ok();
}
println!(
"Errors were generated. Please check {} for details.",
ERROR_LOG_PATH.to_string()
);
println!("");
}
/// ERRORメッセージを表示する関数
pub fn alert<W: Write>(w: &mut W, contents: &String) -> io::Result<()> {
writeln!(w, "[ERROR] {}", contents)
}
pub fn warn<W: Write>(w: &mut W, contents: String) -> io::Result<()> {
/// WARNメッセージを表示する関数
pub fn warn<W: Write>(w: &mut W, contents: &String) -> io::Result<()> {
writeln!(w, "[WARN] {}", contents)
}
}
@@ -192,6 +245,7 @@ impl AlertMessage {
mod tests {
use crate::detections::print::{AlertMessage, Message};
use serde_json::Value;
use std::io::BufWriter;
#[test]
fn test_create_and_append_message() {
@@ -304,17 +358,21 @@ mod tests {
#[test]
fn test_error_message() {
let input = "TEST!";
let stdout = std::io::stdout();
let mut stdout = stdout.lock();
AlertMessage::alert(&mut stdout, input.to_string()).expect("[ERROR] TEST!");
AlertMessage::alert(
&mut BufWriter::new(std::io::stdout().lock()),
&input.to_string(),
)
.expect("[ERROR] TEST!");
}
#[test]
fn test_warn_message() {
let input = "TESTWarn!";
let stdout = std::io::stdout();
let mut stdout = stdout.lock();
AlertMessage::alert(&mut stdout, input.to_string()).expect("[WARN] TESTWarn!");
AlertMessage::warn(
&mut BufWriter::new(std::io::stdout().lock()),
&input.to_string(),
)
.expect("[WARN] TESTWarn!");
}
#[test]

87
src/detections/rule/count.rs
View File

@@ -1,10 +1,14 @@
use crate::detections::configs;
use crate::detections::print::AlertMessage;
use crate::detections::print::ERROR_LOG_STACK;
use crate::detections::print::QUIET_ERRORS_FLAG;
use crate::detections::rule::AggResult;
use crate::detections::rule::Message;
use crate::detections::rule::RuleNode;
use chrono::{DateTime, TimeZone, Utc};
use hashbrown::HashMap;
use serde_json::Value;
use std::io::BufWriter;
use std::num::ParseIntError;
use std::path::Path;
@@ -65,30 +69,35 @@ fn get_alias_value_in_record(
return Some(value.to_string().replace("\"", ""));
}
None => {
AlertMessage::alert(
&mut std::io::stderr().lock(),
match is_by_alias {
true => format!(
"count by clause alias value not found in count process. rule file:{} EventID:{}",
Path::new(&rule.rulepath)
.file_name()
.unwrap()
.to_str()
.unwrap(),
utils::get_event_value(&utils::get_event_id_key(), record).unwrap()
),
false => format!(
"count field clause alias value not found in count process. rule file:{} EventID:{}",
Path::new(&rule.rulepath)
.file_name()
.unwrap()
.to_str()
.unwrap(),
utils::get_event_value(&utils::get_event_id_key(), record).unwrap()
),
},
)
.ok();
let errmsg = match is_by_alias {
true => format!(
"count by clause alias value not found in count process. rule file:{} EventID:{}",
Path::new(&rule.rulepath)
.file_name()
.unwrap()
.to_str()
.unwrap(),
utils::get_event_value(&utils::get_event_id_key(), record).unwrap()
),
false => format!(
"count field clause alias value not found in count process. rule file:{} EventID:{}",
Path::new(&rule.rulepath)
.file_name()
.unwrap()
.to_str()
.unwrap(),
utils::get_event_value(&utils::get_event_id_key(), record).unwrap()
),
};
if configs::CONFIG.read().unwrap().args.is_present("verbose") {
AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &errmsg).ok();
}
if !*QUIET_ERRORS_FLAG {
ERROR_LOG_STACK
.lock()
.unwrap()
.push(format!("[ERROR] {}", errmsg));
}
return None;
}
};
@@ -181,11 +190,16 @@ impl TimeFrameInfo {
ttype = "d".to_owned();
tnum.retain(|c| c != 'd');
} else {
AlertMessage::alert(
&mut std::io::stderr().lock(),
format!("Timeframe is invalid. Input value:{}", value),
)
.ok();
let errmsg = format!("Timeframe is invalid. Input value:{}", value);
if configs::CONFIG.read().unwrap().args.is_present("verbose") {
AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &errmsg).ok();
}
if !*QUIET_ERRORS_FLAG {
ERROR_LOG_STACK
.lock()
.unwrap()
.push(format!("[ERROR] {}", errmsg));
}
}
return TimeFrameInfo {
timetype: ttype,
@@ -214,11 +228,16 @@ pub fn get_sec_timeframe(rule: &RuleNode) -> Option<i64> {
}
}
Err(err) => {
AlertMessage::alert(
&mut std::io::stderr().lock(),
format!("Timeframe number is invalid. timeframe.{}", err),
)
.ok();
let errmsg = format!("Timeframe number is invalid. timeframe. {}", err);
if configs::CONFIG.read().unwrap().args.is_present("verbose") {
AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &errmsg).ok();
}
if !*QUIET_ERRORS_FLAG {
ERROR_LOG_STACK
.lock()
.unwrap()
.push(format!("[ERROR] {}", errmsg.to_string()));
}
return Option::None;
}
}

61
src/main.rs
View File

@@ -6,6 +6,9 @@ use chrono::{DateTime, Local};
use evtx::{EvtxParser, ParserSettings};
use hayabusa::detections::detection::{self, EvtxRecordInfo};
use hayabusa::detections::print::AlertMessage;
use hayabusa::detections::print::ERROR_LOG_PATH;
use hayabusa::detections::print::ERROR_LOG_STACK;
use hayabusa::detections::print::QUIET_ERRORS_FLAG;
use hayabusa::detections::rule::{get_detection_keys, RuleNode};
use hayabusa::filter;
use hayabusa::omikuji::Omikuji;
@@ -16,6 +19,8 @@ use pbr::ProgressBar;
use serde_json::Value;
use std::collections::{HashMap, HashSet};
use std::fmt::Display;
use std::io::BufWriter;
use std::path::Path;
use std::sync::Arc;
use std::{
fs::{self, File},
@@ -66,11 +71,24 @@ impl App {
);
return;
}
if let Some(csv_path) = configs::CONFIG.read().unwrap().args.value_of("output") {
if Path::new(csv_path).exists() {
AlertMessage::alert(
&mut BufWriter::new(std::io::stderr().lock()),
&format!(
" The file {} already exists. Please specify a different filename.",
csv_path
),
)
.ok();
return;
}
}
if let Some(filepath) = configs::CONFIG.read().unwrap().args.value_of("filepath") {
if !filepath.ends_with(".evtx") {
AlertMessage::alert(
&mut std::io::stderr().lock(),
"--filepath only accepts .evtx files.".to_owned(),
&mut BufWriter::new(std::io::stderr().lock()),
&"--filepath only accepts .evtx files.".to_string(),
)
.ok();
return;
@@ -80,8 +98,8 @@ impl App {
let evtx_files = self.collect_evtxfiles(&directory);
if evtx_files.len() == 0 {
AlertMessage::alert(
&mut std::io::stderr().lock(),
"No .evtx files were found.".to_owned(),
&mut BufWriter::new(std::io::stderr().lock()),
&"No .evtx files were found.".to_string(),
)
.ok();
return;
@@ -100,14 +118,26 @@ impl App {
let analysis_duration = analysis_end_time.signed_duration_since(analysis_start_time);
println!("Elapsed Time: {}", &analysis_duration.hhmmssxxx());
println!("");
// Qオプションを付けた場合もしくはパースのエラーがない場合はerrorのstackが9となるのでエラーログファイル自体が生成されない。
if ERROR_LOG_STACK.lock().unwrap().len() > 0 {
AlertMessage::create_error_log(ERROR_LOG_PATH.to_string());
}
}
fn collect_evtxfiles(&self, dirpath: &str) -> Vec<PathBuf> {
let entries = fs::read_dir(dirpath);
if entries.is_err() {
let stderr = std::io::stderr();
let mut stderr = stderr.lock();
AlertMessage::alert(&mut stderr, format!("{}", entries.unwrap_err())).ok();
let errmsg = format!("{}", entries.unwrap_err());
if configs::CONFIG.read().unwrap().args.is_present("verbose") {
AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &errmsg).ok();
}
if !*QUIET_ERRORS_FLAG {
ERROR_LOG_STACK
.lock()
.unwrap()
.push(format!("[ERROR] {}", errmsg));
}
return vec![];
}
@@ -139,7 +169,11 @@ impl App {
match fs::read_to_string("./contributors.txt") {
Ok(contents) => println!("{}", contents),
Err(err) => {
AlertMessage::alert(&mut std::io::stderr().lock(), format!("{}", err)).ok();
AlertMessage::alert(
&mut BufWriter::new(std::io::stderr().lock()),
&format!("{}", err),
)
.ok();
}
}
}
@@ -207,7 +241,16 @@ impl App {
evtx_filepath,
record_result.unwrap_err()
);
AlertMessage::alert(&mut std::io::stderr().lock(), errmsg).ok();
if configs::CONFIG.read().unwrap().args.is_present("verbose") {
AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &errmsg)
.ok();
}
if !*QUIET_ERRORS_FLAG {
ERROR_LOG_STACK
.lock()
.unwrap()
.push(format!("[ERROR] {}", errmsg));
}
continue;
}

55
src/yaml.rs
View File

@@ -3,11 +3,14 @@ extern crate yaml_rust;
use crate::detections::configs;
use crate::detections::print::AlertMessage;
use crate::detections::print::ERROR_LOG_STACK;
use crate::detections::print::QUIET_ERRORS_FLAG;
use crate::filter::RuleExclude;
use std::collections::HashMap;
use std::ffi::OsStr;
use std::fs;
use std::io;
use std::io::BufWriter;
use std::io::{BufReader, Read};
use std::path::{Path, PathBuf};
use yaml_rust::Yaml;
@@ -71,14 +74,20 @@ impl ParseYaml {
// 個別のファイルの読み込みは即終了としない。
let read_content = self.read_file(path);
if read_content.is_err() {
AlertMessage::warn(
&mut std::io::stdout().lock(),
format!(
"fail to read file: {}\n{} ",
entry.path().display(),
read_content.unwrap_err()
),
)?;
let errmsg = format!(
"fail to read file: {}\n{} ",
entry.path().display(),
read_content.unwrap_err()
);
if configs::CONFIG.read().unwrap().args.is_present("verbose") {
AlertMessage::warn(&mut BufWriter::new(std::io::stderr().lock()), &errmsg)?;
}
if !*QUIET_ERRORS_FLAG {
ERROR_LOG_STACK
.lock()
.unwrap()
.push(format!("[WARN] {}", errmsg));
}
self.errorrule_count += 1;
return io::Result::Ok(ret);
}
@@ -86,14 +95,20 @@ impl ParseYaml {
// ここも個別のファイルの読み込みは即終了としない。
let yaml_contents = YamlLoader::load_from_str(&read_content.unwrap());
if yaml_contents.is_err() {
AlertMessage::warn(
&mut std::io::stdout().lock(),
format!(
"Failed to parse yml: {}\n{} ",
entry.path().display(),
yaml_contents.unwrap_err()
),
)?;
let errmsg = format!(
"Failed to parse yml: {}\n{} ",
entry.path().display(),
yaml_contents.unwrap_err()
);
if configs::CONFIG.read().unwrap().args.is_present("verbose") {
AlertMessage::warn(&mut BufWriter::new(std::io::stderr().lock()), &errmsg)?;
}
if !*QUIET_ERRORS_FLAG {
ERROR_LOG_STACK
.lock()
.unwrap()
.push(format!("[WARN] {}", errmsg));
}
self.errorrule_count += 1;
return io::Result::Ok(ret);
}
@@ -176,6 +191,8 @@ impl ParseYaml {
#[cfg(test)]
mod tests {
use crate::detections::print::AlertMessage;
use crate::detections::print::ERROR_LOG_PATH;
use crate::filter;
use crate::yaml;
use crate::yaml::RuleExclude;
@@ -185,6 +202,8 @@ mod tests {
#[test]
fn test_read_dir_yaml() {
AlertMessage::create_error_log(ERROR_LOG_PATH.to_string());
let mut yaml = yaml::ParseYaml::new();
let exclude_ids = RuleExclude {
no_use_rule: HashSet::new(),
@@ -275,6 +294,8 @@ mod tests {
}
#[test]
fn test_all_exclude_rules_file() {
AlertMessage::create_error_log(ERROR_LOG_PATH.to_string());
let mut yaml = yaml::ParseYaml::new();
let path = Path::new("test_files/rules/yaml");
yaml.read_dir(path.to_path_buf(), &"", &filter::exclude_ids())
@@ -283,6 +304,8 @@ mod tests {
}
#[test]
fn test_none_exclude_rules_file() {
AlertMessage::create_error_log(ERROR_LOG_PATH.to_string());
let mut yaml = yaml::ParseYaml::new();
let path = Path::new("test_files/rules/yaml");
let exclude_ids = RuleExclude {