From 276889338d45afae2605c28044988f6ac3a7d1d3 Mon Sep 17 00:00:00 2001 From: itiB Date: Mon, 28 Feb 2022 21:58:35 +0900 Subject: [PATCH 01/42] Add: --level-tuning option's outline --- config/rule_level.txt | 0 src/detections/configs.rs | 6 +++++- src/main.rs | 23 +++++++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 config/rule_level.txt diff --git a/config/rule_level.txt b/config/rule_level.txt new file mode 100644 index 00000000..e69de29b diff --git a/src/detections/configs.rs b/src/detections/configs.rs index 9ad2185a..ef8e6d50 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -1,7 +1,7 @@ use crate::detections::print::AlertMessage; use crate::detections::utils; use chrono::{DateTime, Utc}; -use clap::{App, AppSettings, ArgMatches}; +use clap::{App, Arg, AppSettings, ArgMatches}; use hashbrown::HashMap; use hashbrown::HashSet; use lazy_static::lazy_static; @@ -92,6 +92,10 @@ fn build_app<'a>() -> ArgMatches<'a> { .version("1.1.0") .author("Yamato Security (https://github.com/Yamato-Security/hayabusa)") .setting(AppSettings::VersionlessSubcommands) + .arg( // TODO: When update claps to 3.x, these can write in usage texts... + Arg::from_usage("--level-tuning=[RULE_LEVEL_FILE] 'Fix rule file's level'") + .default_value("./config/rule_level.txt") + ) .usage(usages) .args_from_usage(usages) .get_matches() diff --git a/src/main.rs b/src/main.rs index 3461b1d9..a09bd0c0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -104,6 +104,29 @@ impl App { println!(); return; } + + if configs::CONFIG + .read() + .unwrap() + .args.is_present("level-tuning") + { + if let Some(level_tuning_path) = configs::CONFIG.read().unwrap().args.value_of("level-tuning") { + if Path::new(level_tuning_path).exists() { + println!("level-tuning file exist: {}", level_tuning_path); + println!("WIP: level-tuning...."); + } else { + AlertMessage::alert( + &mut BufWriter::new(std::io::stderr().lock()), + &format!( + "Need rule_levels.txt file to use --level-tuning option" + ), + ) + .ok(); + return; + } + } + } + if !Path::new("./config").exists() { AlertMessage::alert( &mut BufWriter::new(std::io::stderr().lock()), From a15bef4b303a3ee27f5b658e414c7cbfa75f04bd Mon Sep 17 00:00:00 2001 From: itiB Date: Thu, 24 Mar 2022 02:01:09 +0900 Subject: [PATCH 02/42] Add: read Rule files --- src/main.rs | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index a09bd0c0..9c3ac70d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,7 +18,8 @@ use hayabusa::filter; use hayabusa::omikuji::Omikuji; use hayabusa::yaml::ParseYaml; use hayabusa::{afterfact::after_fact, detections::utils}; -use hayabusa::{detections::configs, timeline::timelines::Timeline}; +use hayabusa::{detections::configs, timeline::timeline::Timeline}; +use hayabusa::yaml::ParseYaml; use hhmmss::Hhmmss; use pbr::ProgressBar; use serde_json::Value; @@ -112,6 +113,44 @@ impl App { { if let Some(level_tuning_path) = configs::CONFIG.read().unwrap().args.value_of("level-tuning") { if Path::new(level_tuning_path).exists() { + let read_result = utils::read_csv(level_tuning_path); + if read_result.is_err() { + // color情報がない場合は通常の白色の出力が出てくるのみで動作への影響を与えない為warnとして処理する + AlertMessage::warn( + &mut BufWriter::new(std::io::stderr().lock()), + &read_result.as_ref().unwrap_err(), + ) + .ok(); + return; + } + let mut tuning_map: HashMap = HashMap::new(); + read_result.unwrap().into_iter().for_each(|line| { + if line.len() != 2 { + return; + } + let id = line.get(0).unwrap(); + // TODO: id validation + let level = line.get(1).unwrap(); + // TODO: level validation + tuning_map.insert(id.to_string(), level.to_string()); + + let mut rulefile_loader = ParseYaml::new(); + let result_readdir = + rulefile_loader.read_dir( + &level, + configs::CONFIG.read().unwrap().args.value_of("rules").unwrap_or(&"rules"), + &filter::exclude_ids(), + ); + if result_readdir.is_err() { + let errmsg = format!("{}", result_readdir.unwrap_err()); + AlertMessage::warn( + &mut BufWriter::new(std::io::stderr().lock()), + &errmsg, + ) + .ok(); + return; + } + }); println!("level-tuning file exist: {}", level_tuning_path); println!("WIP: level-tuning...."); } else { From d38834e20eabca91b2a76e8043aea2e805743062 Mon Sep 17 00:00:00 2001 From: itiB Date: Thu, 24 Mar 2022 08:44:19 +0900 Subject: [PATCH 03/42] Add: input rule_level.txt files & read rules --- config/rule_level.txt | 2 ++ src/main.rs | 44 +++++++++++++++++++++++-------------------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/config/rule_level.txt b/config/rule_level.txt index e69de29b..2f8de0ea 100644 --- a/config/rule_level.txt +++ b/config/rule_level.txt @@ -0,0 +1,2 @@ +id,next_level +671bb7e3-a020-4824-a00e-2ee5b55f385e,high # CACTUSTORCH Remote Thread Creation \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 9c3ac70d..cbe233ba 100644 --- a/src/main.rs +++ b/src/main.rs @@ -115,7 +115,6 @@ impl App { if Path::new(level_tuning_path).exists() { let read_result = utils::read_csv(level_tuning_path); if read_result.is_err() { - // color情報がない場合は通常の白色の出力が出てくるのみで動作への影響を与えない為warnとして処理する AlertMessage::warn( &mut BufWriter::new(std::io::stderr().lock()), &read_result.as_ref().unwrap_err(), @@ -132,27 +131,32 @@ impl App { // TODO: id validation let level = line.get(1).unwrap(); // TODO: level validation + // Cut Comments tuning_map.insert(id.to_string(), level.to_string()); - - let mut rulefile_loader = ParseYaml::new(); - let result_readdir = - rulefile_loader.read_dir( - &level, - configs::CONFIG.read().unwrap().args.value_of("rules").unwrap_or(&"rules"), - &filter::exclude_ids(), - ); - if result_readdir.is_err() { - let errmsg = format!("{}", result_readdir.unwrap_err()); - AlertMessage::warn( - &mut BufWriter::new(std::io::stderr().lock()), - &errmsg, - ) - .ok(); - return; - } }); - println!("level-tuning file exist: {}", level_tuning_path); - println!("WIP: level-tuning...."); + let mut rulefile_loader = ParseYaml::new(); + let result_readdir = + rulefile_loader.read_dir( + configs::CONFIG.read().unwrap().args.value_of("rules").unwrap_or(&"rules"), + &"informational", + &filter::exclude_ids(), + ); + if result_readdir.is_err() { + let errmsg = format!("{}", result_readdir.unwrap_err()); + AlertMessage::warn( + &mut BufWriter::new(std::io::stderr().lock()), + &errmsg, + ) + .ok(); + return; + } + for (path, rule) in rulefile_loader.files { + if let Some(new_level) = tuning_map.get(rule["id"].as_str().unwrap()) { + println!("{}", rule["id"].as_str().unwrap()); + println!("path: {}", path); + println!("level: {} -> {}", rule["level"].as_str().unwrap(), new_level); + } + } } else { AlertMessage::alert( &mut BufWriter::new(std::io::stderr().lock()), From 814f5a61cbbd45da7863d318d6530cb4f5caa9ee Mon Sep 17 00:00:00 2001 From: itiB Date: Sun, 3 Apr 2022 22:01:40 +0900 Subject: [PATCH 04/42] cargo fmt --- src/detections/configs.rs | 7 +++--- src/main.rs | 46 +++++++++++++++++++++++---------------- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index ef8e6d50..0fcd1f08 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -1,7 +1,7 @@ use crate::detections::print::AlertMessage; use crate::detections::utils; use chrono::{DateTime, Utc}; -use clap::{App, Arg, AppSettings, ArgMatches}; +use clap::{App, AppSettings, Arg, ArgMatches}; use hashbrown::HashMap; use hashbrown::HashSet; use lazy_static::lazy_static; @@ -92,9 +92,10 @@ fn build_app<'a>() -> ArgMatches<'a> { .version("1.1.0") .author("Yamato Security (https://github.com/Yamato-Security/hayabusa)") .setting(AppSettings::VersionlessSubcommands) - .arg( // TODO: When update claps to 3.x, these can write in usage texts... + .arg( + // TODO: When update claps to 3.x, these can write in usage texts... Arg::from_usage("--level-tuning=[RULE_LEVEL_FILE] 'Fix rule file's level'") - .default_value("./config/rule_level.txt") + .default_value("./config/rule_level.txt"), ) .usage(usages) .args_from_usage(usages) diff --git a/src/main.rs b/src/main.rs index cbe233ba..964c1aa1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,8 +18,7 @@ use hayabusa::filter; use hayabusa::omikuji::Omikuji; use hayabusa::yaml::ParseYaml; use hayabusa::{afterfact::after_fact, detections::utils}; -use hayabusa::{detections::configs, timeline::timeline::Timeline}; -use hayabusa::yaml::ParseYaml; +use hayabusa::{detections::configs, timeline::timelines::Timeline}; use hhmmss::Hhmmss; use pbr::ProgressBar; use serde_json::Value; @@ -109,9 +108,15 @@ impl App { if configs::CONFIG .read() .unwrap() - .args.is_present("level-tuning") + .args + .is_present("level-tuning") { - if let Some(level_tuning_path) = configs::CONFIG.read().unwrap().args.value_of("level-tuning") { + if let Some(level_tuning_path) = configs::CONFIG + .read() + .unwrap() + .args + .value_of("level-tuning") + { if Path::new(level_tuning_path).exists() { let read_result = utils::read_csv(level_tuning_path); if read_result.is_err() { @@ -135,34 +140,37 @@ impl App { tuning_map.insert(id.to_string(), level.to_string()); }); let mut rulefile_loader = ParseYaml::new(); - let result_readdir = - rulefile_loader.read_dir( - configs::CONFIG.read().unwrap().args.value_of("rules").unwrap_or(&"rules"), - &"informational", - &filter::exclude_ids(), - ); + let result_readdir = rulefile_loader.read_dir( + configs::CONFIG + .read() + .unwrap() + .args + .value_of("rules") + .unwrap_or(&"rules"), + &"informational", + &filter::exclude_ids(), + ); if result_readdir.is_err() { let errmsg = format!("{}", result_readdir.unwrap_err()); - AlertMessage::warn( - &mut BufWriter::new(std::io::stderr().lock()), - &errmsg, - ) - .ok(); + AlertMessage::warn(&mut BufWriter::new(std::io::stderr().lock()), &errmsg) + .ok(); return; } for (path, rule) in rulefile_loader.files { if let Some(new_level) = tuning_map.get(rule["id"].as_str().unwrap()) { println!("{}", rule["id"].as_str().unwrap()); println!("path: {}", path); - println!("level: {} -> {}", rule["level"].as_str().unwrap(), new_level); + println!( + "level: {} -> {}", + rule["level"].as_str().unwrap(), + new_level + ); } } } else { AlertMessage::alert( &mut BufWriter::new(std::io::stderr().lock()), - &format!( - "Need rule_levels.txt file to use --level-tuning option" - ), + &format!("Need rule_levels.txt file to use --level-tuning option"), ) .ok(); return; From 9149500b40bcf289b21f8d2c8ddb873cf27dfe4f Mon Sep 17 00:00:00 2001 From: itiB Date: Sun, 3 Apr 2022 23:41:32 +0900 Subject: [PATCH 05/42] Add: level-tuning function --- src/main.rs | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 964c1aa1..635eff34 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,6 +13,7 @@ use hayabusa::detections::detection::{self, EvtxRecordInfo}; use hayabusa::detections::print::{ AlertMessage, ERROR_LOG_PATH, ERROR_LOG_STACK, QUIET_ERRORS_FLAG, STATISTICS_FLAG, }; +use std::io::{Read, Write}; use hayabusa::detections::rule::{get_detection_keys, RuleNode}; use hayabusa::filter; use hayabusa::omikuji::Omikuji; @@ -156,10 +157,41 @@ impl App { .ok(); return; } + for (path, rule) in rulefile_loader.files { if let Some(new_level) = tuning_map.get(rule["id"].as_str().unwrap()) { - println!("{}", rule["id"].as_str().unwrap()); println!("path: {}", path); + let mut file = match fs::File::options() + .create(true) + .write(true) + .read(true) + .append(false) + .open(&path) + { + Err(e) => panic!("Couldn't open {}: {}", path, e), + Ok(file) => file, + }; + + let mut content = String::new(); + file.read_to_string(&mut content).unwrap(); + let past_level = "level: ".to_string() + rule["level"].as_str().unwrap(); + + if new_level.starts_with("informational") { + content = content.replace(&past_level, "level: informational"); + } + if new_level.starts_with("low") { + content = content.replace(&past_level, "level: informational"); + } + if new_level.starts_with("medium") { + content = content.replace(&past_level, "level: medium"); + } + if new_level.starts_with("high") { + content = content.replace(&past_level, "level: high"); + } + if new_level.starts_with("critical") { + content = content.replace(&past_level, "level: critical"); + } + file.write_all(content.as_bytes()).unwrap(); // TODO: use result println!( "level: {} -> {}", rule["level"].as_str().unwrap(), From 6805bd6a0a13ba78bdf3f850c2179ffa52c8a4a5 Mon Sep 17 00:00:00 2001 From: itiB Date: Mon, 4 Apr 2022 00:31:21 +0900 Subject: [PATCH 06/42] Reface: split to options file --- src/lib.rs | 1 + src/main.rs | 89 ++--------------------------------- src/options/level_tuning.rs | 93 +++++++++++++++++++++++++++++++++++++ src/options/mod.rs | 1 + 4 files changed, 100 insertions(+), 84 deletions(-) create mode 100644 src/options/level_tuning.rs create mode 100644 src/options/mod.rs diff --git a/src/lib.rs b/src/lib.rs index 9bd8f144..5faf0723 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,5 +3,6 @@ pub mod detections; pub mod filter; pub mod notify; pub mod omikuji; +pub mod options; pub mod timeline; pub mod yaml; diff --git a/src/main.rs b/src/main.rs index 635eff34..50906427 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,10 +13,10 @@ use hayabusa::detections::detection::{self, EvtxRecordInfo}; use hayabusa::detections::print::{ AlertMessage, ERROR_LOG_PATH, ERROR_LOG_STACK, QUIET_ERRORS_FLAG, STATISTICS_FLAG, }; -use std::io::{Read, Write}; use hayabusa::detections::rule::{get_detection_keys, RuleNode}; use hayabusa::filter; use hayabusa::omikuji::Omikuji; +use hayabusa::options::level_tuning::LevelTuning; use hayabusa::yaml::ParseYaml; use hayabusa::{afterfact::after_fact, detections::utils}; use hayabusa::{detections::configs, timeline::timelines::Timeline}; @@ -112,97 +112,18 @@ impl App { .args .is_present("level-tuning") { - if let Some(level_tuning_path) = configs::CONFIG + if let Some(level_tuning_config_path) = configs::CONFIG .read() .unwrap() .args .value_of("level-tuning") { - if Path::new(level_tuning_path).exists() { - let read_result = utils::read_csv(level_tuning_path); - if read_result.is_err() { - AlertMessage::warn( - &mut BufWriter::new(std::io::stderr().lock()), - &read_result.as_ref().unwrap_err(), - ) - .ok(); - return; - } - let mut tuning_map: HashMap = HashMap::new(); - read_result.unwrap().into_iter().for_each(|line| { - if line.len() != 2 { - return; - } - let id = line.get(0).unwrap(); - // TODO: id validation - let level = line.get(1).unwrap(); - // TODO: level validation - // Cut Comments - tuning_map.insert(id.to_string(), level.to_string()); - }); - let mut rulefile_loader = ParseYaml::new(); - let result_readdir = rulefile_loader.read_dir( - configs::CONFIG - .read() - .unwrap() - .args - .value_of("rules") - .unwrap_or(&"rules"), - &"informational", - &filter::exclude_ids(), - ); - if result_readdir.is_err() { - let errmsg = format!("{}", result_readdir.unwrap_err()); - AlertMessage::warn(&mut BufWriter::new(std::io::stderr().lock()), &errmsg) - .ok(); - return; - } - - for (path, rule) in rulefile_loader.files { - if let Some(new_level) = tuning_map.get(rule["id"].as_str().unwrap()) { - println!("path: {}", path); - let mut file = match fs::File::options() - .create(true) - .write(true) - .read(true) - .append(false) - .open(&path) - { - Err(e) => panic!("Couldn't open {}: {}", path, e), - Ok(file) => file, - }; - - let mut content = String::new(); - file.read_to_string(&mut content).unwrap(); - let past_level = "level: ".to_string() + rule["level"].as_str().unwrap(); - - if new_level.starts_with("informational") { - content = content.replace(&past_level, "level: informational"); - } - if new_level.starts_with("low") { - content = content.replace(&past_level, "level: informational"); - } - if new_level.starts_with("medium") { - content = content.replace(&past_level, "level: medium"); - } - if new_level.starts_with("high") { - content = content.replace(&past_level, "level: high"); - } - if new_level.starts_with("critical") { - content = content.replace(&past_level, "level: critical"); - } - file.write_all(content.as_bytes()).unwrap(); // TODO: use result - println!( - "level: {} -> {}", - rule["level"].as_str().unwrap(), - new_level - ); - } - } + if Path::new(level_tuning_config_path).exists() { + LevelTuning::run(level_tuning_config_path); } else { AlertMessage::alert( &mut BufWriter::new(std::io::stderr().lock()), - &format!("Need rule_levels.txt file to use --level-tuning option"), + "Need rule_levels.txt file to use --level-tuning option", ) .ok(); return; diff --git a/src/options/level_tuning.rs b/src/options/level_tuning.rs new file mode 100644 index 00000000..f7ff84e8 --- /dev/null +++ b/src/options/level_tuning.rs @@ -0,0 +1,93 @@ +use crate::detections::{configs, print::AlertMessage, utils}; +use crate::filter; +use crate::yaml::ParseYaml; +use std::collections::HashMap; +use std::fs::File; +use std::io::BufWriter; +use std::io::{Read, Write}; + +pub struct LevelTuning {} + +impl LevelTuning { + pub fn run(level_tuning_config_path: &str) { + let read_result = utils::read_csv(level_tuning_config_path); + if read_result.is_err() { + AlertMessage::warn( + &mut BufWriter::new(std::io::stderr().lock()), + &read_result.as_ref().unwrap_err(), + ) + .ok(); + return; + } + let mut tuning_map: HashMap = HashMap::new(); + read_result.unwrap().into_iter().for_each(|line| { + if line.len() != 2 { + return; + } + let id = line.get(0).unwrap(); + // TODO: id validation + let level = line.get(1).unwrap(); + // TODO: level validation + // Cut Comments + tuning_map.insert(id.to_string(), level.to_string()); + }); + let mut rulefile_loader = ParseYaml::new(); + let result_readdir = rulefile_loader.read_dir( + configs::CONFIG + .read() + .unwrap() + .args + .value_of("rules") + .unwrap_or(&"rules"), + &"informational", + &filter::exclude_ids(), + ); + if result_readdir.is_err() { + let errmsg = format!("{}", result_readdir.unwrap_err()); + AlertMessage::warn(&mut BufWriter::new(std::io::stderr().lock()), &errmsg).ok(); + return; + } + + for (path, rule) in rulefile_loader.files { + if let Some(new_level) = tuning_map.get(rule["id"].as_str().unwrap()) { + println!("path: {}", path); + let mut file = match File::options() + .create(true) + .write(true) + .read(true) + .append(false) + .open(&path) + { + Err(e) => panic!("Couldn't open {}: {}", path, e), + Ok(file) => file, + }; + + let mut content = String::new(); + file.read_to_string(&mut content).unwrap(); + let past_level = "level: ".to_string() + rule["level"].as_str().unwrap(); + + if new_level.starts_with("informational") { + content = content.replace(&past_level, "level: informational"); + } + if new_level.starts_with("low") { + content = content.replace(&past_level, "level: informational"); + } + if new_level.starts_with("medium") { + content = content.replace(&past_level, "level: medium"); + } + if new_level.starts_with("high") { + content = content.replace(&past_level, "level: high"); + } + if new_level.starts_with("critical") { + content = content.replace(&past_level, "level: critical"); + } + file.write_all(content.as_bytes()).unwrap(); // TODO: use result + println!( + "level: {} -> {}", + rule["level"].as_str().unwrap(), + new_level + ); + } + } + } +} diff --git a/src/options/mod.rs b/src/options/mod.rs new file mode 100644 index 00000000..1f3c32b6 --- /dev/null +++ b/src/options/mod.rs @@ -0,0 +1 @@ +pub mod level_tuning; From 5891a1aca16c6b429553e000596476236a7f88fa Mon Sep 17 00:00:00 2001 From: itiB Date: Mon, 4 Apr 2022 01:44:04 +0900 Subject: [PATCH 07/42] WIP: Text overwrite failed... --- src/options/level_tuning.rs | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/options/level_tuning.rs b/src/options/level_tuning.rs index f7ff84e8..988596e2 100644 --- a/src/options/level_tuning.rs +++ b/src/options/level_tuning.rs @@ -2,7 +2,7 @@ use crate::detections::{configs, print::AlertMessage, utils}; use crate::filter; use crate::yaml::ParseYaml; use std::collections::HashMap; -use std::fs::File; +use std::fs::{self, File}; use std::io::BufWriter; use std::io::{Read, Write}; @@ -14,7 +14,7 @@ impl LevelTuning { if read_result.is_err() { AlertMessage::warn( &mut BufWriter::new(std::io::stderr().lock()), - &read_result.as_ref().unwrap_err(), + read_result.as_ref().unwrap_err(), ) .ok(); return; @@ -38,8 +38,8 @@ impl LevelTuning { .unwrap() .args .value_of("rules") - .unwrap_or(&"rules"), - &"informational", + .unwrap_or("rules"), + "informational", &filter::exclude_ids(), ); if result_readdir.is_err() { @@ -52,18 +52,16 @@ impl LevelTuning { if let Some(new_level) = tuning_map.get(rule["id"].as_str().unwrap()) { println!("path: {}", path); let mut file = match File::options() - .create(true) - .write(true) .read(true) - .append(false) + .write(true) .open(&path) { Err(e) => panic!("Couldn't open {}: {}", path, e), Ok(file) => file, }; - let mut content = String::new(); - file.read_to_string(&mut content).unwrap(); + let mut content = fs::read_to_string(&path).unwrap(); // TODO: Error Handling + // file.read_to_string(&mut content).unwrap(); let past_level = "level: ".to_string() + rule["level"].as_str().unwrap(); if new_level.starts_with("informational") { @@ -81,6 +79,16 @@ impl LevelTuning { if new_level.starts_with("critical") { content = content.replace(&past_level, "level: critical"); } + println!("{:?}", &content); + + let mut file = match File::options() + .write(true) + .open(&path) + { + Err(e) => panic!("Couldn't open {}: {}", path, e), + Ok(file) => file, + }; + file.write_all(content.as_bytes()).unwrap(); // TODO: use result println!( "level: {} -> {}", From 6b08752120439319999fbd1c7a45eaf2b0886e78 Mon Sep 17 00:00:00 2001 From: itiB Date: Mon, 4 Apr 2022 23:44:54 +0900 Subject: [PATCH 08/42] Fix: Text overwrite was failed --- src/options/level_tuning.rs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/options/level_tuning.rs b/src/options/level_tuning.rs index 988596e2..ca0121bb 100644 --- a/src/options/level_tuning.rs +++ b/src/options/level_tuning.rs @@ -51,17 +51,7 @@ impl LevelTuning { for (path, rule) in rulefile_loader.files { if let Some(new_level) = tuning_map.get(rule["id"].as_str().unwrap()) { println!("path: {}", path); - let mut file = match File::options() - .read(true) - .write(true) - .open(&path) - { - Err(e) => panic!("Couldn't open {}: {}", path, e), - Ok(file) => file, - }; - let mut content = fs::read_to_string(&path).unwrap(); // TODO: Error Handling - // file.read_to_string(&mut content).unwrap(); let past_level = "level: ".to_string() + rule["level"].as_str().unwrap(); if new_level.starts_with("informational") { @@ -79,10 +69,10 @@ impl LevelTuning { if new_level.starts_with("critical") { content = content.replace(&past_level, "level: critical"); } - println!("{:?}", &content); let mut file = match File::options() .write(true) + .truncate(true) .open(&path) { Err(e) => panic!("Couldn't open {}: {}", path, e), From 026d18a60559786ebddb2e21a5623c3e24f8d458 Mon Sep 17 00:00:00 2001 From: itiB Date: Tue, 5 Apr 2022 01:30:11 +0900 Subject: [PATCH 09/42] Add: Error handlings --- src/main.rs | 9 +++++-- src/options/level_tuning.rs | 52 +++++++++++++++++++------------------ 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/src/main.rs b/src/main.rs index 50906427..b7fda73d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -119,15 +119,20 @@ impl App { .value_of("level-tuning") { if Path::new(level_tuning_config_path).exists() { - LevelTuning::run(level_tuning_config_path); + if let Err(err) = LevelTuning::run(level_tuning_config_path) { + AlertMessage::alert( + &mut BufWriter::new(std::io::stderr().lock()), + &err, + ).ok(); + } } else { AlertMessage::alert( &mut BufWriter::new(std::io::stderr().lock()), "Need rule_levels.txt file to use --level-tuning option", ) .ok(); - return; } + return; } } diff --git a/src/options/level_tuning.rs b/src/options/level_tuning.rs index ca0121bb..d195ff42 100644 --- a/src/options/level_tuning.rs +++ b/src/options/level_tuning.rs @@ -1,36 +1,35 @@ -use crate::detections::{configs, print::AlertMessage, utils}; +use crate::detections::{configs, utils}; use crate::filter; use crate::yaml::ParseYaml; use std::collections::HashMap; use std::fs::{self, File}; -use std::io::BufWriter; -use std::io::{Read, Write}; +use std::io::Write; pub struct LevelTuning {} impl LevelTuning { - pub fn run(level_tuning_config_path: &str) { + pub fn run(level_tuning_config_path: &str) -> Result<(), String> { let read_result = utils::read_csv(level_tuning_config_path); if read_result.is_err() { - AlertMessage::warn( - &mut BufWriter::new(std::io::stderr().lock()), - read_result.as_ref().unwrap_err(), - ) - .ok(); - return; + return Result::Err(read_result.as_ref().unwrap_err().to_string()); } + + // Read Tuning files let mut tuning_map: HashMap = HashMap::new(); - read_result.unwrap().into_iter().for_each(|line| { - if line.len() != 2 { - return; - } - let id = line.get(0).unwrap(); - // TODO: id validation - let level = line.get(1).unwrap(); - // TODO: level validation - // Cut Comments + read_result.unwrap().into_iter().try_for_each(|line| -> Result<(), String> { + let id = match line.get(0) { + Some(_id) => _id, // TODO: Validate id + _ => return Result::Err("Failed to read id...".to_string()) + }; + let level = match line.get(1) { + Some(_level) => _level, // TODO: Validate level + _ => return Result::Err("Failed to read level...".to_string()) + }; tuning_map.insert(id.to_string(), level.to_string()); - }); + Ok(()) + })?; + + // Read Rule files let mut rulefile_loader = ParseYaml::new(); let result_readdir = rulefile_loader.read_dir( configs::CONFIG @@ -43,15 +42,17 @@ impl LevelTuning { &filter::exclude_ids(), ); if result_readdir.is_err() { - let errmsg = format!("{}", result_readdir.unwrap_err()); - AlertMessage::warn(&mut BufWriter::new(std::io::stderr().lock()), &errmsg).ok(); - return; + return Result::Err(format!("{}", result_readdir.unwrap_err())); } + // Convert rule files for (path, rule) in rulefile_loader.files { if let Some(new_level) = tuning_map.get(rule["id"].as_str().unwrap()) { println!("path: {}", path); - let mut content = fs::read_to_string(&path).unwrap(); // TODO: Error Handling + let mut content = match fs::read_to_string(&path) { + Ok(_content) => _content, + Err(e) => return Result::Err(e.to_string()), + }; let past_level = "level: ".to_string() + rule["level"].as_str().unwrap(); if new_level.starts_with("informational") { @@ -75,8 +76,8 @@ impl LevelTuning { .truncate(true) .open(&path) { - Err(e) => panic!("Couldn't open {}: {}", path, e), Ok(file) => file, + Err(e) => return Result::Err(e.to_string()), }; file.write_all(content.as_bytes()).unwrap(); // TODO: use result @@ -87,5 +88,6 @@ impl LevelTuning { ); } } + Result::Ok(()) } } From 373dd0f8c74bbc056feca67805297d198976ed84 Mon Sep 17 00:00:00 2001 From: itiB Date: Tue, 5 Apr 2022 01:52:44 +0900 Subject: [PATCH 10/42] Add: id, level validation --- src/filter.rs | 2 +- src/main.rs | 6 ++---- src/options/level_tuning.rs | 28 +++++++++++++++++++--------- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/filter.rs b/src/filter.rs index 92293a62..0ce16e1a 100644 --- a/src/filter.rs +++ b/src/filter.rs @@ -10,7 +10,7 @@ use std::io::BufWriter; use std::io::{BufRead, BufReader}; lazy_static! { - static ref IDS_REGEX: Regex = + pub static ref IDS_REGEX: Regex = Regex::new(r"^[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12}$").unwrap(); } diff --git a/src/main.rs b/src/main.rs index b7fda73d..cbbdf7a7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -120,10 +120,8 @@ impl App { { if Path::new(level_tuning_config_path).exists() { if let Err(err) = LevelTuning::run(level_tuning_config_path) { - AlertMessage::alert( - &mut BufWriter::new(std::io::stderr().lock()), - &err, - ).ok(); + AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &err) + .ok(); } } else { AlertMessage::alert( diff --git a/src/options/level_tuning.rs b/src/options/level_tuning.rs index d195ff42..88e14ab5 100644 --- a/src/options/level_tuning.rs +++ b/src/options/level_tuning.rs @@ -4,7 +4,6 @@ use crate::yaml::ParseYaml; use std::collections::HashMap; use std::fs::{self, File}; use std::io::Write; - pub struct LevelTuning {} impl LevelTuning { @@ -18,11 +17,26 @@ impl LevelTuning { let mut tuning_map: HashMap = HashMap::new(); read_result.unwrap().into_iter().try_for_each(|line| -> Result<(), String> { let id = match line.get(0) { - Some(_id) => _id, // TODO: Validate id + Some(_id) => { + if !filter::IDS_REGEX.is_match(_id) { + return Result::Err(format!("Failed to read level tuning file. {} is not correct id format, fix it.", _id)); + } + _id + } _ => return Result::Err("Failed to read id...".to_string()) }; let level = match line.get(1) { - Some(_level) => _level, // TODO: Validate level + Some(_level) => { + if _level.starts_with("informational") + || _level.starts_with("low") + || _level.starts_with("medium") + || _level.starts_with("high") + || _level.starts_with("critical") { + _level + } else { + return Result::Err("level tuning file's level must in informational, low, medium, high, critical".to_string()) + } + } _ => return Result::Err("Failed to read level...".to_string()) }; tuning_map.insert(id.to_string(), level.to_string()); @@ -71,16 +85,12 @@ impl LevelTuning { content = content.replace(&past_level, "level: critical"); } - let mut file = match File::options() - .write(true) - .truncate(true) - .open(&path) - { + let mut file = match File::options().write(true).truncate(true).open(&path) { Ok(file) => file, Err(e) => return Result::Err(e.to_string()), }; - file.write_all(content.as_bytes()).unwrap(); // TODO: use result + file.write_all(content.as_bytes()).unwrap(); println!( "level: {} -> {}", rule["level"].as_str().unwrap(), From 015691e1295646c8912dd1784b8c16419c7804da Mon Sep 17 00:00:00 2001 From: itiB Date: Tue, 5 Apr 2022 01:59:56 +0900 Subject: [PATCH 11/42] mv: IDS_REGEX to configs file --- src/detections/configs.rs | 3 +++ src/filter.rs | 8 +------- src/options/level_tuning.rs | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index 0fcd1f08..a0fb6db8 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -7,6 +7,7 @@ use hashbrown::HashSet; use lazy_static::lazy_static; use std::io::BufWriter; use std::sync::RwLock; +use regex::Regex; lazy_static! { pub static ref CONFIG: RwLock = RwLock::new(ConfigReader::new()); pub static ref LEVELMAP: HashMap = { @@ -22,6 +23,8 @@ lazy_static! { "{}/eventkey_alias.txt", CONFIG.read().unwrap().folder_path )); + pub static ref IDS_REGEX: Regex = + Regex::new(r"^[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12}$").unwrap(); } #[derive(Clone)] diff --git a/src/filter.rs b/src/filter.rs index 0ce16e1a..366f82da 100644 --- a/src/filter.rs +++ b/src/filter.rs @@ -3,17 +3,11 @@ use crate::detections::print::AlertMessage; use crate::detections::print::ERROR_LOG_STACK; use crate::detections::print::QUIET_ERRORS_FLAG; use hashbrown::HashSet; -use lazy_static::lazy_static; use regex::Regex; use std::fs::File; use std::io::BufWriter; use std::io::{BufRead, BufReader}; -lazy_static! { - pub static ref IDS_REGEX: Regex = - Regex::new(r"^[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12}$").unwrap(); -} - #[derive(Debug)] pub struct DataFilterRule { pub regex_rule: Regex, @@ -74,7 +68,7 @@ impl RuleExclude { let v = v.unwrap().split('#').collect::>()[0] .trim() .to_string(); - if v.is_empty() || !IDS_REGEX.is_match(&v) { + if v.is_empty() || !configs::IDS_REGEX.is_match(&v) { // 空行は無視する。IDの検証 continue; } diff --git a/src/options/level_tuning.rs b/src/options/level_tuning.rs index 88e14ab5..bd909923 100644 --- a/src/options/level_tuning.rs +++ b/src/options/level_tuning.rs @@ -18,7 +18,7 @@ impl LevelTuning { read_result.unwrap().into_iter().try_for_each(|line| -> Result<(), String> { let id = match line.get(0) { Some(_id) => { - if !filter::IDS_REGEX.is_match(_id) { + if !configs::IDS_REGEX.is_match(_id) { return Result::Err(format!("Failed to read level tuning file. {} is not correct id format, fix it.", _id)); } _id From 9f8f12ec2fd7e5338221bc9e1a38ca2394ec3516 Mon Sep 17 00:00:00 2001 From: itiB Date: Tue, 5 Apr 2022 02:03:49 +0900 Subject: [PATCH 12/42] fix: level tuning's file name --- config/{rule_level.txt => level_tuning.txt} | 0 src/detections/configs.rs | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename config/{rule_level.txt => level_tuning.txt} (100%) diff --git a/config/rule_level.txt b/config/level_tuning.txt similarity index 100% rename from config/rule_level.txt rename to config/level_tuning.txt diff --git a/src/detections/configs.rs b/src/detections/configs.rs index a0fb6db8..5df4f981 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -97,8 +97,8 @@ fn build_app<'a>() -> ArgMatches<'a> { .setting(AppSettings::VersionlessSubcommands) .arg( // TODO: When update claps to 3.x, these can write in usage texts... - Arg::from_usage("--level-tuning=[RULE_LEVEL_FILE] 'Fix rule file's level'") - .default_value("./config/rule_level.txt"), + Arg::from_usage("--level-tuning=[LEVEL_TUNING_FILE] 'Fix rule file's level'") + .default_value("./config/level_tuning.txt"), ) .usage(usages) .args_from_usage(usages) From 90822aa563584f0752549f774d5c2c3b40b6af5a Mon Sep 17 00:00:00 2001 From: itiB Date: Tue, 5 Apr 2022 02:04:10 +0900 Subject: [PATCH 13/42] Cargo fmt --- src/detections/configs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index 5df4f981..f17ea5a0 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -5,9 +5,9 @@ use clap::{App, AppSettings, Arg, ArgMatches}; use hashbrown::HashMap; use hashbrown::HashSet; use lazy_static::lazy_static; +use regex::Regex; use std::io::BufWriter; use std::sync::RwLock; -use regex::Regex; lazy_static! { pub static ref CONFIG: RwLock = RwLock::new(ConfigReader::new()); pub static ref LEVELMAP: HashMap = { From 51f8d405f8c86c17d8ea7fb13854aedd20c14b50 Mon Sep 17 00:00:00 2001 From: itiB Date: Wed, 6 Apr 2022 01:34:48 +0900 Subject: [PATCH 14/42] Add: test --- src/options/level_tuning.rs | 57 +++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/src/options/level_tuning.rs b/src/options/level_tuning.rs index bd909923..f9e2ab66 100644 --- a/src/options/level_tuning.rs +++ b/src/options/level_tuning.rs @@ -101,3 +101,60 @@ impl LevelTuning { Result::Ok(()) } } + +mod tests { + use super::*; + + #[test] + fn rule_level_failed_to_open_file() -> Result<(), String> { + let level_tuning_config_path = "./none.txt"; + let res = LevelTuning::run(level_tuning_config_path); + let expected = Result::Err("Cannot open file. [file:./none.txt]".to_string()); + assert_eq!(res, expected); + Ok(()) + } + + #[test] + fn rule_level_id_error_file() -> Result<(), String> { + let level_tuning_config_path = "./test_files/config/level_tuning_error1.txt"; + let res = LevelTuning::run(level_tuning_config_path); + let expected = Result::Err("Failed to read level tuning file. 12345678-1234-1234-1234-12 is not correct id format, fix it.".to_string()); + assert_eq!(res, expected); + Ok(()) + } + + #[test] + fn rule_level_level_error_file() -> Result<(), String> { + let level_tuning_config_path = "./test_files/config/level_tuning_error2.txt"; + let res = LevelTuning::run(level_tuning_config_path); + let expected = Result::Err("level tuning file's level must in informational, low, medium, high, critical".to_string()); + assert_eq!(res, expected); + Ok(()) + } + + + // TODO: make test option for read ./test_files/rules/ dir. + // #[test] + // fn test_detect_mutiple_regex_and() { + // let level_tuning_config_path = "./test_files/config/level_tuning.txt"; + // let rule_str = r#" + // id: 12345678-1234-1234-1234-123456789012 + // level: informational + // "#; + + // let expected_rule = r#" + // level: high + // "#; + + // let path = "test_files/rules/level_tuning_sample.yml"; + // let mut file = File::create(path).unwrap(); + // let buf = rule_str.as_bytes(); + // file.write_all(buf).unwrap(); + // file.flush().unwrap(); + + // let res = LevelTuning::run(level_tuning_config_path); + // assert_eq!(res, Ok(())); + // assert_eq!(fs::read_to_string(path).unwrap(), expected_rule); + // fs::remove_file(path).unwrap(); + // } +} From 52bc918cfb4c5fec75eb8ce2cec4545e60aeb1be Mon Sep 17 00:00:00 2001 From: itiB Date: Wed, 6 Apr 2022 01:54:09 +0900 Subject: [PATCH 15/42] Add: README.md --- README-Japanese.md | 14 ++++++++++++++ README.md | 15 +++++++++++++++ config/level_tuning.txt | 2 +- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/README-Japanese.md b/README-Japanese.md index 9e13daae..5e5a49c5 100644 --- a/README-Japanese.md +++ b/README-Japanese.md @@ -60,6 +60,7 @@ Hayabusaは、日本の[Yamato Security](https://yamatosecurity.connpass.com/) - [Hayabusa v.s. 変換されたSigmaルール](#hayabusa-vs-変換されたsigmaルール) - [検知ルールのチューニング](#検知ルールのチューニング) - [イベントIDフィルタリング](#イベントidフィルタリング) + - [検知レベルのチューニング](#検知レベルのチューニング) - [その他のWindowsイベントログ解析ツールおよび関連プロジェクト](#その他のwindowsイベントログ解析ツールおよび関連プロジェクト) - [Sigmaをサポートする他の類似ツールとの比較](#sigmaをサポートする他の類似ツールとの比較) - [コミュニティによるドキュメンテーション](#コミュニティによるドキュメンテーション) @@ -506,6 +507,19 @@ Sigmaルールは、最初にHayabusaルール形式に変換する必要があ 最高のパフォーマンスを得たい場合はこのリストを使用してください。ただし、検出漏れの可能性が若干あることにご注意ください。 +## 検知レベルのチューニング + +Hayabusaルール、Sigmaルールはそれぞれの作者が検知した際の脅威レベルを決めています。 +ユーザが独自の脅威レベルに設定するには `./config/level_tuning.txt` に変換情報を書き、`hayabusa --level-tuning` を実行することでルールファイルが書き換えられます。 +ルールファイルが直接書き換えられることに注意して使用してください。 + +`./config/level_tuning.txt` の例 +``` +id,next_level +00000000-0000-0000-0000-000000000000,informational # sample leveltunig line +``` +ルールディレクトリ内の `00000000-0000-0000-0000-000000000000` に該当するルールの脅威レベルが `informational` に書き換えられます。 + # その他のWindowsイベントログ解析ツールおよび関連プロジェクト 「すべてを統治する1つのツール」というものはなく、それぞれにメリットがあるため、これらの他の優れたツールやプロジェクトをチェックして、どれが気に入ったかを確認することをお勧めします。 diff --git a/README.md b/README.md index 17b57bc4..5742c975 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ Hayabusa is a **Windows event log fast forensics timeline generator** and **thre - [Hayabusa v.s. Converted Sigma Rules](#hayabusa-vs-converted-sigma-rules) - [Detection Rule Tuning](#detection-rule-tuning) - [Event ID Filtering](#event-id-filtering) + - [Detection Level Tuning](#detection-level-tuning) - [Other Windows Event Log Analyzers and Related Projects](#other-windows-event-log-analyzers-and-related-projects) - [Comparison To Other Similar Tools](#comparison-to-other-similar-tools) - [Community Documentation](#community-documentation) @@ -498,6 +499,20 @@ We have provided a sample ID filter list at [`config/target_eventids_sample.txt` Please use this list if you want the best performance but be aware that there is a slight possibility for missing events (false negatives). +## Detection Level Tuning + +The Hayabusa rule and Sigma rule determine the threat level when each author detects it. +To set the user to their own threat level, write the conversion information in `./config/level_tuning.txt` and execute` hayabusa --level-tuning` to rewrite the rule file. +Please note that the rule file will be rewritten directly. + +`./config/level_tuning.txt` Sample +``` +id,next_level +00000000-0000-0000-0000-000000000000,informational # sample leveltunig line +``` + +The threat level of the rule corresponding to `00000000-0000-0000-0000-000000000000` in the rules directory is rewritten to` informational`. + # Other Windows Event Log Analyzers and Related Projects There is no "one tool to rule them all" and we have found that each has its own merits so we recommend checking out these other great tools and projects and seeing which ones you like. diff --git a/config/level_tuning.txt b/config/level_tuning.txt index 2f8de0ea..7f43ac4d 100644 --- a/config/level_tuning.txt +++ b/config/level_tuning.txt @@ -1,2 +1,2 @@ id,next_level -671bb7e3-a020-4824-a00e-2ee5b55f385e,high # CACTUSTORCH Remote Thread Creation \ No newline at end of file +00000000-0000-0000-0000-000000000000,informational # sample leveltunig line From 04c0e6ae07025e5a9441a32a91986de2590e85d5 Mon Sep 17 00:00:00 2001 From: itiB Date: Wed, 6 Apr 2022 01:55:19 +0900 Subject: [PATCH 16/42] Cargo fmt --- src/options/level_tuning.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/options/level_tuning.rs b/src/options/level_tuning.rs index f9e2ab66..c59f61c7 100644 --- a/src/options/level_tuning.rs +++ b/src/options/level_tuning.rs @@ -127,12 +127,14 @@ mod tests { fn rule_level_level_error_file() -> Result<(), String> { let level_tuning_config_path = "./test_files/config/level_tuning_error2.txt"; let res = LevelTuning::run(level_tuning_config_path); - let expected = Result::Err("level tuning file's level must in informational, low, medium, high, critical".to_string()); + let expected = Result::Err( + "level tuning file's level must in informational, low, medium, high, critical" + .to_string(), + ); assert_eq!(res, expected); Ok(()) } - // TODO: make test option for read ./test_files/rules/ dir. // #[test] // fn test_detect_mutiple_regex_and() { From 11b5a3d394dc2b4c38cb880977a29413ac07439b Mon Sep 17 00:00:00 2001 From: itiB Date: Wed, 6 Apr 2022 02:04:23 +0900 Subject: [PATCH 17/42] Use #[cfg(test)] --- src/options/level_tuning.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/options/level_tuning.rs b/src/options/level_tuning.rs index c59f61c7..6619e09e 100644 --- a/src/options/level_tuning.rs +++ b/src/options/level_tuning.rs @@ -102,6 +102,7 @@ impl LevelTuning { } } +#[cfg(test)] mod tests { use super::*; From dab91e5e61d537901d030b05043a763ce1f6beb6 Mon Sep 17 00:00:00 2001 From: DustInDark Date: Wed, 6 Apr 2022 22:34:32 +0900 Subject: [PATCH 18/42] fixed level tuning test and added test files #390 --- src/options/level_tuning.rs | 65 +++++++++++++++-------- test_files/config/level_tuning_error1.txt | 2 + test_files/config/level_tuning_error2.txt | 2 + 3 files changed, 48 insertions(+), 21 deletions(-) create mode 100644 test_files/config/level_tuning_error1.txt create mode 100644 test_files/config/level_tuning_error2.txt diff --git a/src/options/level_tuning.rs b/src/options/level_tuning.rs index 6619e09e..2f82c8ce 100644 --- a/src/options/level_tuning.rs +++ b/src/options/level_tuning.rs @@ -104,6 +104,10 @@ impl LevelTuning { #[cfg(test)] mod tests { + + use crate::{filter::RuleExclude, yaml}; + use hashbrown::HashSet; + use super::*; #[test] @@ -136,28 +140,47 @@ mod tests { Ok(()) } - // TODO: make test option for read ./test_files/rules/ dir. - // #[test] - // fn test_detect_mutiple_regex_and() { - // let level_tuning_config_path = "./test_files/config/level_tuning.txt"; - // let rule_str = r#" - // id: 12345678-1234-1234-1234-123456789012 - // level: informational - // "#; + #[test] + fn test_detect_mutiple_regex_and() { + let level_tuning_config_path = "./test_files/config/level_tuning.txt"; + let rule_str = r#" + id: 12345678-1234-1234-1234-123456789012 + level: informational + "#; - // let expected_rule = r#" - // level: high - // "#; + let expected_rule = r#" + level: high + "#; - // let path = "test_files/rules/level_tuning_sample.yml"; - // let mut file = File::create(path).unwrap(); - // let buf = rule_str.as_bytes(); - // file.write_all(buf).unwrap(); - // file.flush().unwrap(); + let path = "test_files/rules/level_tuning_sample.yml"; + let mut file = File::create(path).unwrap(); + let buf = rule_str.as_bytes(); + file.write_all(buf).unwrap(); + file.flush().unwrap(); - // let res = LevelTuning::run(level_tuning_config_path); - // assert_eq!(res, Ok(())); - // assert_eq!(fs::read_to_string(path).unwrap(), expected_rule); - // fs::remove_file(path).unwrap(); - // } + let mut parser = yaml::ParseYaml::new(); + parser + .read_dir( + "test_files/rules", + "informational", + &RuleExclude { + no_use_rule: HashSet::new(), + }, + ) + .ok(); + + for (_filepath, yaml) in parser.files { + if yaml["id"].as_str().unwrap_or(&String::default()) + == "12345678-1234-1234-1234-123456789012" + { + println!("{}", _filepath); + assert_eq!("high", yaml["level"].as_str().unwrap()); + } + } + + let res = LevelTuning::run(level_tuning_config_path); + assert_eq!(res, Ok(())); + assert_eq!(fs::read_to_string(path).unwrap(), expected_rule); + fs::remove_file(path).unwrap(); + } } diff --git a/test_files/config/level_tuning_error1.txt b/test_files/config/level_tuning_error1.txt new file mode 100644 index 00000000..c91f760d --- /dev/null +++ b/test_files/config/level_tuning_error1.txt @@ -0,0 +1,2 @@ +id,next_level +12345678-1234-1234-1234-12,informational # sample leveltunig line diff --git a/test_files/config/level_tuning_error2.txt b/test_files/config/level_tuning_error2.txt new file mode 100644 index 00000000..33788388 --- /dev/null +++ b/test_files/config/level_tuning_error2.txt @@ -0,0 +1,2 @@ +id,next_level +00000000-0000-0000-0000-000000000000,no_exist_level # sample leveltunig line From cd8c856d0520b43858c14f0cd30aa29dd310af33 Mon Sep 17 00:00:00 2001 From: DustInDark Date: Wed, 6 Apr 2022 22:46:35 +0900 Subject: [PATCH 19/42] changed level_tuning.txt header from next_level to new_level --- README-Japanese.md | 4 ++-- README.md | 4 ++-- config/level_tuning.txt | 4 ++-- test_files/config/level_tuning_error1.txt | 4 ++-- test_files/config/level_tuning_error2.txt | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README-Japanese.md b/README-Japanese.md index 5e5a49c5..377d7a38 100644 --- a/README-Japanese.md +++ b/README-Japanese.md @@ -515,8 +515,8 @@ Hayabusaルール、Sigmaルールはそれぞれの作者が検知した際の `./config/level_tuning.txt` の例 ``` -id,next_level -00000000-0000-0000-0000-000000000000,informational # sample leveltunig line +id,new_level +00000000-0000-0000-0000-000000000000,informational # sample level tuning line ``` ルールディレクトリ内の `00000000-0000-0000-0000-000000000000` に該当するルールの脅威レベルが `informational` に書き換えられます。 diff --git a/README.md b/README.md index 5742c975..4db65105 100644 --- a/README.md +++ b/README.md @@ -507,8 +507,8 @@ Please note that the rule file will be rewritten directly. `./config/level_tuning.txt` Sample ``` -id,next_level -00000000-0000-0000-0000-000000000000,informational # sample leveltunig line +id,new_level +00000000-0000-0000-0000-000000000000,informational # sample level tuning line ``` The threat level of the rule corresponding to `00000000-0000-0000-0000-000000000000` in the rules directory is rewritten to` informational`. diff --git a/config/level_tuning.txt b/config/level_tuning.txt index 7f43ac4d..8482d822 100644 --- a/config/level_tuning.txt +++ b/config/level_tuning.txt @@ -1,2 +1,2 @@ -id,next_level -00000000-0000-0000-0000-000000000000,informational # sample leveltunig line +id,new_level +00000000-0000-0000-0000-000000000000,informational # sample level tuning line diff --git a/test_files/config/level_tuning_error1.txt b/test_files/config/level_tuning_error1.txt index c91f760d..bed1b758 100644 --- a/test_files/config/level_tuning_error1.txt +++ b/test_files/config/level_tuning_error1.txt @@ -1,2 +1,2 @@ -id,next_level -12345678-1234-1234-1234-12,informational # sample leveltunig line +id,new_level +12345678-1234-1234-1234-12,informational # sample level tuning line diff --git a/test_files/config/level_tuning_error2.txt b/test_files/config/level_tuning_error2.txt index 33788388..c8c1c1e4 100644 --- a/test_files/config/level_tuning_error2.txt +++ b/test_files/config/level_tuning_error2.txt @@ -1,2 +1,2 @@ -id,next_level -00000000-0000-0000-0000-000000000000,no_exist_level # sample leveltunig line +id,new_level +00000000-0000-0000-0000-000000000000,no_exist_level # sample level tuning line From 3b4c4dd36e0d68531006401f752b086d61e77062 Mon Sep 17 00:00:00 2001 From: DustInDark Date: Wed, 6 Apr 2022 22:47:28 +0900 Subject: [PATCH 20/42] fixed convert miss change to low level --- src/options/level_tuning.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/options/level_tuning.rs b/src/options/level_tuning.rs index 2f82c8ce..a00ce355 100644 --- a/src/options/level_tuning.rs +++ b/src/options/level_tuning.rs @@ -73,7 +73,7 @@ impl LevelTuning { content = content.replace(&past_level, "level: informational"); } if new_level.starts_with("low") { - content = content.replace(&past_level, "level: informational"); + content = content.replace(&past_level, "level: low"); } if new_level.starts_with("medium") { content = content.replace(&past_level, "level: medium"); From 0c27b13c85d4667aea3a27b74f4a4883cdf65279 Mon Sep 17 00:00:00 2001 From: DustInDark Date: Wed, 6 Apr 2022 23:28:55 +0900 Subject: [PATCH 21/42] added run args rules path to check test easy #390 --- src/main.rs | 7 ++++++- src/options/level_tuning.rs | 26 ++++++++++---------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/main.rs b/src/main.rs index cbbdf7a7..133c8d40 100644 --- a/src/main.rs +++ b/src/main.rs @@ -119,7 +119,12 @@ impl App { .value_of("level-tuning") { if Path::new(level_tuning_config_path).exists() { - if let Err(err) = LevelTuning::run(level_tuning_config_path) { + if let Err(err) = LevelTuning::run(level_tuning_config_path, configs::CONFIG + .read() + .unwrap() + .args + .value_of("rules") + .unwrap_or("rules")) { AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &err) .ok(); } diff --git a/src/options/level_tuning.rs b/src/options/level_tuning.rs index a00ce355..3a0510fa 100644 --- a/src/options/level_tuning.rs +++ b/src/options/level_tuning.rs @@ -7,7 +7,7 @@ use std::io::Write; pub struct LevelTuning {} impl LevelTuning { - pub fn run(level_tuning_config_path: &str) -> Result<(), String> { + pub fn run(level_tuning_config_path: &str, rules_path: &str) -> Result<(), String> { let read_result = utils::read_csv(level_tuning_config_path); if read_result.is_err() { return Result::Err(read_result.as_ref().unwrap_err().to_string()); @@ -45,16 +45,8 @@ impl LevelTuning { // Read Rule files let mut rulefile_loader = ParseYaml::new(); - let result_readdir = rulefile_loader.read_dir( - configs::CONFIG - .read() - .unwrap() - .args - .value_of("rules") - .unwrap_or("rules"), - "informational", - &filter::exclude_ids(), - ); + let result_readdir = + rulefile_loader.read_dir(rules_path, "informational", &filter::exclude_ids()); if result_readdir.is_err() { return Result::Err(format!("{}", result_readdir.unwrap_err())); } @@ -113,7 +105,7 @@ mod tests { #[test] fn rule_level_failed_to_open_file() -> Result<(), String> { let level_tuning_config_path = "./none.txt"; - let res = LevelTuning::run(level_tuning_config_path); + let res = LevelTuning::run(level_tuning_config_path, ""); let expected = Result::Err("Cannot open file. [file:./none.txt]".to_string()); assert_eq!(res, expected); Ok(()) @@ -122,7 +114,7 @@ mod tests { #[test] fn rule_level_id_error_file() -> Result<(), String> { let level_tuning_config_path = "./test_files/config/level_tuning_error1.txt"; - let res = LevelTuning::run(level_tuning_config_path); + let res = LevelTuning::run(level_tuning_config_path, ""); let expected = Result::Err("Failed to read level tuning file. 12345678-1234-1234-1234-12 is not correct id format, fix it.".to_string()); assert_eq!(res, expected); Ok(()) @@ -131,7 +123,7 @@ mod tests { #[test] fn rule_level_level_error_file() -> Result<(), String> { let level_tuning_config_path = "./test_files/config/level_tuning_error2.txt"; - let res = LevelTuning::run(level_tuning_config_path); + let res = LevelTuning::run(level_tuning_config_path, ""); let expected = Result::Err( "level tuning file's level must in informational, low, medium, high, critical" .to_string(), @@ -149,6 +141,7 @@ mod tests { "#; let expected_rule = r#" + id: 12345678-1234-1234-1234-123456789012 level: high "#; @@ -158,6 +151,9 @@ mod tests { file.write_all(buf).unwrap(); file.flush().unwrap(); + let res = LevelTuning::run(level_tuning_config_path, path); + assert_eq!(res, Ok(())); + let mut parser = yaml::ParseYaml::new(); parser .read_dir( @@ -178,8 +174,6 @@ mod tests { } } - let res = LevelTuning::run(level_tuning_config_path); - assert_eq!(res, Ok(())); assert_eq!(fs::read_to_string(path).unwrap(), expected_rule); fs::remove_file(path).unwrap(); } From 6931724ec419732f7fd4ce493fff3e699d416c16 Mon Sep 17 00:00:00 2001 From: DustInDark Date: Wed, 6 Apr 2022 23:30:32 +0900 Subject: [PATCH 22/42] fixed comment out processing in level_tuning.txt --- src/options/level_tuning.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/options/level_tuning.rs b/src/options/level_tuning.rs index 3a0510fa..94c9aef8 100644 --- a/src/options/level_tuning.rs +++ b/src/options/level_tuning.rs @@ -32,7 +32,7 @@ impl LevelTuning { || _level.starts_with("medium") || _level.starts_with("high") || _level.starts_with("critical") { - _level + _level.split('#').collect::>()[0] } else { return Result::Err("level tuning file's level must in informational, low, medium, high, critical".to_string()) } From a35e8ad5cbec077fb295e6caccc89ed395cba8da Mon Sep 17 00:00:00 2001 From: DustInDark Date: Thu, 7 Apr 2022 00:08:32 +0900 Subject: [PATCH 23/42] fixed config to show level-tuning option --- src/detections/configs.rs | 8 ++------ src/main.rs | 33 ++++++++++++++++++--------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index f17ea5a0..5f97037e 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -1,7 +1,7 @@ use crate::detections::print::AlertMessage; use crate::detections::utils; use chrono::{DateTime, Utc}; -use clap::{App, AppSettings, Arg, ArgMatches}; +use clap::{App, AppSettings, ArgMatches}; use hashbrown::HashMap; use hashbrown::HashSet; use lazy_static::lazy_static; @@ -80,6 +80,7 @@ fn build_app<'a>() -> ArgMatches<'a> { -u --update-rules 'Update to the latest rules in the hayabusa-rules github repository.' -m --min-level=[LEVEL] 'Minimum level for rules. (Default: informational)' -l --live-analysis 'Analyze the local C:\\Windows\\System32\\winevt\\Logs folder (Windows Only. Administrator privileges required.)' + --level-tuning= 'Fix rule file's level [default: ./config/level_tuning.txt]' --start-timeline=[STARTTIMELINE] 'Start time of the event logs to load. (Example: '2018/11/28 12:00:00 +09:00')' --end-timeline=[ENDTIMELINE] 'End time of the event logs to load. (Example: '2018/11/28 12:00:00 +09:00')' --rfc-2822 'Output date and time in RFC 2822 format. (Example: Mon, 07 Aug 2006 12:34:56 -0600)' @@ -95,11 +96,6 @@ fn build_app<'a>() -> ArgMatches<'a> { .version("1.1.0") .author("Yamato Security (https://github.com/Yamato-Security/hayabusa)") .setting(AppSettings::VersionlessSubcommands) - .arg( - // TODO: When update claps to 3.x, these can write in usage texts... - Arg::from_usage("--level-tuning=[LEVEL_TUNING_FILE] 'Fix rule file's level'") - .default_value("./config/level_tuning.txt"), - ) .usage(usages) .args_from_usage(usages) .get_matches() diff --git a/src/main.rs b/src/main.rs index 133c8d40..9c08e7d6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -112,31 +112,34 @@ impl App { .args .is_present("level-tuning") { - if let Some(level_tuning_config_path) = configs::CONFIG + let level_tuning_config_path = configs::CONFIG .read() .unwrap() .args .value_of("level-tuning") - { - if Path::new(level_tuning_config_path).exists() { - if let Err(err) = LevelTuning::run(level_tuning_config_path, configs::CONFIG + .unwrap_or("./config/level_tuning.txt") + .to_string(); + + if Path::new(&level_tuning_config_path).exists() { + if let Err(err) = LevelTuning::run( + &level_tuning_config_path, + configs::CONFIG .read() .unwrap() .args .value_of("rules") - .unwrap_or("rules")) { - AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &err) - .ok(); - } - } else { - AlertMessage::alert( - &mut BufWriter::new(std::io::stderr().lock()), - "Need rule_levels.txt file to use --level-tuning option", - ) - .ok(); + .unwrap_or("rules"), + ) { + AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &err).ok(); } - return; + } else { + AlertMessage::alert( + &mut BufWriter::new(std::io::stderr().lock()), + "Need rule_levels.txt file to use --level-tuning option", + ) + .ok(); } + return; } if !Path::new("./config").exists() { From e715935bb6e11c2451aedcfa4c15c7ce30c4ad6c Mon Sep 17 00:00:00 2001 From: DustInDark Date: Thu, 7 Apr 2022 00:17:51 +0900 Subject: [PATCH 24/42] fixed level-tuning option usage from required to option --- src/detections/configs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index 5f97037e..34c581a7 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -80,7 +80,7 @@ fn build_app<'a>() -> ArgMatches<'a> { -u --update-rules 'Update to the latest rules in the hayabusa-rules github repository.' -m --min-level=[LEVEL] 'Minimum level for rules. (Default: informational)' -l --live-analysis 'Analyze the local C:\\Windows\\System32\\winevt\\Logs folder (Windows Only. Administrator privileges required.)' - --level-tuning= 'Fix rule file's level [default: ./config/level_tuning.txt]' + --level-tuning=[LEVEL_TUNING_FILE] 'Fix rule file's level [default: ./config/level_tuning.txt]' --start-timeline=[STARTTIMELINE] 'Start time of the event logs to load. (Example: '2018/11/28 12:00:00 +09:00')' --end-timeline=[ENDTIMELINE] 'End time of the event logs to load. (Example: '2018/11/28 12:00:00 +09:00')' --rfc-2822 'Output date and time in RFC 2822 format. (Example: Mon, 07 Aug 2006 12:34:56 -0600)' From e119ba8f1418d6a6babd8bb8af5d85e66f8b1b61 Mon Sep 17 00:00:00 2001 From: itiB Date: Thu, 7 Apr 2022 01:24:26 +0900 Subject: [PATCH 25/42] Fix: test file's path was incorrect --- src/options/level_tuning.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/options/level_tuning.rs b/src/options/level_tuning.rs index 94c9aef8..cc5e8adb 100644 --- a/src/options/level_tuning.rs +++ b/src/options/level_tuning.rs @@ -145,7 +145,7 @@ mod tests { level: high "#; - let path = "test_files/rules/level_tuning_sample.yml"; + let path = "test_files/rules/level_tuning.txt"; let mut file = File::create(path).unwrap(); let buf = rule_str.as_bytes(); file.write_all(buf).unwrap(); From 4056975b1dae8d4ab359681d9689330784855503 Mon Sep 17 00:00:00 2001 From: itiB Date: Thu, 7 Apr 2022 01:33:35 +0900 Subject: [PATCH 26/42] Add: add test_files/config/level_tuning.txt --- test_files/config/level_tuning.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 test_files/config/level_tuning.txt diff --git a/test_files/config/level_tuning.txt b/test_files/config/level_tuning.txt new file mode 100644 index 00000000..58a9604a --- /dev/null +++ b/test_files/config/level_tuning.txt @@ -0,0 +1,2 @@ +id,next_level +12345678-1234-1234-1234-123456789012,high \ No newline at end of file From f3a679d845936742a7867c0d6b50aaab121ed3d0 Mon Sep 17 00:00:00 2001 From: itiB Date: Thu, 7 Apr 2022 01:44:02 +0900 Subject: [PATCH 27/42] Add: Flush method. --- src/options/level_tuning.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/options/level_tuning.rs b/src/options/level_tuning.rs index cc5e8adb..4809e0d2 100644 --- a/src/options/level_tuning.rs +++ b/src/options/level_tuning.rs @@ -83,6 +83,7 @@ impl LevelTuning { }; file.write_all(content.as_bytes()).unwrap(); + file.flush().unwrap(); println!( "level: {} -> {}", rule["level"].as_str().unwrap(), From b8c442ca22e0c7d632d53567ab3934dcd78cef34 Mon Sep 17 00:00:00 2001 From: DustInDark Date: Thu, 7 Apr 2022 01:55:03 +0900 Subject: [PATCH 28/42] inserted debug data --- src/options/level_tuning.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/options/level_tuning.rs b/src/options/level_tuning.rs index 94c9aef8..a275ebe5 100644 --- a/src/options/level_tuning.rs +++ b/src/options/level_tuning.rs @@ -164,8 +164,9 @@ mod tests { }, ) .ok(); - + println!("{:?}", parser); for (_filepath, yaml) in parser.files { + println!(); if yaml["id"].as_str().unwrap_or(&String::default()) == "12345678-1234-1234-1234-123456789012" { From 0bac82699f6bf7516dc7fcb4d465a78b600ea892 Mon Sep 17 00:00:00 2001 From: DustInDark Date: Thu, 7 Apr 2022 02:22:57 +0900 Subject: [PATCH 29/42] reverted config usage --- src/detections/configs.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index 34c581a7..f17ea5a0 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -1,7 +1,7 @@ use crate::detections::print::AlertMessage; use crate::detections::utils; use chrono::{DateTime, Utc}; -use clap::{App, AppSettings, ArgMatches}; +use clap::{App, AppSettings, Arg, ArgMatches}; use hashbrown::HashMap; use hashbrown::HashSet; use lazy_static::lazy_static; @@ -80,7 +80,6 @@ fn build_app<'a>() -> ArgMatches<'a> { -u --update-rules 'Update to the latest rules in the hayabusa-rules github repository.' -m --min-level=[LEVEL] 'Minimum level for rules. (Default: informational)' -l --live-analysis 'Analyze the local C:\\Windows\\System32\\winevt\\Logs folder (Windows Only. Administrator privileges required.)' - --level-tuning=[LEVEL_TUNING_FILE] 'Fix rule file's level [default: ./config/level_tuning.txt]' --start-timeline=[STARTTIMELINE] 'Start time of the event logs to load. (Example: '2018/11/28 12:00:00 +09:00')' --end-timeline=[ENDTIMELINE] 'End time of the event logs to load. (Example: '2018/11/28 12:00:00 +09:00')' --rfc-2822 'Output date and time in RFC 2822 format. (Example: Mon, 07 Aug 2006 12:34:56 -0600)' @@ -96,6 +95,11 @@ fn build_app<'a>() -> ArgMatches<'a> { .version("1.1.0") .author("Yamato Security (https://github.com/Yamato-Security/hayabusa)") .setting(AppSettings::VersionlessSubcommands) + .arg( + // TODO: When update claps to 3.x, these can write in usage texts... + Arg::from_usage("--level-tuning=[LEVEL_TUNING_FILE] 'Fix rule file's level'") + .default_value("./config/level_tuning.txt"), + ) .usage(usages) .args_from_usage(usages) .get_matches() From 4cc8d80d20e4b6bfb3dd4c87ced084c0b3d4ab10 Mon Sep 17 00:00:00 2001 From: DustInDark Date: Thu, 7 Apr 2022 02:30:15 +0900 Subject: [PATCH 30/42] fixed test yaml file path --- src/options/level_tuning.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/options/level_tuning.rs b/src/options/level_tuning.rs index 0281cb99..869e9b15 100644 --- a/src/options/level_tuning.rs +++ b/src/options/level_tuning.rs @@ -146,7 +146,7 @@ mod tests { level: high "#; - let path = "test_files/rules/level_tuning.txt"; + let path = "test_files/rules/level_tuning_test.yml"; let mut file = File::create(path).unwrap(); let buf = rule_str.as_bytes(); file.write_all(buf).unwrap(); @@ -165,13 +165,10 @@ mod tests { }, ) .ok(); - println!("{:?}", parser); for (_filepath, yaml) in parser.files { - println!(); if yaml["id"].as_str().unwrap_or(&String::default()) == "12345678-1234-1234-1234-123456789012" { - println!("{}", _filepath); assert_eq!("high", yaml["level"].as_str().unwrap()); } } From 8061733fd90e1d263b94bf1b76aaf8478721491a Mon Sep 17 00:00:00 2001 From: itiB Date: Sun, 10 Apr 2022 00:50:40 +0900 Subject: [PATCH 31/42] Fix: show usage when hayabusa has no args --- src/main.rs | 86 ++++++++++++++++++++++++++++------------------------- 1 file changed, 45 insertions(+), 41 deletions(-) diff --git a/src/main.rs b/src/main.rs index 9c08e7d6..d9d67b7b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -73,6 +73,14 @@ impl App { fn exec(&mut self) { let analysis_start_time: DateTime = Local::now(); + + // Show usage when no arguments. + if std::env::args().len() == 1 { + println!("{}", configs::CONFIG.read().unwrap().args.usage()); + println!(); + return; + } + if !configs::CONFIG.read().unwrap().args.is_present("quiet") { self.output_logo(); println!(); @@ -106,42 +114,6 @@ impl App { return; } - if configs::CONFIG - .read() - .unwrap() - .args - .is_present("level-tuning") - { - let level_tuning_config_path = configs::CONFIG - .read() - .unwrap() - .args - .value_of("level-tuning") - .unwrap_or("./config/level_tuning.txt") - .to_string(); - - if Path::new(&level_tuning_config_path).exists() { - if let Err(err) = LevelTuning::run( - &level_tuning_config_path, - configs::CONFIG - .read() - .unwrap() - .args - .value_of("rules") - .unwrap_or("rules"), - ) { - AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &err).ok(); - } - } else { - AlertMessage::alert( - &mut BufWriter::new(std::io::stderr().lock()), - "Need rule_levels.txt file to use --level-tuning option", - ) - .ok(); - } - return; - } - if !Path::new("./config").exists() { AlertMessage::alert( &mut BufWriter::new(std::io::stderr().lock()), @@ -150,11 +122,7 @@ impl App { .ok(); return; } - if configs::CONFIG.read().unwrap().args.args.is_empty() { - println!("{}", configs::CONFIG.read().unwrap().args.usage()); - println!(); - return; - } + if let Some(csv_path) = configs::CONFIG.read().unwrap().args.value_of("output") { if Path::new(csv_path).exists() { AlertMessage::alert( @@ -220,7 +188,43 @@ impl App { { self.print_contributors(); return; + } else if configs::CONFIG + .read() + .unwrap() + .args + .is_present("level-tuning") + { + println!("Level_tuning: {:?}", std::env::args().len()); + let level_tuning_config_path = configs::CONFIG + .read() + .unwrap() + .args + .value_of("level-tuning") + .unwrap_or("./config/level_tuning.txt") + .to_string(); + + if Path::new(&level_tuning_config_path).exists() { + if let Err(err) = LevelTuning::run( + &level_tuning_config_path, + configs::CONFIG + .read() + .unwrap() + .args + .value_of("rules") + .unwrap_or("rules"), + ) { + AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &err).ok(); + } + } else { + AlertMessage::alert( + &mut BufWriter::new(std::io::stderr().lock()), + "Need rule_levels.txt file to use --level-tuning option", + ) + .ok(); + } + return; } + let analysis_end_time: DateTime = Local::now(); let analysis_duration = analysis_end_time.signed_duration_since(analysis_start_time); println!(); From bb834618d6f319bd96d6c26a021f02e7f1acd1a7 Mon Sep 17 00:00:00 2001 From: itiB Date: Sun, 10 Apr 2022 00:52:06 +0900 Subject: [PATCH 32/42] rm: debug line --- src/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index d9d67b7b..b5000d06 100644 --- a/src/main.rs +++ b/src/main.rs @@ -194,7 +194,6 @@ impl App { .args .is_present("level-tuning") { - println!("Level_tuning: {:?}", std::env::args().len()); let level_tuning_config_path = configs::CONFIG .read() .unwrap() From e6a74090a3ddc85399906aeaca8a4fd5ba98832a Mon Sep 17 00:00:00 2001 From: DustInDark Date: Sun, 10 Apr 2022 16:40:37 +0900 Subject: [PATCH 33/42] added --level-tuning option to usage --- src/detections/configs.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index f17ea5a0..92b45fd8 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -80,6 +80,7 @@ fn build_app<'a>() -> ArgMatches<'a> { -u --update-rules 'Update to the latest rules in the hayabusa-rules github repository.' -m --min-level=[LEVEL] 'Minimum level for rules. (Default: informational)' -l --live-analysis 'Analyze the local C:\\Windows\\System32\\winevt\\Logs folder (Windows Only. Administrator privileges required.)' + --level-tuning=[LEVEL_TUNING_FILE] 'Fix rule file's level(Default: ./config/level_tuning.txt)' --start-timeline=[STARTTIMELINE] 'Start time of the event logs to load. (Example: '2018/11/28 12:00:00 +09:00')' --end-timeline=[ENDTIMELINE] 'End time of the event logs to load. (Example: '2018/11/28 12:00:00 +09:00')' --rfc-2822 'Output date and time in RFC 2822 format. (Example: Mon, 07 Aug 2006 12:34:56 -0600)' @@ -95,13 +96,13 @@ fn build_app<'a>() -> ArgMatches<'a> { .version("1.1.0") .author("Yamato Security (https://github.com/Yamato-Security/hayabusa)") .setting(AppSettings::VersionlessSubcommands) + .usage(usages) + .args_from_usage(usages) .arg( // TODO: When update claps to 3.x, these can write in usage texts... Arg::from_usage("--level-tuning=[LEVEL_TUNING_FILE] 'Fix rule file's level'") .default_value("./config/level_tuning.txt"), ) - .usage(usages) - .args_from_usage(usages) .get_matches() } From 265d42df30817476fc717fa3f30284e93d8f302c Mon Sep 17 00:00:00 2001 From: DustInDark Date: Sun, 10 Apr 2022 16:57:04 +0900 Subject: [PATCH 34/42] Revert "added --level-tuning option to usage" This reverts commit e6a74090a3ddc85399906aeaca8a4fd5ba98832a. --- src/detections/configs.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index 92b45fd8..f17ea5a0 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -80,7 +80,6 @@ fn build_app<'a>() -> ArgMatches<'a> { -u --update-rules 'Update to the latest rules in the hayabusa-rules github repository.' -m --min-level=[LEVEL] 'Minimum level for rules. (Default: informational)' -l --live-analysis 'Analyze the local C:\\Windows\\System32\\winevt\\Logs folder (Windows Only. Administrator privileges required.)' - --level-tuning=[LEVEL_TUNING_FILE] 'Fix rule file's level(Default: ./config/level_tuning.txt)' --start-timeline=[STARTTIMELINE] 'Start time of the event logs to load. (Example: '2018/11/28 12:00:00 +09:00')' --end-timeline=[ENDTIMELINE] 'End time of the event logs to load. (Example: '2018/11/28 12:00:00 +09:00')' --rfc-2822 'Output date and time in RFC 2822 format. (Example: Mon, 07 Aug 2006 12:34:56 -0600)' @@ -96,13 +95,13 @@ fn build_app<'a>() -> ArgMatches<'a> { .version("1.1.0") .author("Yamato Security (https://github.com/Yamato-Security/hayabusa)") .setting(AppSettings::VersionlessSubcommands) - .usage(usages) - .args_from_usage(usages) .arg( // TODO: When update claps to 3.x, these can write in usage texts... Arg::from_usage("--level-tuning=[LEVEL_TUNING_FILE] 'Fix rule file's level'") .default_value("./config/level_tuning.txt"), ) + .usage(usages) + .args_from_usage(usages) .get_matches() } From 1dc9a11d944ac1e8e6e1502aaf1be55d37690aef Mon Sep 17 00:00:00 2001 From: Tanaka Zakku <71482215+YamatoSecurity@users.noreply.github.com> Date: Mon, 11 Apr 2022 09:15:23 +0900 Subject: [PATCH 35/42] readme update --- README-Japanese.md | 58 ++++++++++++++++--------------------------- README.md | 61 +++++++++++++++++----------------------------- 2 files changed, 44 insertions(+), 75 deletions(-) diff --git a/README-Japanese.md b/README-Japanese.md index 377d7a38..b82d18ef 100644 --- a/README-Japanese.md +++ b/README-Japanese.md @@ -59,10 +59,9 @@ Hayabusaは、日本の[Yamato Security](https://yamatosecurity.connpass.com/) - [Hayabusaルール](#hayabusaルール) - [Hayabusa v.s. 変換されたSigmaルール](#hayabusa-vs-変換されたsigmaルール) - [検知ルールのチューニング](#検知ルールのチューニング) + - [検知レベルのlevelチューニング](#検知レベルのlevelチューニング) - [イベントIDフィルタリング](#イベントidフィルタリング) - - [検知レベルのチューニング](#検知レベルのチューニング) - [その他のWindowsイベントログ解析ツールおよび関連プロジェクト](#その他のwindowsイベントログ解析ツールおよび関連プロジェクト) - - [Sigmaをサポートする他の類似ツールとの比較](#sigmaをサポートする他の類似ツールとの比較) - [コミュニティによるドキュメンテーション](#コミュニティによるドキュメンテーション) - [英語](#英語) - [日本語](#日本語) @@ -128,6 +127,7 @@ CSVのタイムラインをExcelやTimeline Explorerで分析する方法は[こ * イベントログの統計。(どのような種類のイベントがあるのかを把握し、ログ設定のチューニングに有効です。) * 不良ルールやノイズの多いルールを除外するルールチューニング設定が可能です。 * MITRE ATT&CKとのマッピング (CSVの出力ファイルのみ)。 +* ルールレベルのチューニング。 # 予定されている機能 @@ -312,6 +312,7 @@ USAGE: -s --statistics 'イベント ID の統計情報を表示する。' -q --quiet 'Quietモード。起動バナーを表示しない。' -Q --quiet-errors 'Quiet errorsモード。エラーログを保存しない。' + --level-tuning 'ルールlevelのチューニング [default: ./config/level_tuning.txt]' --contributors 'コントリビュータの一覧表示。' ``` @@ -468,9 +469,9 @@ Hayabusaルールのディレクトリ構造は、3つのディレクトリに ルールはさらにログタイプ（例：Security、Systemなど）によってディレクトリに分けられ、次の形式で名前が付けられます。 -* アラート形式: `<イベントID>__<詳細>.yml` -* アラート例: `1102_IndicatorRemovalOnHost-ClearWindowsEventLogs_SecurityLogCleared.yml` -* イベント形式: `<イベントID>_<詳細>.yml` +* アラート形式: `<イベントID>_<イベントの説明>_<リスクの説明>.yml` +* アラート例: `1102_SecurityLogCleared_PossibleAntiForensics.yml` +* イベント形式: `<イベントID>_<イベントの説明>.yml` * イベント例: `4776_NTLM-LogonToLocalAccount.yml` 現在のルールをご確認いただき、新規作成時のテンプレートとして、また検知ロジックの確認用としてご利用ください。 @@ -487,8 +488,7 @@ Sigmaルールは、最初にHayabusaルール形式に変換する必要があ 1. [Rust正規表現クレート](https://docs.rs/regex/1.5.4/regex/)では機能しない正規表現を使用するルール。 2. [Sigmaルール仕様](https://github.com/SigmaHQ/Sigma/wiki/Specification)の`count`以外の集計式。 - -> 注意：この制限はSigmaルールの変換ツールにあり、Hayabusa自身にあるわけではありません。 +3. `|near`を使用するルール。 ## 検知ルールのチューニング @@ -498,6 +498,20 @@ Sigmaルールは、最初にHayabusaルール形式に変換する必要があ ルールIDを `rules/config/noisy_rules.txt`に追加して、デフォルトでルールを無視することもできますが、`-n`または `--enable-noisy-rules`オプションを指定してルールを使用することもできます。 +## 検知レベルのlevelチューニング + +Hayabusaルール、Sigmaルールはそれぞれの作者が検知した際のリスクレベルを決めています。 +ユーザが独自のリスクレベルに設定するには`./config/level_tuning.txt`に変換情報を書き、`hayabusa.exe --level-tuning`を実行することでルールファイルが書き換えられます。 +ルールファイルが直接書き換えられることに注意して使用してください。 + +`./config/level_tuning.txt`の例: +``` +id,new_level +00000000-0000-0000-0000-000000000000,informational # sample level tuning line +``` + +ルールディレクトリ内で`id`が`00000000-0000-0000-0000-000000000000`のルールのリスクレベルが`informational`に書き換えられます。 + ## イベントIDフィルタリング `config/target_eventids.txt`にイベントID番号を追加することで、イベントIDでフィルタリングすることができます。 @@ -507,19 +521,6 @@ Sigmaルールは、最初にHayabusaルール形式に変換する必要があ 最高のパフォーマンスを得たい場合はこのリストを使用してください。ただし、検出漏れの可能性が若干あることにご注意ください。 -## 検知レベルのチューニング - -Hayabusaルール、Sigmaルールはそれぞれの作者が検知した際の脅威レベルを決めています。 -ユーザが独自の脅威レベルに設定するには `./config/level_tuning.txt` に変換情報を書き、`hayabusa --level-tuning` を実行することでルールファイルが書き換えられます。 -ルールファイルが直接書き換えられることに注意して使用してください。 - -`./config/level_tuning.txt` の例 -``` -id,new_level -00000000-0000-0000-0000-000000000000,informational # sample level tuning line -``` -ルールディレクトリ内の `00000000-0000-0000-0000-000000000000` に該当するルールの脅威レベルが `informational` に書き換えられます。 - # その他のWindowsイベントログ解析ツールおよび関連プロジェクト 「すべてを統治する1つのツール」というものはなく、それぞれにメリットがあるため、これらの他の優れたツールやプロジェクトをチェックして、どれが気に入ったかを確認することをお勧めします。 @@ -546,23 +547,6 @@ id,new_level * [WELA (Windows Event Log Analyzer)](https://github.com/Yamato-Security/WELA/) - [Yamato Security](https://github.com/Yamato-Security/)によるWindowsイベントログ解析のマルチツール。 * [Zircolite](https://github.com/wagga40/Zircolite) - Pythonで書かれたSigmaベースの攻撃検知ツール。 -## Sigmaをサポートする他の類似ツールとの比較 - -対象となるサンプルデータ、コマンドラインオプション、ルールのチューニング等によって結果が異なるため、完全な比較はできませんが、ご了承ください。 -我々のテストでは、Hayabusaはすべてのツールの中で最も多くのSigmaルールをサポートしながらも、非常に高速な速度を維持し、大量のメモリを必要としないことが分かっています。 - -以下のベンチマークは、2021/12/23に [sample-evtx repository](https://github.com/Yamato-Security/Hayabusa-sample-evtx) から約500個のevtxファイル（130MB）を基に、Lenovo P51で計測したものです。Hayabusa 1.0.0を使いました。 - -| | 経過時間 | メモリ使用量 | 利用可能のSigmaルール数 | -| :-------: | :---------: | :--------------------------------------------: | :---------------------: | -| Chainsaw | 7.5 seconds | 70 MB | 170 | -| Hayabusa | 7.8 seconds | 340 MB | 267 | -| Zircolite | 34 seconds | 380 MB (通常、ログファイルの3倍のサイズが必要) | 237 | - -* Hayabusaルールも有効にすると、約300のユニークなアラートとイベントを検知します。 -* 合計7.5GBの多数のイベントログファイルでテストしたところ、7分以内に終了し、1GB以上のメモリを使用しませんでした。消費されるメモリ量は、ターゲットのevtxファイルのサイズではなく、結果のサイズによって増えます。 -* [Timeline Explorer](https://ericzimmerman.github.io/#!index.md)などのツールで解析するために、結果を1つのCSVタイムラインにまとめる唯一のツールです。 - # コミュニティによるドキュメンテーション ## 英語 diff --git a/README.md b/README.md index 4db65105..467cf05e 100644 --- a/README.md +++ b/README.md @@ -59,10 +59,9 @@ Hayabusa is a **Windows event log fast forensics timeline generator** and **thre - [Hayabusa Rules](#hayabusa-rules) - [Hayabusa v.s. Converted Sigma Rules](#hayabusa-vs-converted-sigma-rules) - [Detection Rule Tuning](#detection-rule-tuning) - - [Event ID Filtering](#event-id-filtering) - [Detection Level Tuning](#detection-level-tuning) + - [Event ID Filtering](#event-id-filtering) - [Other Windows Event Log Analyzers and Related Projects](#other-windows-event-log-analyzers-and-related-projects) - - [Comparison To Other Similar Tools](#comparison-to-other-similar-tools) - [Community Documentation](#community-documentation) - [English](#english) - [Japanese](#japanese) @@ -126,6 +125,7 @@ You can learn how to analyze CSV timelines in Excel and Timeline Explorer [here] * Event log statistics. (Useful for getting a picture of what types of events there are and for tuning your log settings.) * Rule tuning configuration by excluding unneeded or noisy rules. * MITRE ATT&CK mapping of tactics (only in saved CSV files). +* Rule level tuning. # Planned Features @@ -306,6 +306,7 @@ USAGE: -s --statistics 'Prints statistics of event IDs.' -q --quiet 'Quiet mode. Do not display the launch banner.' -Q --quiet-errors 'Quiet errors mode. Do not save error logs.' + --level-tuning 'Tune the rule level [default: ./config/level_tuning.txt]' --contributors 'Prints the list of contributors.' ``` @@ -460,9 +461,9 @@ The hayabusa rule directory structure is separated into 3 directories: Rules are further seperated into directories by log type (Example: Security, System, etc...) and are named in the following format: -* Alert format: `__.yml` -* Alert example: `1102_IndicatorRemovalOnHost-ClearWindowsEventLogs_SecurityLogCleared.yml` -* Event format: `_.yml` +* Alert format: `__.yml` +* Alert example: `1102_SecurityLogCleared_PossibleAntiForensics.yml` +* Event format: `_.yml` * Event example: `4776_NTLM-LogonToLocalAccount.yml` Please check out the current rules to use as a template in creating new ones or for checking the detection logic. @@ -479,8 +480,7 @@ Sigma rules need to first be converted to hayabusa rule format explained [here]( 1. Rules that use regular expressions that do not work with the [Rust regex crate](https://docs.rs/regex/1.5.4/regex/) 2. Aggregation expressions besides `count` in the [sigma rule specification](https://github.com/SigmaHQ/sigma/wiki/Specification). - -> Note: the limitation is in the sigma rule converter and not in hayabusa itself. +3. Rules that use `|near`. ## Detection Rule Tuning @@ -490,6 +490,22 @@ You can add a rule ID (Example: `4fe151c2-ecf9-4fae-95ae-b88ec9c2fca6`) to `rule You can also add a rule ID to `rules/config/noisy_rules.txt` in order to ignore the rule by default but still be able to use the rule with the `-n` or `--enable-noisy-rules` option. +## Detection Level Tuning + +Hayabusa and Sigma rule authors will determine the risk level of the alert when writing their rules. +However, the actual risk level will differ between environments. +You can tune the risk level of the rules by adding them to `./config/level_tuning.txt` and executing `hayabusa.exe --level-tuning` which will update the `level` line in the rule file. +Please note that the rule file will be updated directly. + +`./config/level_tuning.txt` sample line: + +``` +id,new_level +00000000-0000-0000-0000-000000000000,informational # sample level tuning line +``` + +In this case, the risk level of the rule with an `id` of `00000000-0000-0000-0000-000000000000` in the rules directory will have its `level` rewritten to `informational`. + ## Event ID Filtering You can filter on event IDs by placing event ID numbers in `config/target_eventids.txt`. @@ -499,20 +515,6 @@ We have provided a sample ID filter list at [`config/target_eventids_sample.txt` Please use this list if you want the best performance but be aware that there is a slight possibility for missing events (false negatives). -## Detection Level Tuning - -The Hayabusa rule and Sigma rule determine the threat level when each author detects it. -To set the user to their own threat level, write the conversion information in `./config/level_tuning.txt` and execute` hayabusa --level-tuning` to rewrite the rule file. -Please note that the rule file will be rewritten directly. - -`./config/level_tuning.txt` Sample -``` -id,new_level -00000000-0000-0000-0000-000000000000,informational # sample level tuning line -``` - -The threat level of the rule corresponding to `00000000-0000-0000-0000-000000000000` in the rules directory is rewritten to` informational`. - # Other Windows Event Log Analyzers and Related Projects There is no "one tool to rule them all" and we have found that each has its own merits so we recommend checking out these other great tools and projects and seeing which ones you like. @@ -539,23 +541,6 @@ There is no "one tool to rule them all" and we have found that each has its own * [WELA (Windows Event Log Analyzer)](https://github.com/Yamato-Security/WELA) - The swiff-army knife for Windows event logs by [Yamato Security](https://github.com/Yamato-Security/) * [Zircolite](https://github.com/wagga40/Zircolite) - Sigma-based attack detection tool written in Python. -## Comparison To Other Similar Tools - -Please understand that it is not possible to do a perfect comparison as results will differ based on the target sample data, command-line options, rule tuning, etc... -In our tests, we have found hayabusa to support the largest number of sigma rules compared to other similar tools while still maintaining very fast speeds and does not require a great amount of memory. - -The following benchmarks were taken on a Lenovo P51 based on approximately 500 evtx files (130MB) from our [sample-evtx repository](https://github.com/Yamato-Security/hayabusa-sample-evtx) at 2021/12/23 with hayabusa version 1.0.0. - -| | Elapsed Time | Memory Usage | Unique Sigma Rules With Detections | -| :-------: | :----------: | :----------------------------------------------------------: | :--------------------------------: | -| Chainsaw | 7.5 seconds | 75 MB | 170 | -| Hayabusa | 7.8 seconds | 340 MB (memory usage depends on the amount of alerts) | 267 | -| Zircolite | 34 seconds | 380 MB (normally requires 3 times the size of the log files) | 237 | - -* With hayabusa rules enabled, it will detect around 300 unique alerts and events. -* When tested on many event logs files totaling 7.5 GB, it finished in under 7 minutes and used around 1 GB of memory. The amount of memory consumed is based on the size of the results, not on the size of the target evtx files. -* It is the only tool that provides a consolidated single CSV timeline to analysis in tools like [Timeline Explorer](https://ericzimmerman.github.io/#!index.md). - # Community Documentation ## English From b67aaebf8abdca92104898cdfab4af041f243d86 Mon Sep 17 00:00:00 2001 From: Yamato Security <71482215+YamatoSecurity@users.noreply.github.com> Date: Mon, 11 Apr 2022 17:29:38 +0900 Subject: [PATCH 36/42] Update README-Japanese.md --- README-Japanese.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README-Japanese.md b/README-Japanese.md index e8fe8317..832d4e36 100644 --- a/README-Japanese.md +++ b/README-Japanese.md @@ -62,6 +62,8 @@ Hayabusaは、日本の[Yamato Security](https://yamatosecurity.connpass.com/) - [検知レベルのlevelチューニング](#検知レベルのlevelチューニング) - [イベントIDフィルタリング](#イベントidフィルタリング) - [その他のWindowsイベントログ解析ツールおよび関連プロジェクト](#その他のwindowsイベントログ解析ツールおよび関連プロジェクト) +- [Windowsイベントログ設定のススメ](#Windowsイベントログ設定のススメ) +- [Sysmon関係のプロジェクト](#Sysmon関係のプロジェクト) - [コミュニティによるドキュメンテーション](#コミュニティによるドキュメンテーション) - [英語](#英語) - [日本語](#日本語) From 0db51b1f87ca8432b6395d01fe8adc84173b16a4 Mon Sep 17 00:00:00 2001 From: Tanaka Zakku <71482215+YamatoSecurity@users.noreply.github.com> Date: Mon, 11 Apr 2022 18:05:56 +0900 Subject: [PATCH 37/42] readme, version, cargo update --- Cargo.lock | 121 ++++++++++++++++++++------------------ README-Japanese.md | 9 ++- README.md | 7 ++- src/detections/configs.rs | 4 +- 4 files changed, 79 insertions(+), 62 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a02ea5f9..32666b64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -108,6 +108,12 @@ dependencies = [ "byteorder", ] +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + [[package]] name = "bitflags" version = "1.3.2" @@ -193,7 +199,7 @@ checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" dependencies = [ "camino", "cargo-platform", - "semver 1.0.6", + "semver 1.0.7", "serde", "serde_json", ] @@ -580,9 +586,9 @@ checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" [[package]] name = "encoding_rs" -version = "0.8.30" +version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df" +checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" dependencies = [ "cfg-if 1.0.0", ] @@ -765,9 +771,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" dependencies = [ "cfg-if 1.0.0", "libc", @@ -838,7 +844,7 @@ dependencies = [ name = "hayabusa" version = "1.2.0" dependencies = [ - "base64", + "base64 0.13.0", "chrono", "clap", "colored", @@ -1004,9 +1010,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" +checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" dependencies = [ "autocfg 1.1.0", "hashbrown 0.11.2", @@ -1108,9 +1114,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.121" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f" +checksum = "ec647867e2bf0772e28c8bcde4f0d19a9216916e890543b5a03ed8ef27b8f259" [[package]] name = "libgit2-sys" @@ -1169,10 +1175,11 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" dependencies = [ + "autocfg 1.1.0", "scopeguard", ] @@ -1303,9 +1310,9 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" dependencies = [ "lazy_static", "libc", @@ -1454,8 +1461,8 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" dependencies = [ - "lock_api 0.4.6", - "parking_lot_core 0.9.1", + "lock_api 0.4.7", + "parking_lot_core 0.9.2", ] [[package]] @@ -1475,13 +1482,13 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" +checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.2.12", + "redox_syscall 0.2.13", "smallvec 1.8.0", "windows-sys", ] @@ -1518,9 +1525,9 @@ checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" [[package]] name = "pkg-config" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" [[package]] name = "proc-macro-hack" @@ -1530,9 +1537,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" +checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" dependencies = [ "unicode-xid", ] @@ -1576,9 +1583,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "632d02bff7f874a36f33ea8bb416cd484b90cc66c1194b1a1110d067a7013f58" +checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" dependencies = [ "proc-macro2", ] @@ -1731,9 +1738,9 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_syscall" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae183fc1b06c149f0c1793e1eb447c8b04bfe46d48e9e48bfb8d2d7ed64ecf0" +checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ "bitflags", ] @@ -1776,7 +1783,7 @@ version = "0.9.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f88643aea3c1343c804950d7bf983bd2067f5ab59db6d613a08e05572f2714ab" dependencies = [ - "base64", + "base64 0.10.1", "bytes 0.4.12", "cookie", "cookie_store", @@ -1904,9 +1911,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a3381e03edd24287172047536f20cabde766e2cd3e65e6b00fb3af51c4f38d" +checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4" dependencies = [ "serde", ] @@ -2012,9 +2019,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" +checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" [[package]] name = "slack-hook" @@ -2138,9 +2145,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "syn" -version = "1.0.89" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea297be220d52398dcc07ce15a209fce436d361735ac1db700cab3b6cdfb9f54" +checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d" dependencies = [ "proc-macro2", "quote", @@ -2168,7 +2175,7 @@ dependencies = [ "cfg-if 1.0.0", "fastrand", "libc", - "redox_syscall 0.2.12", + "redox_syscall 0.2.13", "remove_dir_all", "winapi 0.3.9", ] @@ -2602,9 +2609,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" +checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -2612,9 +2619,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" +checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" dependencies = [ "bumpalo", "lazy_static", @@ -2627,9 +2634,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" +checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2637,9 +2644,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" +checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" dependencies = [ "proc-macro2", "quote", @@ -2650,9 +2657,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" +checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" [[package]] name = "winapi" @@ -2699,9 +2706,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-sys" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" +checksum = "5acdd78cb4ba54c0045ac14f62d8f94a03d10047904ae2a40afa1e99d8f70825" dependencies = [ "windows_aarch64_msvc", "windows_i686_gnu", @@ -2712,33 +2719,33 @@ dependencies = [ [[package]] name = "windows_aarch64_msvc" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" +checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" [[package]] name = "windows_i686_gnu" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" +checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" [[package]] name = "windows_i686_msvc" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" +checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" [[package]] name = "windows_x86_64_gnu" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" +checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" [[package]] name = "windows_x86_64_msvc" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" +checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" [[package]] name = "winreg" diff --git a/README-Japanese.md b/README-Japanese.md index 832d4e36..1260aa7c 100644 --- a/README-Japanese.md +++ b/README-Japanese.md @@ -62,14 +62,15 @@ Hayabusaは、日本の[Yamato Security](https://yamatosecurity.connpass.com/) - [検知レベルのlevelチューニング](#検知レベルのlevelチューニング) - [イベントIDフィルタリング](#イベントidフィルタリング) - [その他のWindowsイベントログ解析ツールおよび関連プロジェクト](#その他のwindowsイベントログ解析ツールおよび関連プロジェクト) -- [Windowsイベントログ設定のススメ](#Windowsイベントログ設定のススメ) -- [Sysmon関係のプロジェクト](#Sysmon関係のプロジェクト) +- [Windowsイベントログ設定のススメ](#windowsイベントログ設定のススメ) +- [Sysmon関係のプロジェクト](#sysmon関係のプロジェクト) - [コミュニティによるドキュメンテーション](#コミュニティによるドキュメンテーション) - [英語](#英語) - [日本語](#日本語) - [貢献](#貢献) - [バグの報告](#バグの報告) - [ライセンス](#ライセンス) +- [Twitter](#twitter) ## 主な目的 @@ -609,3 +610,7 @@ Windows機での悪性な活動を検知する為には、デフォルトのロ # ライセンス Hayabusaは[GPLv3](https://www.gnu.org/licenses/gpl-3.0.en.html)で公開され、すべてのルールは[Detection Rule License (DRL) 1.1](https://github.com/SigmaHQ/sigma/blob/master/LICENSE.Detection.Rules.md)で公開されています。 + +# Twitter + +[https://twitter.com/SecurityYamato](@SecurityYamato)でHayabusa、ルール更新、その他の大和セキュリティツール等々について情報を提供しています。 \ No newline at end of file diff --git a/README.md b/README.md index 29d6657d..60379943 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ Hayabusa is a **Windows event log fast forensics timeline generator** and **thre - [Contribution](#contribution) - [Bug Submission](#bug-submission) - [License](#license) +- [Twitter](#twitter) ## Main Goals @@ -601,4 +602,8 @@ This project is currently actively maintained and we are happy to fix any bugs r # License -Hayabusa is released under [GPLv3](https://www.gnu.org/licenses/gpl-3.0.en.html) and all rules are released under the [Detection Rule License (DRL) 1.1](https://github.com/SigmaHQ/sigma/blob/master/LICENSE.Detection.Rules.md). \ No newline at end of file +Hayabusa is released under [GPLv3](https://www.gnu.org/licenses/gpl-3.0.en.html) and all rules are released under the [Detection Rule License (DRL) 1.1](https://github.com/SigmaHQ/sigma/blob/master/LICENSE.Detection.Rules.md). + +# Twitter + +You can recieve the latest news about Hayaabusa, rule updates, other Yamato Security tools, etc... by following us on Twitter at [https://twitter.com/SecurityYamato](@SecurityYamato). \ No newline at end of file diff --git a/src/detections/configs.rs b/src/detections/configs.rs index f214deca..9497376f 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -96,8 +96,8 @@ fn build_app<'a>() -> ArgMatches<'a> { --contributors 'Prints the list of contributors.'"; App::new(&program) .about("Hayabusa: Aiming to be the world's greatest Windows event log analysis tool!") - .version("1.1.0") - .author("Yamato Security (https://github.com/Yamato-Security/hayabusa)") + .version("1.2.0") + .author("Yamato Security (https://github.com/Yamato-Security/hayabusa) @SecurityYamato") .setting(AppSettings::VersionlessSubcommands) .arg( // TODO: When update claps to 3.x, these can write in usage texts... From 30939f8ab3c8dccd45a6cfc7208621abcd2bbe43 Mon Sep 17 00:00:00 2001 From: Tanaka Zakku <71482215+YamatoSecurity@users.noreply.github.com> Date: Mon, 11 Apr 2022 18:17:52 +0900 Subject: [PATCH 38/42] typo fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 60379943..3c80bbb8 100644 --- a/README.md +++ b/README.md @@ -606,4 +606,4 @@ Hayabusa is released under [GPLv3](https://www.gnu.org/licenses/gpl-3.0.en.html) # Twitter -You can recieve the latest news about Hayaabusa, rule updates, other Yamato Security tools, etc... by following us on Twitter at [https://twitter.com/SecurityYamato](@SecurityYamato). \ No newline at end of file +You can recieve the latest news about Hayabusa, rule updates, other Yamato Security tools, etc... by following us on Twitter at [https://twitter.com/SecurityYamato](@SecurityYamato). \ No newline at end of file From 4af7958ff64f26128847c1e238df062271326d6e Mon Sep 17 00:00:00 2001 From: Tanaka Zakku <71482215+YamatoSecurity@users.noreply.github.com> Date: Mon, 11 Apr 2022 20:16:39 +0900 Subject: [PATCH 39/42] typo fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3c80bbb8..de69a540 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,7 @@ You may receive warning from anti-virus or EDR when trying to run hayabusa. Thes ## Windows -In Command Prompt or Windows Terminal, just run 32-bit or 64-bit Windoows binary from the hayabusa root directory. +In Command Prompt or Windows Terminal, just run the 32-bit or 64-bit Windows binary from the hayabusa root directory. Example: `hayabusa-1.2.0-windows-x64.exe` ## Linux From a1812e013b549ff1c4f2027230e3f7e0231c6e50 Mon Sep 17 00:00:00 2001 From: itiB Date: Tue, 12 Apr 2022 00:20:25 +0900 Subject: [PATCH 40/42] rm: duplicated test & fix test name --- src/options/level_tuning.rs | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/src/options/level_tuning.rs b/src/options/level_tuning.rs index 869e9b15..9ca51d18 100644 --- a/src/options/level_tuning.rs +++ b/src/options/level_tuning.rs @@ -98,8 +98,8 @@ impl LevelTuning { #[cfg(test)] mod tests { - use crate::{filter::RuleExclude, yaml}; - use hashbrown::HashSet; + // use crate::{filter::RuleExclude, yaml}; + // use hashbrown::HashSet; use super::*; @@ -134,7 +134,7 @@ mod tests { } #[test] - fn test_detect_mutiple_regex_and() { + fn test_level_tuning_update_rule_files() { let level_tuning_config_path = "./test_files/config/level_tuning.txt"; let rule_str = r#" id: 12345678-1234-1234-1234-123456789012 @@ -155,24 +155,6 @@ mod tests { let res = LevelTuning::run(level_tuning_config_path, path); assert_eq!(res, Ok(())); - let mut parser = yaml::ParseYaml::new(); - parser - .read_dir( - "test_files/rules", - "informational", - &RuleExclude { - no_use_rule: HashSet::new(), - }, - ) - .ok(); - for (_filepath, yaml) in parser.files { - if yaml["id"].as_str().unwrap_or(&String::default()) - == "12345678-1234-1234-1234-123456789012" - { - assert_eq!("high", yaml["level"].as_str().unwrap()); - } - } - assert_eq!(fs::read_to_string(path).unwrap(), expected_rule); fs::remove_file(path).unwrap(); } From 9fa60dd26dd0729113d0c0539d5e99469b637c12 Mon Sep 17 00:00:00 2001 From: itiB Date: Tue, 12 Apr 2022 00:27:02 +0900 Subject: [PATCH 41/42] Add: show logo, and some infos --- src/main.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index 868bea31..c6d77e11 100644 --- a/src/main.rs +++ b/src/main.rs @@ -84,6 +84,7 @@ impl App { // Show usage when no arguments. if std::env::args().len() == 1 { + self.output_logo(); println!("{}", configs::CONFIG.read().unwrap().args.usage()); println!(); return; @@ -251,7 +252,7 @@ impl App { } else { AlertMessage::alert( &mut BufWriter::new(std::io::stderr().lock()), - "Need rule_levels.txt file to use --level-tuning option", + "Need rule_levels.txt file to use --level-tuning option [default: ./config/level_tuning.txt]", ) .ok(); } From c09f9d4f5fd9d9406f9c31ecef19c4f3cc1f9f67 Mon Sep 17 00:00:00 2001 From: Tanaka Zakku <71482215+YamatoSecurity@users.noreply.github.com> Date: Tue, 12 Apr 2022 08:38:57 +0900 Subject: [PATCH 42/42] small english fix --- src/detections/configs.rs | 2 +- src/main.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index 9497376f..f336aa1e 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -101,7 +101,7 @@ fn build_app<'a>() -> ArgMatches<'a> { .setting(AppSettings::VersionlessSubcommands) .arg( // TODO: When update claps to 3.x, these can write in usage texts... - Arg::from_usage("--level-tuning=[LEVEL_TUNING_FILE] 'Fix rule file's level'") + Arg::from_usage("--level-tuning=[LEVEL_TUNING_FILE] 'Adjust rule level.'") .default_value("./config/level_tuning.txt"), ) .usage(usages) diff --git a/src/main.rs b/src/main.rs index c6d77e11..dc3fdf9d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -85,6 +85,7 @@ impl App { // Show usage when no arguments. if std::env::args().len() == 1 { self.output_logo(); + println!(); println!("{}", configs::CONFIG.read().unwrap().args.usage()); println!(); return; @@ -103,7 +104,7 @@ impl App { if !self.is_matched_architecture_and_binary() { AlertMessage::alert( &mut BufWriter::new(std::io::stderr().lock()), - "The hayabusa version you ran does not match your PC architecture.\n Please use the correct architecture. (Binary ending in -x64.exe for 64-bit and -x86.exe for 32-bit.)", + "The hayabusa version you ran does not match your PC architecture.\nPlease use the correct architecture. (Binary ending in -x64.exe for 64-bit and -x86.exe for 32-bit.)", ) .ok(); println!();