CSEC_PUBLIC/hayabusa
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Pivot Keyword List機能の追加 (#412)

Browse Source 
* add get_pivot_keyword() func

* change function name and call it's function

* [WIP] support config file

* compilete output

* cargo fmt

* [WIP] add test

* add test

* support -o option in pivot

* add pivot mod

* fix miss

* pass test in pivot.rs

* add comment

* pass all test

* add fast return

* fix output

* add test config file

* review

* rebase

* cargo fmt

* test pass

* fix clippy in my commit

* cargo fmt

* little refactor

* change file input logic and config format

* [WIP] change output

* [wip] change deta structure

* change output & change data structure

* pass test

* add config

* cargo fmt & clippy & rebase

* fix cllipy

* delete /rules/ in .gitignore

* clean comment

* clean

* clean

* fix rebase miss

* fix rebase miss

* fix clippy

* file name output on -o to stdout

* add pivot_keywords.txt to ./config

* updated english

* Documentation update

* cargo fmt and clean

* updated translate japanese

* readme update

* readme update

Co-authored-by: DustInDark <nextsasasa@gmail.com>
Co-authored-by: Tanaka Zakku <71482215+YamatoSecurity@users.noreply.github.com>
This commit is contained in:
kazuminn
2022-04-05 21:17:23 +09:00
committed by GitHub
parent 545119bdfe
commit c8efa95447
13 changed files with 511 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
src/detections/configs.rs
View File

@@ -1,3 +1,5 @@
use crate::detections::pivot::PivotKeyword;
use crate::detections::pivot::PIVOT_KEYWORD;
use crate::detections::print::AlertMessage;
use crate::detections::utils;
use chrono::{DateTime, Utc};
@@ -67,7 +69,7 @@ fn build_app<'a>() -> ArgMatches<'a> {
let usages = "-d --directory=[DIRECTORY] 'Directory of multiple .evtx files.'
-f --filepath=[FILEPATH] 'File path to one .evtx file.'
-r --rules=[RULEFILE/RULEDIRECTORY] 'Rule file or directory. (Default: ./rules)'
-r --rules=[RULEDIRECTORY/RULEFILE] 'Rule file or directory (default: ./rules)'
-c --color 'Output with color. (Terminal needs to support True Color.)'
-C --config=[RULECONFIGDIRECTORY] 'Rule config folder. (Default: ./rules/config)'
-o --output=[CSV_TIMELINE] 'Save the timeline in CSV format. (Example: results.csv)'
@@ -86,6 +88,7 @@ fn build_app<'a>() -> ArgMatches<'a> {
-s --statistics 'Prints statistics of event IDs.'
-q --quiet 'Quiet mode. Do not display the launch banner.'
-Q --quiet-errors 'Quiet errors mode. Do not save error logs.'
-p --pivot-keywords-list 'Create a list of pivot keywords.'
--contributors 'Prints the list of contributors.'";
App::new(&program)
.about("Hayabusa: Aiming to be the world's greatest Windows event log analysis tool!")
@@ -268,6 +271,7 @@ impl Default for EventKeyAliasConfig {
fn load_eventkey_alias(path: &str) -> EventKeyAliasConfig {
let mut config = EventKeyAliasConfig::new();
// eventkey_aliasが読み込めなかったらエラーで終了とする。
let read_result = utils::read_csv(path);
if read_result.is_err() {
AlertMessage::alert(
@@ -277,7 +281,7 @@ fn load_eventkey_alias(path: &str) -> EventKeyAliasConfig {
.ok();
return config;
}
// eventkey_aliasが読み込めなかったらエラーで終了とする。
read_result.unwrap().into_iter().for_each(|line| {
if line.len() != 2 {
return;
@@ -302,6 +306,40 @@ fn load_eventkey_alias(path: &str) -> EventKeyAliasConfig {
config
}
///設定ファイルを読み込み、keyとfieldsのマップをPIVOT_KEYWORD大域変数にロードする。
pub fn load_pivot_keywords(path: &str) {
let read_result = utils::read_txt(path);
if read_result.is_err() {
AlertMessage::alert(
&mut BufWriter::new(std::io::stderr().lock()),
read_result.as_ref().unwrap_err(),
)
.ok();
}
read_result.unwrap().into_iter().for_each(|line| {
let map: Vec<&str> = line.split('.').collect();
if map.len() != 2 {
return;
}
//存在しなければ、keyを作成
PIVOT_KEYWORD
.write()
.unwrap()
.entry(map[0].to_string())
.or_insert(PivotKeyword::new());
PIVOT_KEYWORD
.write()
.unwrap()
.get_mut(&map[0].to_string())
.unwrap()
.fields
.insert(map[1].to_string());
});
}
#[derive(Debug, Clone)]
pub struct EventInfo {
pub evttitle: String,

8
src/detections/detection.rs
View File

@@ -1,10 +1,12 @@
extern crate csv;
use crate::detections::configs;
use crate::detections::pivot::insert_pivot_keyword;
use crate::detections::print::AlertMessage;
use crate::detections::print::DetectInfo;
use crate::detections::print::ERROR_LOG_STACK;
use crate::detections::print::MESSAGES;
use crate::detections::print::PIVOT_KEYWORD_LIST_FLAG;
use crate::detections::print::QUIET_ERRORS_FLAG;
use crate::detections::print::STATISTICS_FLAG;
use crate::detections::rule;
@@ -177,6 +179,12 @@ impl Detection {
if !result {
continue;
}
if *PIVOT_KEYWORD_LIST_FLAG {
insert_pivot_keyword(&record_info.record);
continue;
}
// aggregation conditionが存在しない場合はそのまま出力対応を行う
if !agg_condition {
Detection::insert_message(&rule, record_info);

1
src/detections/mod.rs
View File

@@ -1,5 +1,6 @@
pub mod configs;
pub mod detection;
pub mod pivot;
pub mod print;
pub mod rule;
pub mod utils;

270
src/detections/pivot.rs Normal file
View File

@@ -0,0 +1,270 @@
use hashbrown::HashMap;
use hashbrown::HashSet;
use lazy_static::lazy_static;
use serde_json::Value;
use std::sync::RwLock;
use crate::detections::configs;
use crate::detections::utils::get_serde_number_to_string;
#[derive(Debug)]
pub struct PivotKeyword {
pub keywords: HashSet<String>,
pub fields: HashSet<String>,
}
lazy_static! {
pub static ref PIVOT_KEYWORD: RwLock<HashMap<String, PivotKeyword>> =
RwLock::new(HashMap::new());
}
impl Default for PivotKeyword {
fn default() -> Self {
Self::new()
}
}
impl PivotKeyword {
pub fn new() -> PivotKeyword {
PivotKeyword {
keywords: HashSet::new(),
fields: HashSet::new(),
}
}
}
///levelがlowより大きいレコードの場合、keywordがrecord内にみつかれば、
///それをPIVOT_KEYWORD.keywordsに入れる。
pub fn insert_pivot_keyword(event_record: &Value) {
//levelがlow異常なら続ける
let mut is_exist_event_key = false;
let mut tmp_event_record: &Value = event_record;
for s in ["Event", "System", "Level"] {
if let Some(record) = tmp_event_record.get(s) {
is_exist_event_key = true;
tmp_event_record = record;
}
}
if is_exist_event_key {
let hash_value = get_serde_number_to_string(tmp_event_record);
if hash_value.is_some() && hash_value.as_ref().unwrap() == "infomational"
|| hash_value.as_ref().unwrap() == "undefined"
|| hash_value.as_ref().unwrap() == "-"
{
return;
}
} else {
return;
}
for (_, pivot) in PIVOT_KEYWORD.write().unwrap().iter_mut() {
for field in &pivot.fields {
if let Some(array_str) = configs::EVENTKEY_ALIAS.get_event_key(&String::from(field)) {
let split: Vec<&str> = array_str.split('.').collect();
let mut is_exist_event_key = false;
let mut tmp_event_record: &Value = event_record;
for s in split {
if let Some(record) = tmp_event_record.get(s) {
is_exist_event_key = true;
tmp_event_record = record;
}
}
if is_exist_event_key {
let hash_value = get_serde_number_to_string(tmp_event_record);
if let Some(value) = hash_value {
if value == "-" || value == "127.0.0.1" || value == "::1" {
continue;
}
pivot.keywords.insert(value);
};
}
}
}
}
}
#[cfg(test)]
mod tests {
use crate::detections::configs::load_pivot_keywords;
use crate::detections::pivot::insert_pivot_keyword;
use crate::detections::pivot::PIVOT_KEYWORD;
use serde_json;
//PIVOT_KEYWORDはグローバルなので、他の関数の影響も考慮する必要がある。
#[test]
fn insert_pivot_keyword_local_ip4() {
load_pivot_keywords("test_files/config/pivot_keywords.txt");
let record_json_str = r#"
{
"Event": {
"System": {
"Level": "high"
},
"EventData": {
"IpAddress": "127.0.0.1"
}
}
}"#;
insert_pivot_keyword(&serde_json::from_str(record_json_str).unwrap());
assert!(!PIVOT_KEYWORD
.write()
.unwrap()
.get_mut("Ip Addresses")
.unwrap()
.keywords
.contains("127.0.0.1"));
}
#[test]
fn insert_pivot_keyword_ip4() {
load_pivot_keywords("test_files/config/pivot_keywords.txt");
let record_json_str = r#"
{
"Event": {
"System": {
"Level": "high"
},
"EventData": {
"IpAddress": "10.0.0.1"
}
}
}"#;
insert_pivot_keyword(&serde_json::from_str(record_json_str).unwrap());
assert!(PIVOT_KEYWORD
.write()
.unwrap()
.get_mut("Ip Addresses")
.unwrap()
.keywords
.contains("10.0.0.1"));
}
#[test]
fn insert_pivot_keyword_ip_empty() {
load_pivot_keywords("test_files/config/pivot_keywords.txt");
let record_json_str = r#"
{
"Event": {
"System": {
"Level": "high"
},
"EventData": {
"IpAddress": "-"
}
}
}"#;
insert_pivot_keyword(&serde_json::from_str(record_json_str).unwrap());
assert!(!PIVOT_KEYWORD
.write()
.unwrap()
.get_mut("Ip Addresses")
.unwrap()
.keywords
.contains("-"));
}
#[test]
fn insert_pivot_keyword_local_ip6() {
load_pivot_keywords("test_files/config/pivot_keywords.txt");
let record_json_str = r#"
{
"Event": {
"System": {
"Level": "high"
},
"EventData": {
"IpAddress": "::1"
}
}
}"#;
insert_pivot_keyword(&serde_json::from_str(record_json_str).unwrap());
assert!(!PIVOT_KEYWORD
.write()
.unwrap()
.get_mut("Ip Addresses")
.unwrap()
.keywords
.contains("::1"));
}
#[test]
fn insert_pivot_keyword_level_infomational() {
load_pivot_keywords("test_files/config/pivot_keywords.txt");
let record_json_str = r#"
{
"Event": {
"System": {
"Level": "infomational"
},
"EventData": {
"IpAddress": "10.0.0.2"
}
}
}"#;
insert_pivot_keyword(&serde_json::from_str(record_json_str).unwrap());
assert!(!PIVOT_KEYWORD
.write()
.unwrap()
.get_mut("Ip Addresses")
.unwrap()
.keywords
.contains("10.0.0.2"));
}
#[test]
fn insert_pivot_keyword_level_low() {
load_pivot_keywords("test_files/config/pivot_keywords.txt");
let record_json_str = r#"
{
"Event": {
"System": {
"Level": "low"
},
"EventData": {
"IpAddress": "10.0.0.1"
}
}
}"#;
insert_pivot_keyword(&serde_json::from_str(record_json_str).unwrap());
assert!(PIVOT_KEYWORD
.write()
.unwrap()
.get_mut("Ip Addresses")
.unwrap()
.keywords
.contains("10.0.0.1"));
}
#[test]
fn insert_pivot_keyword_level_none() {
load_pivot_keywords("test_files/config/pivot_keywords.txt");
let record_json_str = r#"
{
"Event": {
"System": {
"Level": "-"
},
"EventData": {
"IpAddress": "10.0.0.3"
}
}
}"#;
insert_pivot_keyword(&serde_json::from_str(record_json_str).unwrap());
assert!(!PIVOT_KEYWORD
.write()
.unwrap()
.get_mut("Ip Addresses")
.unwrap()
.keywords
.contains("10.0.0.3"));
}
}

5
src/detections/print.rs
View File

@@ -53,6 +53,11 @@ lazy_static! {
.unwrap()
.args
.is_present("statistics");
pub static ref PIVOT_KEYWORD_LIST_FLAG: bool = configs::CONFIG
.read()
.unwrap()
.args
.is_present("pivot-keywords-list");
}
impl Default for Message {

2
src/detections/utils.rs
View File

@@ -87,7 +87,7 @@ pub fn read_csv(filename: &str) -> Result<Vec<Vec<String>>, String> {
return Result::Err(e.to_string());
}
let mut rdr = csv::Reader::from_reader(contents.as_bytes());
let mut rdr = csv::ReaderBuilder::new().from_reader(contents.as_bytes());
rdr.records().for_each(|r| {
if r.is_err() {
return;

2
src/filter.rs
View File

@@ -65,7 +65,7 @@ impl RuleExclude {
ERROR_LOG_STACK
.lock()
.unwrap()
.push(format!("[WARN] {} does not exist", filename));
.push(format!("{} does not exist", filename));
}
return;
}

82
src/main.rs
View File

@@ -9,9 +9,12 @@ use chrono::{DateTime, Datelike, Local, TimeZone};
use evtx::{EvtxParser, ParserSettings};
use git2::Repository;
use hashbrown::{HashMap, HashSet};
use hayabusa::detections::configs::load_pivot_keywords;
use hayabusa::detections::detection::{self, EvtxRecordInfo};
use hayabusa::detections::pivot::PIVOT_KEYWORD;
use hayabusa::detections::print::{
AlertMessage, ERROR_LOG_PATH, ERROR_LOG_STACK, QUIET_ERRORS_FLAG, STATISTICS_FLAG,
AlertMessage, ERROR_LOG_PATH, ERROR_LOG_STACK, PIVOT_KEYWORD_LIST_FLAG, QUIET_ERRORS_FLAG,
STATISTICS_FLAG,
};
use hayabusa::detections::rule::{get_detection_keys, RuleNode};
use hayabusa::filter;
@@ -26,7 +29,7 @@ use std::cmp::Ordering;
use std::ffi::OsStr;
use std::fmt::Display;
use std::fs::create_dir;
use std::io::BufWriter;
use std::io::{BufWriter, Write};
use std::path::Path;
use std::sync::Arc;
use std::time::SystemTime;
@@ -71,6 +74,10 @@ impl App {
}
fn exec(&mut self) {
if *PIVOT_KEYWORD_LIST_FLAG {
load_pivot_keywords("config/pivot_keywords.txt");
}
let analysis_start_time: DateTime<Local> = Local::now();
if !configs::CONFIG.read().unwrap().args.is_present("quiet") {
self.output_logo();
@@ -118,6 +125,20 @@ impl App {
return;
}
if let Some(csv_path) = configs::CONFIG.read().unwrap().args.value_of("output") {
for (key, _) in PIVOT_KEYWORD.read().unwrap().iter() {
let keywords_file_name = csv_path.to_owned() + "-" + key + ".txt";
if Path::new(&keywords_file_name).exists() {
AlertMessage::alert(
&mut BufWriter::new(std::io::stderr().lock()),
&format!(
" The file {} already exists. Please specify a different filename.",
&keywords_file_name
),
)
.ok();
return;
}
}
if Path::new(csv_path).exists() {
AlertMessage::alert(
&mut BufWriter::new(std::io::stderr().lock()),
@@ -130,6 +151,7 @@ impl App {
return;
}
}
if *STATISTICS_FLAG {
println!("Generating Event ID Statistics");
println!();
@@ -193,6 +215,60 @@ impl App {
if ERROR_LOG_STACK.lock().unwrap().len() > 0 {
AlertMessage::create_error_log(ERROR_LOG_PATH.to_string());
}
if *PIVOT_KEYWORD_LIST_FLAG {
//ファイル出力の場合
if let Some(pivot_file) = configs::CONFIG.read().unwrap().args.value_of("output") {
for (key, pivot_keyword) in PIVOT_KEYWORD.read().unwrap().iter() {
let mut f = BufWriter::new(
fs::File::create(pivot_file.to_owned() + "-" + key + ".txt").unwrap(),
);
let mut output = "".to_string();
output += &format!("{}: ", key).to_string();
output += "( ";
for i in pivot_keyword.fields.iter() {
output += &format!("%{}% ", i).to_string();
}
output += "):";
output += "\n";
for i in pivot_keyword.keywords.iter() {
output += &format!("{}\n", i).to_string();
}
f.write_all(output.as_bytes()).unwrap();
}
//output to stdout
let mut output =
"Pivot keyword results saved to the following files:\n".to_string();
for (key, _) in PIVOT_KEYWORD.read().unwrap().iter() {
output += &(pivot_file.to_owned() + "-" + key + ".txt" + "\n");
}
println!("{}", output);
} else {
//標準出力の場合
let mut output = "The following pivot keywords were found:\n".to_string();
for (key, pivot_keyword) in PIVOT_KEYWORD.read().unwrap().iter() {
output += &format!("{}: ", key).to_string();
output += "( ";
for i in pivot_keyword.fields.iter() {
output += &format!("%{}% ", i).to_string();
}
output += "):";
output += "\n";
for i in pivot_keyword.keywords.iter() {
output += &format!("{}\n", i).to_string();
}
output += "\n";
}
print!("{}", output);
}
}
}
#[cfg(not(target_os = "windows"))]
@@ -327,7 +403,7 @@ impl App {
pb.inc();
}
detection.add_aggcondition_msges(&self.rt);
if !*STATISTICS_FLAG {
if !*STATISTICS_FLAG && !*PIVOT_KEYWORD_LIST_FLAG {
after_fact();
}
}