CSEC_PUBLIC/hayabusa
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into feature/level-tuning#390

Browse Source
This commit is contained in:
Yamato Security
2022-04-11 17:24:28 +09:00
committed by GitHub
parent 1dc9a11d94 620b6f375d
commit 861e1c5c4e
18 changed files with 896 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/afterfact.rs
+70 -70
View File
@@ -21,9 +21,11 @@ pub struct CsvFormat<'a> {
computer: &'a str,
event_i_d: &'a str,
level: &'a str,
mitre_attack: &'a str,
rule_title: &'a str,
details: &'a str,
mitre_attack: &'a str,
#[serde(skip_serializing_if = "Option::is_none")]
record_information: Option<&'a str>,
rule_path: &'a str,
file_path: &'a str,
}
@@ -37,6 +39,8 @@ pub struct DisplayFormat<'a> {
pub level: &'a str,
pub rule_title: &'a str,
pub details: &'a str,
#[serde(skip_serializing_if = "Option::is_none")]
pub record_information: Option<&'a str>,
}
/// level_color.txtファイルを読み込み対応する文字色のマッピングを返却する関数
@@ -139,81 +143,44 @@ fn emit_csv<W: std::io::Write>(
for (time, detect_infos) in messages.iter() {
for detect_info in detect_infos {
if displayflag {
if color_map.is_some() {
let output_color =
_get_output_color(color_map.as_ref().unwrap(), &detect_info.level);
wtr.serialize(DisplayFormat {
timestamp: &format!(
"{} ",
&format_time(time).truecolor(
output_color[0],
output_color[1],
output_color[2]
)
),
level: &format!(
" {} ",
&detect_info.level.truecolor(
output_color[0],
output_color[1],
output_color[2]
)
),
computer: &format!(
" {} ",
&detect_info.computername.truecolor(
output_color[0],
output_color[1],
output_color[2]
)
),
event_i_d: &format!(
" {} ",
&detect_info.eventid.truecolor(
output_color[0],
output_color[1],
output_color[2]
)
),
rule_title: &format!(
" {} ",
&detect_info.alert.truecolor(
output_color[0],
output_color[1],
output_color[2]
)
),
details: &format!(
" {}",
&detect_info.detail.truecolor(
output_color[0],
output_color[1],
output_color[2]
)
),
})?;
} else {
wtr.serialize(DisplayFormat {
timestamp: &format!("{} ", &format_time(time)),
level: &format!(" {} ", &detect_info.level),
computer: &format!(" {} ", &detect_info.computername),
event_i_d: &format!(" {} ", &detect_info.eventid),
rule_title: &format!(" {} ", &detect_info.alert),
details: &format!(" {}", &detect_info.detail),
})?;
}
let colors = color_map
.as_ref()
.map(|cl_mp| _get_output_color(cl_mp, &detect_info.level));
let colors = colors.as_ref();
let recinfo = detect_info
.record_information
.as_ref()
.map(|recinfo| _format_cell(recinfo, ColPos::Last, colors));
let details = detect_info
.detail
.chars()
.filter(|&c| !c.is_control())
.collect::<String>();
let dispformat = DisplayFormat {
timestamp: &_format_cell(&format_time(time), ColPos::First, colors),
level: &_format_cell(&detect_info.level, ColPos::Other, colors),
computer: &_format_cell(&detect_info.computername, ColPos::Other, colors),
event_i_d: &_format_cell(&detect_info.eventid, ColPos::Other, colors),
rule_title: &_format_cell(&detect_info.alert, ColPos::Other, colors),
details: &_format_cell(&details, ColPos::Other, colors),
record_information: recinfo.as_deref(),
};
wtr.serialize(dispformat)?;
} else {
// csv出力時フォーマット
wtr.serialize(CsvFormat {
timestamp: &format_time(time),
file_path: &detect_info.filepath,
rule_path: &detect_info.rulepath,
level: &detect_info.level,
computer: &detect_info.computername,
event_i_d: &detect_info.eventid,
mitre_attack: &detect_info.tag_info,
rule_title: &detect_info.alert,
details: &detect_info.detail,
mitre_attack: &detect_info.tag_info,
record_information: detect_info.record_information.as_deref(),
file_path: &detect_info.filepath,
rule_path: &detect_info.rulepath,
})?;
}
let level_suffix = *configs::LEVELMAP
@@ -245,6 +212,29 @@ fn emit_csv<W: std::io::Write>(
Ok(())
}
enum ColPos {
First, // 先頭
Last, // 最後
Other, // それ以外
}
fn _format_cellpos(column: ColPos, colval: &str) -> String {
return match column {
ColPos::First => format!("{} ", colval),
ColPos::Last => format!(" {}", colval),
ColPos::Other => format!(" {} ", colval),
};
}
fn _format_cell(word: &str, column: ColPos, output_color: Option<&Vec<u8>>) -> String {
if let Some(color) = output_color {
let colval = format!("{}", word.truecolor(color[0], color[1], color[2]));
_format_cellpos(column, &colval)
} else {
_format_cellpos(column, word)
}
}
/// 与えられたユニークな検知数と全体の検知数の情報(レベル別と総計)を元に結果文を標準出力に表示する関数
fn _print_unique_results(
mut counts_by_level: Vec<u128>,
@@ -351,6 +341,7 @@ mod tests {
let test_eventid = "1111";
let output = "pokepoke";
let test_attack = "execution/txxxx.yyy";
let test_recinfo = "record_infoinfo11";
{
let mut messages = print::MESSAGES.lock().unwrap();
messages.clear();
@@ -381,6 +372,7 @@ mod tests {
alert: test_title.to_string(),
detail: String::default(),
tag_info: test_attack.to_string(),
record_information: Option::Some(test_recinfo.to_string()),
},
);
}
@@ -389,7 +381,7 @@ mod tests {
.unwrap();
let expect_tz = expect_time.with_timezone(&Local);
let expect =
"Timestamp,Computer,EventID,Level,RuleTitle,Details,MitreAttack,RulePath,FilePath\n"
"Timestamp,Computer,EventID,Level,MitreAttack,RuleTitle,Details,RecordInformation,RulePath,FilePath\n"
.to_string()
+ &expect_tz
.clone()
@@ -402,11 +394,13 @@ mod tests {
+ ","
+ test_level
+ ","
+ test_attack
+ ","
+ test_title
+ ","
+ output
+ ","
+ test_attack
+ test_recinfo
+ ","
+ testrulepath
+ ","
@@ -463,6 +457,7 @@ mod tests {
alert: test_title.to_string(),
detail: String::default(),
tag_info: test_attack.to_string(),
record_information: Option::Some(String::default()),
},
);
messages.debug();
@@ -471,7 +466,8 @@ mod tests {
.datetime_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ")
.unwrap();
let expect_tz = expect_time.with_timezone(&Local);
let expect_header = "Timestamp|Computer|EventID|Level|RuleTitle|Details\n";
let expect_header =
"Timestamp|Computer|EventID|Level|RuleTitle|Details|RecordInformation\n";
let expect_colored = expect_header.to_string()
+ &get_white_color_string(
&expect_tz
@@ -489,6 +485,8 @@ mod tests {
+ &get_white_color_string(test_title)
+ " | "
+ &get_white_color_string(output)
+ " | "
+ &get_white_color_string("")
+ "\n";
let expect_nocoloed = expect_header.to_string()
+ &expect_tz
@@ -505,6 +503,8 @@ mod tests {
+ test_title
+ " | "
+ output
+ " | "
+ ""
+ "\n";
let mut file: Box<dyn io::Write> =
src/detections/configs.rs
+41 -2
View File
@@ -1,3 +1,5 @@
use crate::detections::pivot::PivotKeyword;
use crate::detections::pivot::PIVOT_KEYWORD;
use crate::detections::print::AlertMessage;
use crate::detections::utils;
use chrono::{DateTime, Utc};
@@ -70,7 +72,8 @@ fn build_app<'a>() -> ArgMatches<'a> {
let usages = "-d --directory=[DIRECTORY] 'Directory of multiple .evtx files.'
-f --filepath=[FILEPATH] 'File path to one .evtx file.'
-r --rules=[RULEFILE/RULEDIRECTORY] 'Rule file or directory. (Default: ./rules)'
-F --full-data 'Print all field information.'
-r --rules=[RULEDIRECTORY/RULEFILE] 'Rule file or directory (default: ./rules)'
-c --color 'Output with color. (Terminal needs to support True Color.)'
-C --config=[RULECONFIGDIRECTORY] 'Rule config folder. (Default: ./rules/config)'
-o --output=[CSV_TIMELINE] 'Save the timeline in CSV format. (Example: results.csv)'
@@ -89,6 +92,7 @@ fn build_app<'a>() -> ArgMatches<'a> {
-s --statistics 'Prints statistics of event IDs.'
-q --quiet 'Quiet mode. Do not display the launch banner.'
-Q --quiet-errors 'Quiet errors mode. Do not save error logs.'
-p --pivot-keywords-list 'Create a list of pivot keywords.'
--contributors 'Prints the list of contributors.'";
App::new(&program)
.about("Hayabusa: Aiming to be the world's greatest Windows event log analysis tool!")
@@ -276,6 +280,7 @@ impl Default for EventKeyAliasConfig {
fn load_eventkey_alias(path: &str) -> EventKeyAliasConfig {
let mut config = EventKeyAliasConfig::new();
// eventkey_aliasが読み込めなかったらエラーで終了とする。
let read_result = utils::read_csv(path);
if read_result.is_err() {
AlertMessage::alert(
@@ -285,7 +290,7 @@ fn load_eventkey_alias(path: &str) -> EventKeyAliasConfig {
.ok();
return config;
}
// eventkey_aliasが読み込めなかったらエラーで終了とする。
read_result.unwrap().into_iter().for_each(|line| {
if line.len() != 2 {
return;
@@ -310,6 +315,40 @@ fn load_eventkey_alias(path: &str) -> EventKeyAliasConfig {
config
}
///設定ファイルを読み込み、keyとfieldsのマップをPIVOT_KEYWORD大域変数にロードする。
pub fn load_pivot_keywords(path: &str) {
let read_result = utils::read_txt(path);
if read_result.is_err() {
AlertMessage::alert(
&mut BufWriter::new(std::io::stderr().lock()),
read_result.as_ref().unwrap_err(),
)
.ok();
}
read_result.unwrap().into_iter().for_each(|line| {
let map: Vec<&str> = line.split('.').collect();
if map.len() != 2 {
return;
}
//存在しなければ、keyを作成
PIVOT_KEYWORD
.write()
.unwrap()
.entry(map[0].to_string())
.or_insert(PivotKeyword::new());
PIVOT_KEYWORD
.write()
.unwrap()
.get_mut(&map[0].to_string())
.unwrap()
.fields
.insert(map[1].to_string());
});
}
#[derive(Debug, Clone)]
pub struct EventInfo {
pub evttitle: String,
src/detections/detection.rs
+56 -29
View File
@@ -1,12 +1,15 @@
extern crate csv;
use crate::detections::configs;
use crate::detections::pivot::insert_pivot_keyword;
use crate::detections::print::AlertMessage;
use crate::detections::print::DetectInfo;
use crate::detections::print::ERROR_LOG_STACK;
use crate::detections::print::MESSAGES;
use crate::detections::print::PIVOT_KEYWORD_LIST_FLAG;
use crate::detections::print::QUIET_ERRORS_FLAG;
use crate::detections::print::STATISTICS_FLAG;
use crate::detections::print::TAGS_CONFIG;
use crate::detections::rule;
use crate::detections::rule::AggResult;
use crate::detections::rule::RuleNode;
@@ -29,6 +32,7 @@ pub struct EvtxRecordInfo {
pub record: Value, // 1レコード分のデータをJSON形式にシリアライズしたもの
pub data_string: String,
pub key_2_value: hashbrown::HashMap<String, String>,
pub record_information: Option<String>,
}
impl EvtxRecordInfo {
@@ -177,6 +181,12 @@ impl Detection {
if !result {
continue;
}
if *PIVOT_KEYWORD_LIST_FLAG {
insert_pivot_keyword(&record_info.record);
continue;
}
// aggregation conditionが存在しない場合はそのまま出力対応を行う
if !agg_condition {
Detection::insert_message(&rule, record_info);
@@ -192,26 +202,32 @@ impl Detection {
.as_vec()
.unwrap_or(&Vec::default())
.iter()
.map(|info| info.as_str().unwrap_or("").replace("attack.", ""))
.filter_map(|info| TAGS_CONFIG.get(info.as_str().unwrap_or(&String::default())))
.map(|str| str.to_owned())
.collect();
let recinfo = record_info
.record_information
.as_ref()
.map(|recinfo| recinfo.to_string());
let detect_info = DetectInfo {
filepath: record_info.evtx_filepath.to_string(),
rulepath: rule.rulepath.to_string(),
level: rule.yaml["level"].as_str().unwrap_or("-").to_string(),
computername: record_info.record["Event"]["System"]["Computer"]
.to_string()
.replace('\"', ""),
eventid: get_serde_number_to_string(&record_info.record["Event"]["System"]["EventID"])
.unwrap_or_else(|| "-".to_owned()),
alert: rule.yaml["title"].as_str().unwrap_or("").to_string(),
detail: String::default(),
tag_info: tag_info.join(" | "),
record_information: recinfo,
};
MESSAGES.lock().unwrap().insert(
&record_info.record,
rule.yaml["details"].as_str().unwrap_or("").to_string(),
DetectInfo {
filepath: record_info.evtx_filepath.to_string(),
rulepath: rule.rulepath.to_string(),
level: rule.yaml["level"].as_str().unwrap_or("-").to_string(),
computername: record_info.record["Event"]["System"]["Computer"]
.to_string()
.replace('\"', ""),
eventid: get_serde_number_to_string(
&record_info.record["Event"]["System"]["EventID"],
)
.unwrap_or_else(|| "-".to_owned()),
alert: rule.yaml["title"].as_str().unwrap_or("").to_string(),
detail: String::default(),
tag_info: tag_info.join(" : "),
},
detect_info,
);
}
@@ -224,19 +240,27 @@ impl Detection {
.map(|info| info.as_str().unwrap_or("").replace("attack.", ""))
.collect();
let output = Detection::create_count_output(rule, &agg_result);
MESSAGES.lock().unwrap().insert_message(
DetectInfo {
filepath: "-".to_owned(),
rulepath: rule.rulepath.to_owned(),
level: rule.yaml["level"].as_str().unwrap_or("").to_owned(),
computername: "-".to_owned(),
eventid: "-".to_owned(),
alert: rule.yaml["title"].as_str().unwrap_or("").to_owned(),
detail: output,
tag_info: tag_info.join(" : "),
},
agg_result.start_timedate,
)
let rec_info = if configs::CONFIG.read().unwrap().args.is_present("full-data") {
Option::Some(String::default())
} else {
Option::None
};
let detect_info = DetectInfo {
filepath: "-".to_owned(),
rulepath: rule.rulepath.to_owned(),
level: rule.yaml["level"].as_str().unwrap_or("").to_owned(),
computername: "-".to_owned(),
eventid: "-".to_owned(),
alert: rule.yaml["title"].as_str().unwrap_or("").to_owned(),
detail: output,
record_information: rec_info,
tag_info: tag_info.join(" : "),
};
MESSAGES
.lock()
.unwrap()
.insert_message(detect_info, agg_result.start_timedate)
}
///aggregation conditionのcount部分の検知出力文の文字列を返す関数
@@ -499,4 +523,7 @@ mod tests {
expected_output
);
}
#[test]
fn test_create_fields_value() {}
}
src/detections/mod.rs
+1
View File
@@ -1,5 +1,6 @@
pub mod configs;
pub mod detection;
pub mod pivot;
pub mod print;
pub mod rule;
pub mod utils;
src/detections/pivot.rs
+270
View File
@@ -0,0 +1,270 @@
use hashbrown::HashMap;
use hashbrown::HashSet;
use lazy_static::lazy_static;
use serde_json::Value;
use std::sync::RwLock;
use crate::detections::configs;
use crate::detections::utils::get_serde_number_to_string;
#[derive(Debug)]
pub struct PivotKeyword {
pub keywords: HashSet<String>,
pub fields: HashSet<String>,
}
lazy_static! {
pub static ref PIVOT_KEYWORD: RwLock<HashMap<String, PivotKeyword>> =
RwLock::new(HashMap::new());
}
impl Default for PivotKeyword {
fn default() -> Self {
Self::new()
}
}
impl PivotKeyword {
pub fn new() -> PivotKeyword {
PivotKeyword {
keywords: HashSet::new(),
fields: HashSet::new(),
}
}
}
///levelがlowより大きいレコードの場合、keywordがrecord内にみつかれば、
///それをPIVOT_KEYWORD.keywordsに入れる。
pub fn insert_pivot_keyword(event_record: &Value) {
//levelがlow異常なら続ける
let mut is_exist_event_key = false;
let mut tmp_event_record: &Value = event_record;
for s in ["Event", "System", "Level"] {
if let Some(record) = tmp_event_record.get(s) {
is_exist_event_key = true;
tmp_event_record = record;
}
}
if is_exist_event_key {
let hash_value = get_serde_number_to_string(tmp_event_record);
if hash_value.is_some() && hash_value.as_ref().unwrap() == "infomational"
|| hash_value.as_ref().unwrap() == "undefined"
|| hash_value.as_ref().unwrap() == "-"
{
return;
}
} else {
return;
}
for (_, pivot) in PIVOT_KEYWORD.write().unwrap().iter_mut() {
for field in &pivot.fields {
if let Some(array_str) = configs::EVENTKEY_ALIAS.get_event_key(&String::from(field)) {
let split: Vec<&str> = array_str.split('.').collect();
let mut is_exist_event_key = false;
let mut tmp_event_record: &Value = event_record;
for s in split {
if let Some(record) = tmp_event_record.get(s) {
is_exist_event_key = true;
tmp_event_record = record;
}
}
if is_exist_event_key {
let hash_value = get_serde_number_to_string(tmp_event_record);
if let Some(value) = hash_value {
if value == "-" || value == "127.0.0.1" || value == "::1" {
continue;
}
pivot.keywords.insert(value);
};
}
}
}
}
}
#[cfg(test)]
mod tests {
use crate::detections::configs::load_pivot_keywords;
use crate::detections::pivot::insert_pivot_keyword;
use crate::detections::pivot::PIVOT_KEYWORD;
use serde_json;
//PIVOT_KEYWORDはグローバルなので、他の関数の影響も考慮する必要がある。
#[test]
fn insert_pivot_keyword_local_ip4() {
load_pivot_keywords("test_files/config/pivot_keywords.txt");
let record_json_str = r#"
{
"Event": {
"System": {
"Level": "high"
},
"EventData": {
"IpAddress": "127.0.0.1"
}
}
}"#;
insert_pivot_keyword(&serde_json::from_str(record_json_str).unwrap());
assert!(!PIVOT_KEYWORD
.write()
.unwrap()
.get_mut("Ip Addresses")
.unwrap()
.keywords
.contains("127.0.0.1"));
}
#[test]
fn insert_pivot_keyword_ip4() {
load_pivot_keywords("test_files/config/pivot_keywords.txt");
let record_json_str = r#"
{
"Event": {
"System": {
"Level": "high"
},
"EventData": {
"IpAddress": "10.0.0.1"
}
}
}"#;
insert_pivot_keyword(&serde_json::from_str(record_json_str).unwrap());
assert!(PIVOT_KEYWORD
.write()
.unwrap()
.get_mut("Ip Addresses")
.unwrap()
.keywords
.contains("10.0.0.1"));
}
#[test]
fn insert_pivot_keyword_ip_empty() {
load_pivot_keywords("test_files/config/pivot_keywords.txt");
let record_json_str = r#"
{
"Event": {
"System": {
"Level": "high"
},
"EventData": {
"IpAddress": "-"
}
}
}"#;
insert_pivot_keyword(&serde_json::from_str(record_json_str).unwrap());
assert!(!PIVOT_KEYWORD
.write()
.unwrap()
.get_mut("Ip Addresses")
.unwrap()
.keywords
.contains("-"));
}
#[test]
fn insert_pivot_keyword_local_ip6() {
load_pivot_keywords("test_files/config/pivot_keywords.txt");
let record_json_str = r#"
{
"Event": {
"System": {
"Level": "high"
},
"EventData": {
"IpAddress": "::1"
}
}
}"#;
insert_pivot_keyword(&serde_json::from_str(record_json_str).unwrap());
assert!(!PIVOT_KEYWORD
.write()
.unwrap()
.get_mut("Ip Addresses")
.unwrap()
.keywords
.contains("::1"));
}
#[test]
fn insert_pivot_keyword_level_infomational() {
load_pivot_keywords("test_files/config/pivot_keywords.txt");
let record_json_str = r#"
{
"Event": {
"System": {
"Level": "infomational"
},
"EventData": {
"IpAddress": "10.0.0.2"
}
}
}"#;
insert_pivot_keyword(&serde_json::from_str(record_json_str).unwrap());
assert!(!PIVOT_KEYWORD
.write()
.unwrap()
.get_mut("Ip Addresses")
.unwrap()
.keywords
.contains("10.0.0.2"));
}
#[test]
fn insert_pivot_keyword_level_low() {
load_pivot_keywords("test_files/config/pivot_keywords.txt");
let record_json_str = r#"
{
"Event": {
"System": {
"Level": "low"
},
"EventData": {
"IpAddress": "10.0.0.1"
}
}
}"#;
insert_pivot_keyword(&serde_json::from_str(record_json_str).unwrap());
assert!(PIVOT_KEYWORD
.write()
.unwrap()
.get_mut("Ip Addresses")
.unwrap()
.keywords
.contains("10.0.0.1"));
}
#[test]
fn insert_pivot_keyword_level_none() {
load_pivot_keywords("test_files/config/pivot_keywords.txt");
let record_json_str = r#"
{
"Event": {
"System": {
"Level": "-"
},
"EventData": {
"IpAddress": "10.0.0.3"
}
}
}"#;
insert_pivot_keyword(&serde_json::from_str(record_json_str).unwrap());
assert!(!PIVOT_KEYWORD
.write()
.unwrap()
.get_mut("Ip Addresses")
.unwrap()
.keywords
.contains("10.0.0.3"));
}
}
src/detections/print.rs
+55 -1
View File
@@ -31,6 +31,7 @@ pub struct DetectInfo {
pub alert: String,
pub detail: String,
pub tag_info: String,
pub record_information: Option<String>,
}
pub struct AlertMessage {}
@@ -53,6 +54,13 @@ lazy_static! {
.unwrap()
.args
.is_present("statistics");
pub static ref TAGS_CONFIG: HashMap<String, String> =
Message::create_tags_config("config/output_tag.txt");
pub static ref PIVOT_KEYWORD_LIST_FLAG: bool = configs::CONFIG
.read()
.unwrap()
.args
.is_present("pivot-keywords-list");
}
impl Default for Message {
@@ -67,6 +75,33 @@ impl Message {
Message { map: messages }
}
/// ファイルパスで記載されたtagでのフル名、表示の際に置き換えられる文字列のHashMapを作成する関数。tagではこのHashMapのキーに対応しない出力は出力しないものとする
/// ex. attack.impact,Impact
pub fn create_tags_config(path: &str) -> HashMap<String, String> {
let read_result = utils::read_csv(path);
if read_result.is_err() {
AlertMessage::alert(
&mut BufWriter::new(std::io::stderr().lock()),
read_result.as_ref().unwrap_err(),
)
.ok();
return HashMap::default();
}
let mut ret: HashMap<String, String> = HashMap::new();
read_result.unwrap().into_iter().for_each(|line| {
if line.len() != 2 {
return;
}
let empty = &"".to_string();
let tag_full_str = line.get(0).unwrap_or(empty).trim();
let tag_replace_str = line.get(1).unwrap_or(empty).trim();
ret.insert(tag_full_str.to_owned(), tag_replace_str.to_owned());
});
ret
}
/// メッセージの設定を行う関数。aggcondition対応のためrecordではなく出力をする対象時間がDatetime形式での入力としている
pub fn insert_message(&mut self, detect_info: DetectInfo, event_time: DateTime<Utc>) {
if let Some(v) = self.map.get_mut(&event_time) {
@@ -217,6 +252,7 @@ impl AlertMessage {
mod tests {
use crate::detections::print::DetectInfo;
use crate::detections::print::{AlertMessage, Message};
use hashbrown::HashMap;
use serde_json::Value;
use std::io::BufWriter;
@@ -250,6 +286,7 @@ mod tests {
alert: "test1".to_string(),
detail: String::default(),
tag_info: "txxx.001".to_string(),
record_information: Option::Some("record_information1".to_string()),
},
);
@@ -280,6 +317,7 @@ mod tests {
alert: "test2".to_string(),
detail: String::default(),
tag_info: "txxx.002".to_string(),
record_information: Option::Some("record_information2".to_string()),
},
);
@@ -310,6 +348,7 @@ mod tests {
alert: "test3".to_string(),
detail: String::default(),
tag_info: "txxx.003".to_string(),
record_information: Option::Some("record_information3".to_string()),
},
);
@@ -335,12 +374,13 @@ mod tests {
alert: "test4".to_string(),
detail: String::default(),
tag_info: "txxx.004".to_string(),
record_information: Option::Some("record_information4".to_string()),
},
);
let display = format!("{}", format_args!("{:?}", message));
println!("display::::{}", display);
let expect = "Message { map: {1970-01-01T00:00:00Z: [DetectInfo { filepath: \"a\", rulepath: \"test_rule4\", level: \"medium\", computername: \"testcomputer4\", eventid: \"4\", alert: \"test4\", detail: \"CommandLine4: hoge\", tag_info: \"txxx.004\" }], 1996-02-27T01:05:01Z: [DetectInfo { filepath: \"a\", rulepath: \"test_rule\", level: \"high\", computername: \"testcomputer1\", eventid: \"1\", alert: \"test1\", detail: \"CommandLine1: hoge\", tag_info: \"txxx.001\" }, DetectInfo { filepath: \"a\", rulepath: \"test_rule2\", level: \"high\", computername: \"testcomputer2\", eventid: \"2\", alert: \"test2\", detail: \"CommandLine2: hoge\", tag_info: \"txxx.002\" }], 2000-01-21T09:06:01Z: [DetectInfo { filepath: \"a\", rulepath: \"test_rule3\", level: \"high\", computername: \"testcomputer3\", eventid: \"3\", alert: \"test3\", detail: \"CommandLine3: hoge\", tag_info: \"txxx.003\" }]} }";
let expect = "Message { map: {1970-01-01T00:00:00Z: [DetectInfo { filepath: \"a\", rulepath: \"test_rule4\", level: \"medium\", computername: \"testcomputer4\", eventid: \"4\", alert: \"test4\", detail: \"CommandLine4: hoge\", tag_info: \"txxx.004\", record_information: Some(\"record_information4\") }], 1996-02-27T01:05:01Z: [DetectInfo { filepath: \"a\", rulepath: \"test_rule\", level: \"high\", computername: \"testcomputer1\", eventid: \"1\", alert: \"test1\", detail: \"CommandLine1: hoge\", tag_info: \"txxx.001\", record_information: Some(\"record_information1\") }, DetectInfo { filepath: \"a\", rulepath: \"test_rule2\", level: \"high\", computername: \"testcomputer2\", eventid: \"2\", alert: \"test2\", detail: \"CommandLine2: hoge\", tag_info: \"txxx.002\", record_information: Some(\"record_information2\") }], 2000-01-21T09:06:01Z: [DetectInfo { filepath: \"a\", rulepath: \"test_rule3\", level: \"high\", computername: \"testcomputer3\", eventid: \"3\", alert: \"test3\", detail: \"CommandLine3: hoge\", tag_info: \"txxx.003\", record_information: Some(\"record_information3\") }]} }";
assert_eq!(display, expect);
}
@@ -461,4 +501,18 @@ mod tests {
expected,
);
}
#[test]
/// output_tag.txtの読み込みテスト
fn test_load_output_tag() {
let actual = Message::create_tags_config("test_files/config/output_tag.txt");
let expected: HashMap<String, String> = HashMap::from([
("attack.impact".to_string(), "Impact".to_string()),
("xxx".to_string(), "yyy".to_string()),
]);
assert_eq!(actual.len(), expected.len());
for (k, v) in expected.iter() {
assert!(actual.get(k).unwrap_or(&String::default()) == v);
}
}
}
src/detections/utils.rs
+167 -13
View File
@@ -10,11 +10,13 @@ use tokio::runtime::Runtime;
use chrono::{DateTime, TimeZone, Utc};
use regex::Regex;
use serde_json::Value;
use std::cmp::Ordering;
use std::fs::File;
use std::io::prelude::*;
use std::io::{BufRead, BufReader};
use std::str;
use std::string::String;
use std::vec;
use super::detection::EvtxRecordInfo;
@@ -87,7 +89,7 @@ pub fn read_csv(filename: &str) -> Result<Vec<Vec<String>>, String> {
return Result::Err(e.to_string());
}
let mut rdr = csv::Reader::from_reader(contents.as_bytes());
let mut rdr = csv::ReaderBuilder::new().from_reader(contents.as_bytes());
rdr.records().for_each(|r| {
if r.is_err() {
return;
@@ -199,15 +201,6 @@ pub fn create_tokio_runtime() -> Runtime {
// EvtxRecordInfoを作成します。
pub fn create_rec_info(data: Value, path: String, keys: &[String]) -> EvtxRecordInfo {
// EvtxRecordInfoを作る
let data_str = data.to_string();
let mut rec = EvtxRecordInfo {
evtx_filepath: path,
record: data,
data_string: data_str,
key_2_value: hashbrown::HashMap::new(),
};
// 高速化のための処理
// 例えば、Value型から"Event.System.EventID"の値を取得しようとすると、value["Event"]["System"]["EventID"]のように3回アクセスする必要がある。
@@ -215,8 +208,9 @@ pub fn create_rec_info(data: Value, path: String, keys: &[String]) -> EvtxRecord
// これなら、"Event.System.EventID"というキーを1回指定するだけで値を取得できるようになるので、高速化されるはず。
// あと、serde_jsonのValueからvalue["Event"]みたいな感じで値を取得する処理がなんか遅いので、そういう意味でも早くなるかも
// それと、serde_jsonでは内部的に標準ライブラリのhashmapを使用しているが、hashbrownを使った方が早くなるらしい。
let mut key_2_values = hashbrown::HashMap::new();
for key in keys {
let val = get_event_value(key, &rec.record);
let val = get_event_value(key, &data);
if val.is_none() {
continue;
}
@@ -226,10 +220,110 @@ pub fn create_rec_info(data: Value, path: String, keys: &[String]) -> EvtxRecord
continue;
}
rec.key_2_value.insert(key.trim().to_string(), val.unwrap());
key_2_values.insert(key.to_string(), val.unwrap());
}
rec
// EvtxRecordInfoを作る
let data_str = data.to_string();
let rec_info = if configs::CONFIG.read().unwrap().args.is_present("full-data") {
Option::Some(create_recordinfos(&data))
} else {
Option::None
};
EvtxRecordInfo {
evtx_filepath: path,
record: data,
data_string: data_str,
key_2_value: key_2_values,
record_information: rec_info,
}
}
/**
* CSVのrecord infoカラムに出力する文字列を作る
*/
fn create_recordinfos(record: &Value) -> String {
let mut output = vec![];
_collect_recordinfo(&mut vec![], "", record, &mut output);
// 同じレコードなら毎回同じ出力になるようにソートしておく
output.sort_by(|(left, left_data), (right, right_data)| {
let ord = left.cmp(right);
if ord == Ordering::Equal {
left_data.cmp(right_data)
} else {
ord
}
});
let summary: Vec<String> = output
.iter()
.map(|(key, value)| {
return format!("{}:{}", key, value);
})
.collect();
// 標準出力する時はセルがハイプ区切りになるので、パイプ区切りにしない
if configs::CONFIG.read().unwrap().args.is_present("output") {
summary.join(" | ")
} else {
summary.join(" ")
}
}
/**
* CSVのfieldsカラムに出力する要素を全て収集する
*/
fn _collect_recordinfo<'a>(
keys: &mut Vec<&'a str>,
parent_key: &'a str,
value: &'a Value,
output: &mut Vec<(String, String)>,
) {
match value {
Value::Array(ary) => {
for sub_value in ary {
_collect_recordinfo(keys, parent_key, sub_value, output);
}
}
Value::Object(obj) => {
// lifetimeの関係でちょっと変な実装になっている
if !parent_key.is_empty() {
keys.push(parent_key);
}
for (key, value) in obj {
// 属性は出力しない
if key.ends_with("_attributes") {
continue;
}
// Event.Systemは出力しない
if key.eq("System") && keys.get(0).unwrap_or(&"").eq(&"Event") {
continue;
}
_collect_recordinfo(keys, key, value, output);
}
if !parent_key.is_empty() {
keys.pop();
}
}
Value::Null => (),
_ => {
// 一番子の要素の値しか収集しない
let strval = value_to_string(value);
if let Some(strval) = strval {
let strval = strval.trim().chars().fold(String::default(), |mut acc, c| {
if c.is_control() || c.is_ascii_whitespace() {
acc.push(' ');
} else {
acc.push(c);
};
acc
});
output.push((parent_key.to_string(), strval));
}
}
}
}
#[cfg(test)]
@@ -238,6 +332,66 @@ mod tests {
use regex::Regex;
use serde_json::Value;
#[test]
fn test_create_recordinfos() {
let record_json_str = r#"
{
"Event": {
"System": {"EventID": 4103, "Channel": "PowerShell", "Computer":"DESKTOP-ICHIICHI"},
"UserData": {"User": "u1", "AccessMask": "%%1369", "Process":"lsass.exe"},
"UserData_attributes": {"xmlns": "http://schemas.microsoft.com/win/2004/08/events/event"}
},
"Event_attributes": {"xmlns": "http://schemas.microsoft.com/win/2004/08/events/event"}
}"#;
match serde_json::from_str(record_json_str) {
Ok(record) => {
let ret = utils::create_recordinfos(&record);
// Systemは除外される/属性(_attributesも除外される)/key順に並ぶ
let expected = "AccessMask:%%1369 Process:lsass.exe User:u1".to_string();
assert_eq!(ret, expected);
}
Err(_) => {
panic!("Failed to parse json record.");
}
}
}
#[test]
fn test_create_recordinfos2() {
// EventDataの特殊ケース
let record_json_str = r#"
{
"Event": {
"System": {"EventID": 4103, "Channel": "PowerShell", "Computer":"DESKTOP-ICHIICHI"},
"EventData": {
"Binary": "hogehoge",
"Data":[
"Data1",
"DataData2",
"",
"DataDataData3"
]
},
"EventData_attributes": {"xmlns": "http://schemas.microsoft.com/win/2004/08/events/event"}
},
"Event_attributes": {"xmlns": "http://schemas.microsoft.com/win/2004/08/events/event"}
}"#;
match serde_json::from_str(record_json_str) {
Ok(record) => {
let ret = utils::create_recordinfos(&record);
// Systemは除外される/属性(_attributesも除外される)/key順に並ぶ
let expected = "Binary:hogehoge Data: Data:Data1 Data:DataData2 Data:DataDataData3"
.to_string();
assert_eq!(ret, expected);
}
Err(_) => {
panic!("Failed to parse json record.");
}
}
}
#[test]
fn test_check_regex() {
let regexes: Vec<Regex> =
src/filter.rs
+1 -1
View File
@@ -59,7 +59,7 @@ impl RuleExclude {
ERROR_LOG_STACK
.lock()
.unwrap()
.push(format!("[WARN] {} does not exist", filename));
.push(format!("{} does not exist", filename));
}
return;
}
src/main.rs
+109 -5
View File
@@ -9,9 +9,12 @@ use chrono::{DateTime, Datelike, Local, TimeZone};
use evtx::{EvtxParser, ParserSettings};
use git2::Repository;
use hashbrown::{HashMap, HashSet};
use hayabusa::detections::configs::load_pivot_keywords;
use hayabusa::detections::detection::{self, EvtxRecordInfo};
use hayabusa::detections::pivot::PIVOT_KEYWORD;
use hayabusa::detections::print::{
AlertMessage, ERROR_LOG_PATH, ERROR_LOG_STACK, QUIET_ERRORS_FLAG, STATISTICS_FLAG,
AlertMessage, ERROR_LOG_PATH, ERROR_LOG_STACK, PIVOT_KEYWORD_LIST_FLAG, QUIET_ERRORS_FLAG,
STATISTICS_FLAG,
};
use hayabusa::detections::rule::{get_detection_keys, RuleNode};
use hayabusa::filter;
@@ -24,14 +27,15 @@ use hhmmss::Hhmmss;
use pbr::ProgressBar;
use serde_json::Value;
use std::cmp::Ordering;
use std::ffi::OsStr;
use std::ffi::{OsStr, OsString};
use std::fmt::Display;
use std::fs::create_dir;
use std::io::BufWriter;
use std::io::{BufWriter, Write};
use std::path::Path;
use std::sync::Arc;
use std::time::SystemTime;
use std::{
env,
fs::{self, File},
path::PathBuf,
vec,
@@ -41,7 +45,7 @@ use tokio::spawn;
use tokio::task::JoinHandle;
#[cfg(target_os = "windows")]
use {is_elevated::is_elevated, std::env};
use is_elevated::is_elevated;
// 一度にtimelineやdetectionを実行する行数
const MAX_DETECT_RECORDS: usize = 5000;
@@ -72,6 +76,10 @@ impl App {
}
fn exec(&mut self) {
if *PIVOT_KEYWORD_LIST_FLAG {
load_pivot_keywords("config/pivot_keywords.txt");
}
let analysis_start_time: DateTime<Local> = Local::now();
// Show usage when no arguments.
@@ -90,6 +98,17 @@ impl App {
&analysis_start_time.day().to_owned()
));
}
if !self.is_matched_architecture_and_binary() {
AlertMessage::alert(
&mut BufWriter::new(std::io::stderr().lock()),
"The hayabusa version you ran does not match your PC architecture.\n Please use the correct architecture. (Binary ending in -x64.exe for 64-bit and -x86.exe for 32-bit.)",
)
.ok();
println!();
return;
}
if configs::CONFIG
.read()
.unwrap()
@@ -124,6 +143,20 @@ impl App {
}
if let Some(csv_path) = configs::CONFIG.read().unwrap().args.value_of("output") {
for (key, _) in PIVOT_KEYWORD.read().unwrap().iter() {
let keywords_file_name = csv_path.to_owned() + "-" + key + ".txt";
if Path::new(&keywords_file_name).exists() {
AlertMessage::alert(
&mut BufWriter::new(std::io::stderr().lock()),
&format!(
" The file {} already exists. Please specify a different filename.",
&keywords_file_name
),
)
.ok();
return;
}
}
if Path::new(csv_path).exists() {
AlertMessage::alert(
&mut BufWriter::new(std::io::stderr().lock()),
@@ -136,6 +169,7 @@ impl App {
return;
}
}
if *STATISTICS_FLAG {
println!("Generating Event ID Statistics");
println!();
@@ -234,6 +268,60 @@ impl App {
if ERROR_LOG_STACK.lock().unwrap().len() > 0 {
AlertMessage::create_error_log(ERROR_LOG_PATH.to_string());
}
if *PIVOT_KEYWORD_LIST_FLAG {
//ファイル出力の場合
if let Some(pivot_file) = configs::CONFIG.read().unwrap().args.value_of("output") {
for (key, pivot_keyword) in PIVOT_KEYWORD.read().unwrap().iter() {
let mut f = BufWriter::new(
fs::File::create(pivot_file.to_owned() + "-" + key + ".txt").unwrap(),
);
let mut output = "".to_string();
output += &format!("{}: ", key).to_string();
output += "( ";
for i in pivot_keyword.fields.iter() {
output += &format!("%{}% ", i).to_string();
}
output += "):";
output += "\n";
for i in pivot_keyword.keywords.iter() {
output += &format!("{}\n", i).to_string();
}
f.write_all(output.as_bytes()).unwrap();
}
//output to stdout
let mut output =
"Pivot keyword results saved to the following files:\n".to_string();
for (key, _) in PIVOT_KEYWORD.read().unwrap().iter() {
output += &(pivot_file.to_owned() + "-" + key + ".txt" + "\n");
}
println!("{}", output);
} else {
//標準出力の場合
let mut output = "The following pivot keywords were found:\n".to_string();
for (key, pivot_keyword) in PIVOT_KEYWORD.read().unwrap().iter() {
output += &format!("{}: ", key).to_string();
output += "( ";
for i in pivot_keyword.fields.iter() {
output += &format!("%{}% ", i).to_string();
}
output += "):";
output += "\n";
for i in pivot_keyword.keywords.iter() {
output += &format!("{}\n", i).to_string();
}
output += "\n";
}
print!("{}", output);
}
}
}
#[cfg(not(target_os = "windows"))]
@@ -368,7 +456,7 @@ impl App {
pb.inc();
}
detection.add_aggcondition_msges(&self.rt);
if !*STATISTICS_FLAG {
if !*STATISTICS_FLAG && !*PIVOT_KEYWORD_LIST_FLAG {
after_fact();
}
}
@@ -750,6 +838,22 @@ impl App {
Ok("You currently have the latest rules.".to_string())
}
}
/// check architecture
fn is_matched_architecture_and_binary(&self) -> bool {
if cfg!(target_os = "windows") {
let is_processor_arch_32bit = env::var_os("PROCESSOR_ARCHITECTURE")
.unwrap_or_default()
.eq("x86");
// PROCESSOR_ARCHITEW6432は32bit環境には存在しないため、環境変数存在しなかった場合は32bit環境であると判断する
let not_wow_flag = env::var_os("PROCESSOR_ARCHITEW6432")
.unwrap_or_else(|| OsString::from("x86"))
.eq("x86");
return (cfg!(target_pointer_width = "64") && !is_processor_arch_32bit)
|| (cfg!(target_pointer_width = "32") && is_processor_arch_32bit && not_wow_flag);
}
true
}
}
#[cfg(test)]