CSEC_PUBLIC/hayabusa
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Feature/re tuning and bugfix for regexes keyword (#293)

Browse Source 
* re-tuning

* not effective

* re-tuning

* set key

* fix bug and fix testcase.

* fmt
This commit is contained in:
James Takai / hach1yon
2021-12-18 11:13:51 +09:00
committed by GitHub
parent 17b6b97aa3
commit cbbcb4c068
9 changed files with 522 additions and 614 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/detections/detection.rs
View File

@@ -1,18 +1,18 @@
extern crate csv;
use crate::detections::rule::AggResult;
use serde_json::Value;
use std::collections::HashMap;
use tokio::{runtime::Runtime, spawn, task::JoinHandle};
use crate::detections::configs;
use crate::detections::print::AlertMessage;
use crate::detections::print::MESSAGES;
use crate::detections::rule;
use crate::detections::rule::AggResult;
use crate::detections::rule::RuleNode;
use crate::detections::utils::get_serde_number_to_string;
use crate::filter;
use crate::yaml::ParseYaml;
use hashbrown;
use serde_json::Value;
use std::collections::HashMap;
use tokio::{runtime::Runtime, spawn, task::JoinHandle};
use std::sync::Arc;
@@ -24,15 +24,12 @@ pub struct EvtxRecordInfo {
pub evtx_filepath: String, // イベントファイルのファイルパス　ログで出力するときに使う
pub record: Value, // 1レコード分のデータをJSON形式にシリアライズしたもの
pub data_string: String,
pub key_2_value: hashbrown::HashMap<String, String>,
}
impl EvtxRecordInfo {
pub fn new(evtx_filepath: String, record: Value, data_string: String) -> EvtxRecordInfo {
return EvtxRecordInfo {
evtx_filepath: evtx_filepath,
record: record,
data_string: data_string,
};
pub fn get_value(&self, key: &String) -> Option<&String> {
return self.key_2_value.get(key);
}
}
@@ -185,9 +182,8 @@ impl Detection {
// 複数のイベントレコードに対して、ルールを1個実行します。
fn execute_rule(mut rule: RuleNode, records: Arc<Vec<EvtxRecordInfo>>) -> RuleNode {
let records = &*records;
let agg_condition = rule.has_agg_condition();
for record_info in records {
for record_info in records.as_ref() {
let result = rule.select(&record_info.evtx_filepath, &record_info);
if !result {
continue;

2
src/detections/mod.rs
View File

@@ -1,5 +1,5 @@
pub mod configs;
pub mod detection;
pub mod print;
mod rule;
pub mod rule;
pub mod utils;

26
src/detections/rule/condition_parser.rs
View File

@@ -503,11 +503,10 @@ impl ConditionCompiler {
#[cfg(test)]
mod tests {
use yaml_rust::YamlLoader;
use crate::detections::detection::EvtxRecordInfo;
use crate::detections::rule::create_rule;
use crate::detections::rule::tests::parse_rule_from_str;
use crate::detections::{self, utils};
use yaml_rust::YamlLoader;
const SIMPLE_RECORD_STR: &str = r#"
{
@@ -537,11 +536,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(
rule_node.select(&"testpath".to_owned(), &recinfo),
expect_select
@@ -586,11 +582,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_rec) => {
@@ -633,11 +626,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_rec) => {

19
src/detections/rule/count.rs
View File

@@ -316,10 +316,11 @@ pub fn judge_timeframe(
#[cfg(test)]
mod tests {
use crate::detections::detection::EvtxRecordInfo;
use crate::detections;
use crate::detections::rule::create_rule;
use crate::detections::rule::AggResult;
use std::collections::HashMap;
use crate::detections::utils;
use hashbrown::HashMap;
use chrono::{TimeZone, Utc};
use yaml_rust::YamlLoader;
@@ -642,11 +643,8 @@ mod tests {
for record in target {
match serde_json::from_str(record) {
Ok(rec) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: rec,
data_string: record.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(rec, "testpath".to_owned(), &keys);
let _result = rule_node.select(&"testpath".to_string(), &recinfo);
}
Err(_rec) => {
@@ -735,11 +733,8 @@ mod tests {
for record_str in records_str {
match serde_json::from_str(record_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
let result = &rule_node.select(&"testpath".to_owned(), &recinfo);
assert_eq!(result, &true);
}

293
src/detections/rule/matchers.rs
View File

@@ -1,11 +1,15 @@
use regex::Regex;
use serde_json::Value;
use std::collections::VecDeque;
use yaml_rust::Yaml;
use crate::detections::{detection::EvtxRecordInfo, utils};
use mopa::mopafy;
use lazy_static::lazy_static;
lazy_static! {
pub static ref STR_DEFAULT: String = String::default();
}
// 末端ノードがEventLogの値を比較するロジックを表す。
// 正規条件のマッチや文字数制限など、比較ロジック毎にこのtraitを実装したクラスが存在する。
//
@@ -18,7 +22,7 @@ pub trait LeafMatcher: mopa::Any {
/// 引数に指定されたJSON形式のデータがマッチするかどうか判定する。
/// main.rsでWindows Event LogをJSON形式に変換していて、そのJSON形式のWindowsのイベントログデータがここには来る
/// 例えば正規表現でマッチするロジックなら、ここに正規表現でマッチさせる処理を書く。
fn is_match(&self, event_value: Option<&Value>, recinfo: &EvtxRecordInfo) -> bool;
fn is_match(&self, event_value: Option<&String>, recinfo: &EvtxRecordInfo) -> bool;
/// 初期化ロジックをここに記載します。
/// ルールファイルの書き方が間違っている等の原因により、正しくルールファイルからパースできない場合、戻り値のResult型でエラーを返してください。
@@ -60,11 +64,10 @@ impl LeafMatcher for MinlengthMatcher {
return Result::Ok(());
}
fn is_match(&self, event_value: Option<&Value>, _recinfo: &EvtxRecordInfo) -> bool {
return match event_value.unwrap_or(&Value::Null) {
Value::String(s) => s.len() as i64 >= self.min_len,
Value::Number(n) => n.to_string().len() as i64 >= self.min_len,
_ => false,
fn is_match(&self, event_value: Option<&String>, _recinfo: &EvtxRecordInfo) -> bool {
return match event_value {
Some(s) => s.len() as i64 >= self.min_len,
None => false,
};
}
}
@@ -118,12 +121,10 @@ impl LeafMatcher for RegexesFileMatcher {
return Result::Ok(());
}
fn is_match(&self, event_value: Option<&Value>, _recinfo: &EvtxRecordInfo) -> bool {
//TODO Wildcardの場合、CaseInsensitiveなので、ToLowerする。
return match event_value.unwrap_or(&Value::Null) {
Value::String(s) => !utils::check_regex(s, &self.regexes),
Value::Number(n) => !utils::check_regex(&n.to_string(), &self.regexes),
_ => false,
fn is_match(&self, event_value: Option<&String>, _recinfo: &EvtxRecordInfo) -> bool {
return match event_value {
Some(s) => utils::check_regex(s, &self.regexes),
None => false,
};
}
}
@@ -177,12 +178,10 @@ impl LeafMatcher for AllowlistFileMatcher {
return Result::Ok(());
}
fn is_match(&self, event_value: Option<&Value>, _recinfo: &EvtxRecordInfo) -> bool {
return match event_value.unwrap_or(&Value::Null) {
Value::String(s) => !utils::check_allowlist(s, &self.regexes),
Value::Number(n) => !utils::check_allowlist(&n.to_string(), &self.regexes),
Value::Bool(b) => !utils::check_allowlist(&b.to_string(), &self.regexes),
_ => true,
fn is_match(&self, event_value: Option<&String>, _recinfo: &EvtxRecordInfo) -> bool {
return match event_value {
Some(s) => !utils::check_allowlist(s, &self.regexes),
None => true,
};
}
}
@@ -320,50 +319,18 @@ impl LeafMatcher for DefaultMatcher {
return Result::Ok(());
}
fn is_match(&self, event_value: Option<&Value>, recinfo: &EvtxRecordInfo) -> bool {
// unwrap_orの引数に""ではなく" "を指定しているのは、
// event_valueが文字列じゃない場合にis_event_value_nullの値がfalseになるように、len() == 0とならない値を指定している。
let is_event_value_null = event_value.is_none()
|| event_value.unwrap().is_null()
|| event_value.unwrap().as_str().unwrap_or(" ").len() == 0;
fn is_match(&self, event_value: Option<&String>, _recinfo: &EvtxRecordInfo) -> bool {
// yamlにnullが設定されていた場合
// keylistが空(==JSONのgrep検索)の場合、無視する。
if !self.key_list.is_empty() && self.re.is_none() {
return is_event_value_null;
}
// JSON形式のEventLogデータをstringに変換するための前処理
// 以前のコードはstringに変換に変換する必ずto_string()がするような処理になっていた。
// そうすると、凄く遅くなるので、そうならないように回避
let mut b_str = String::default();
let mut n_str = String::default();
match event_value.unwrap_or(&Value::Null) {
Value::Bool(b) => b_str = b.to_string(),
Value::Number(n) => {
n_str = n.to_string();
}
_ => (),
};
// JSON形式のEventLogデータをstringに変換
let event_value_str: Option<&String> = if self.key_list.is_empty() {
Option::Some(&recinfo.data_string)
} else {
let value = match event_value.unwrap_or(&Value::Null) {
Value::Bool(_) => Option::Some(&b_str),
Value::String(s) => Option::Some(s),
Value::Number(_) => Option::Some(&n_str),
_ => Option::None,
};
value
};
if event_value_str.is_none() {
if self.key_list.is_empty() && self.re.is_none() {
return false;
}
// 変換したデータに対してパイプ処理を実行する。
let event_value_str = event_value_str.unwrap();
if event_value.is_none() {
return false;
}
let event_value_str = event_value.unwrap();
if self.key_list.is_empty() {
// この場合ただのgrep検索なので、ただ正規表現に一致するかどうか調べればよいだけ
return self.re.as_ref().unwrap().is_match(&event_value_str);
@@ -515,9 +482,8 @@ mod tests {
use super::super::selectionnodes::{
AndSelectionNode, LeafSelectionNode, OrSelectionNode, SelectionNode,
};
use crate::detections::detection::EvtxRecordInfo;
use crate::detections::rule::tests::parse_rule_from_str;
use serde_json::Value;
use crate::detections::{self, utils};
#[test]
fn test_rule_parse() {
@@ -739,11 +705,8 @@ mod tests {
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_) => {
@@ -772,11 +735,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_) => {
@@ -805,11 +765,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_) => {
@@ -839,11 +796,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_) => {
@@ -873,11 +827,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_) => {
@@ -906,11 +857,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_) => {
@@ -939,11 +887,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_) => {
@@ -973,11 +918,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_) => {
@@ -1007,11 +949,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_) => {
@@ -1041,11 +980,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_) => {
@@ -1075,11 +1011,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_) => {
@@ -1109,11 +1042,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_) => {
@@ -1142,11 +1072,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_) => {
@@ -1179,11 +1106,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_) => {
@@ -1216,11 +1140,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_) => {
@@ -1252,11 +1173,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_) => {
@@ -1297,11 +1215,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_rec) => {
@@ -1342,11 +1257,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_rec) => {
@@ -1387,11 +1299,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_rec) => {
@@ -1432,11 +1341,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_rec) => {
@@ -1477,11 +1383,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_rec) => {
@@ -1522,11 +1425,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_rec) => {
@@ -1555,11 +1455,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_) => {
@@ -1588,11 +1485,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_) => {
@@ -1621,11 +1515,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_) => {
@@ -1711,13 +1602,9 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(rec) => {
let rec: Value = rec;
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: rec,
data_string: record_json_str.to_string(),
};
Ok(record) => {
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_) => {
@@ -1746,12 +1633,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let rec: Value = record;
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: rec,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_) => {
@@ -1782,11 +1665,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_) => {
@@ -1817,11 +1697,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_) => {

116
src/detections/rule/mod.rs
View File

@@ -9,7 +9,7 @@ use yaml_rust::Yaml;
mod matchers;
mod selectionnodes;
use self::selectionnodes::SelectionNode;
use self::selectionnodes::{LeafSelectionNode, SelectionNode};
mod aggregation_parser;
use self::aggregation_parser::AggregationParseInfo;
@@ -93,6 +93,31 @@ impl RuleNode {
}
}
// RuleNodeのdetectionに定義されているキーの一覧を取得する。
pub fn get_detection_keys(node: &RuleNode) -> Vec<String> {
let mut ret = vec![];
let detection = &node.detection;
for key in detection.name_to_selection.keys() {
let selection = &detection.name_to_selection[key];
let desc = selection.get_descendants();
let keys = desc.iter().filter_map(|node| {
if !node.is::<LeafSelectionNode>() {
return Option::None;
}
let node = node.downcast_ref::<LeafSelectionNode>().unwrap();
let key = node.get_key();
if key.is_empty() {
return Option::None;
}
return Option::Some(key.to_string());
});
ret.extend(keys);
}
return ret;
}
/// Ruleファイルのdetectionを表すノード
struct DetectionNode {
pub name_to_selection: HashMap<String, Arc<Box<dyn SelectionNode>>>,
@@ -300,10 +325,9 @@ impl AggResult {
#[cfg(test)]
mod tests {
use crate::detections::{detection::EvtxRecordInfo, rule::create_rule};
use yaml_rust::YamlLoader;
use super::RuleNode;
use crate::detections::{self, rule::create_rule, utils};
use yaml_rust::YamlLoader;
pub fn parse_rule_from_str(rule_str: &str) -> RuleNode {
let rule_yaml = YamlLoader::load_from_str(rule_str);
@@ -335,11 +359,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_) => {
@@ -368,11 +389,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_) => {
@@ -401,11 +419,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_) => {
@@ -487,11 +502,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_) => {
@@ -549,11 +561,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_) => {
@@ -618,11 +627,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_) => {
@@ -665,11 +671,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_) => {
@@ -713,11 +716,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_) => {
@@ -780,11 +780,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_) => {
@@ -847,11 +844,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_) => {
@@ -896,11 +890,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_rec) => {
@@ -957,11 +948,8 @@ mod tests {
let _init = rule_node.init();
match serde_json::from_str(record_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
let result = rule_node.select(&"testpath".to_string(), &recinfo);
assert_eq!(rule_node.detection.aggregation_condition.is_some(), true);
assert_eq!(result, true);

51
src/detections/rule/selectionnodes.rs
View File

@@ -1,6 +1,5 @@
use crate::detections::{detection::EvtxRecordInfo, utils};
use mopa::mopafy;
use serde_json::Value;
use std::{sync::Arc, vec};
use yaml_rust::Yaml;
@@ -268,13 +267,13 @@ impl LeafSelectionNode {
}
/// JSON形式のEventJSONから値を取得する関数 aliasも考慮されている。
fn get_event_value<'a>(&self, event_value: &'a Value) -> Option<&'a Value> {
fn get_event_value<'a>(&self, record: &'a EvtxRecordInfo) -> Option<&'a String> {
// keyが指定されたいない場合は
if self.key_list.is_empty() {
return Option::Some(event_value);
return Option::Some(&record.data_string);
}
return utils::get_event_value(&self.get_key(), event_value);
return record.get_value(self.get_key());
}
/// matchers::LeafMatcherの一覧を取得する。
@@ -334,7 +333,7 @@ impl SelectionNode for LeafSelectionNode {
.matcher
.as_ref()
.unwrap()
.is_match(Option::Some(eventdata_data), event_record);
.is_match(event_record.get_value(self.get_key()), event_record);
}
// 配列の場合は配列の要素のどれか一つでもルールに合致すれば条件に一致したことにする。
if eventdata_data.is_array() {
@@ -343,11 +342,12 @@ impl SelectionNode for LeafSelectionNode {
.unwrap()
.iter()
.any(|ary_element| {
let aryelement_val = utils::value_to_string(ary_element);
return self
.matcher
.as_ref()
.unwrap()
.is_match(Option::Some(ary_element), event_record);
.is_match(aryelement_val.as_ref(), event_record);
});
} else {
return self
@@ -358,7 +358,7 @@ impl SelectionNode for LeafSelectionNode {
}
}
let event_value = self.get_event_value(&event_record.record);
let event_value = self.get_event_value(&event_record);
return self
.matcher
.as_ref()
@@ -407,7 +407,7 @@ impl SelectionNode for LeafSelectionNode {
#[cfg(test)]
mod tests {
use crate::detections::{detection::EvtxRecordInfo, rule::tests::parse_rule_from_str};
use crate::detections::{self, rule::tests::parse_rule_from_str, utils};
#[test]
fn test_detect_mutiple_regex_and() {
@@ -430,11 +430,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_) => {
@@ -466,11 +463,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_) => {
@@ -501,11 +495,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_) => {
@@ -536,11 +527,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), true);
}
Err(_) => {
@@ -571,11 +559,8 @@ mod tests {
let mut rule_node = parse_rule_from_str(rule_str);
match serde_json::from_str(record_json_str) {
Ok(record) => {
let recinfo = EvtxRecordInfo {
evtx_filepath: "testpath".to_owned(),
record: record,
data_string: record_json_str.to_string(),
};
let keys = detections::rule::get_detection_keys(&rule_node);
let recinfo = utils::create_rec_info(record, "testpath".to_owned(), &keys);
assert_eq!(rule_node.select(&"testpath".to_owned(), &recinfo), false);
}
Err(_) => {

48
src/detections/utils.rs
View File

@@ -16,6 +16,8 @@ use std::io::{BufRead, BufReader};
use std::str;
use std::string::String;
use super::detection::EvtxRecordInfo;
pub fn concat_selection_key(key_list: &Vec<String>) -> String {
return key_list
.iter()
@@ -47,6 +49,17 @@ pub fn check_allowlist(target: &str, regexes: &Vec<Regex>) -> bool {
return false;
}
pub fn value_to_string(value: &Value) -> Option<String> {
return match value {
Value::Null => Option::None,
Value::Bool(b) => Option::Some(b.to_string()),
Value::Number(n) => Option::Some(n.to_string()),
Value::String(s) => Option::Some(s.to_string()),
Value::Array(_) => Option::None,
Value::Object(_) => Option::None,
};
}
pub fn read_txt(filename: &str) -> Result<Vec<String>, String> {
let f = File::open(filename);
if f.is_err() {
@@ -184,6 +197,41 @@ pub fn create_tokio_runtime() -> Runtime {
.unwrap();
}
// EvtxRecordInfoを作成します。
pub fn create_rec_info(data: Value, path: String, keys: &Vec<String>) -> EvtxRecordInfo {
// EvtxRecordInfoを作る
let data_str = data.to_string();
let mut rec = EvtxRecordInfo {
evtx_filepath: path,
record: data,
data_string: data_str,
key_2_value: hashbrown::HashMap::new(),
};
// 高速化のための処理
// 例えば、Value型から"Event.System.EventID"の値を取得しようとすると、value["Event"]["System"]["EventID"]のように3回アクセスする必要がある。
// この処理を高速化するため、rec.key_2_valueというhashmapに"Event.System.EventID"というキーで値を設定しておく。
// これなら、"Event.System.EventID"というキーを1回指定するだけで値を取得できるようになるので、高速化されるはず。
// あと、serde_jsonのValueからvalue["Event"]みたいな感じで値を取得する処理がなんか遅いので、そういう意味でも早くなるかも
// それと、serde_jsonでは内部的に標準ライブラリのhashmapを使用しているが、hashbrownを使った方が早くなるらしい。
for key in keys {
let val = get_event_value(key, &rec.record);
if val.is_none() {
continue;
}
let val = value_to_string(val.unwrap());
if val.is_none() {
continue;
}
rec.key_2_value.insert(key.to_string(), val.unwrap());
}
return rec;
}
#[cfg(test)]
mod tests {
use crate::detections::utils;

559
src/main.rs
View File

@@ -4,9 +4,9 @@ extern crate serde_derive;
use chrono::Datelike;
use chrono::{DateTime, Local};
use evtx::{EvtxParser, ParserSettings};
use hayabusa::detections::detection;
use hayabusa::detections::detection::EvtxRecordInfo;
use hayabusa::detections::detection::{self, EvtxRecordInfo};
use hayabusa::detections::print::AlertMessage;
use hayabusa::detections::rule::{get_detection_keys, RuleNode};
use hayabusa::filter;
use hayabusa::omikuji::Omikuji;
use hayabusa::{afterfact::after_fact, detections::utils};
@@ -14,315 +14,344 @@ use hayabusa::{detections::configs, timeline::timeline::Timeline};
use hhmmss::Hhmmss;
use pbr::ProgressBar;
use serde_json::Value;
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::fmt::Display;
use std::sync::Arc;
use std::{
fs::{self, File},
path::PathBuf,
vec,
};
use tokio::runtime::Runtime;
use tokio::spawn;
use tokio::task::JoinHandle;
// 一度にtimelineやdetectionを実行する行数
const MAX_DETECT_RECORDS: usize = 5000;
fn main() {
let analysis_start_time: DateTime<Local> = Local::now();
if !configs::CONFIG.read().unwrap().args.is_present("q") {
output_logo();
println!("");
output_eggs(&format!(
"{:02}/{:02}",
&analysis_start_time.month().to_owned(),
&analysis_start_time.day().to_owned()
));
}
if configs::CONFIG.read().unwrap().args.args.len() == 0 {
println!(
"{}",
configs::CONFIG.read().unwrap().args.usage().to_string()
);
return;
}
if let Some(filepath) = configs::CONFIG.read().unwrap().args.value_of("filepath") {
if !filepath.ends_with(".evtx") {
AlertMessage::alert(
&mut std::io::stderr().lock(),
"--filepath only accepts .evtx files.".to_owned(),
)
.ok();
return;
}
analysis_files(vec![PathBuf::from(filepath)]);
} else if let Some(directory) = configs::CONFIG.read().unwrap().args.value_of("directory") {
let evtx_files = collect_evtxfiles(&directory);
if evtx_files.len() == 0 {
AlertMessage::alert(
&mut std::io::stderr().lock(),
"No .evtx files were found.".to_owned(),
)
.ok();
return;
}
analysis_files(evtx_files);
} else if configs::CONFIG
.read()
.unwrap()
.args
.is_present("contributors")
{
print_contributors();
return;
}
let analysis_end_time: DateTime<Local> = Local::now();
let analysis_duration = analysis_end_time.signed_duration_since(analysis_start_time);
println!("Elapsed Time: {}", &analysis_duration.hhmmssxxx());
println!("");
let mut app = App::new();
app.exec();
app.rt.shutdown_background();
}
fn collect_evtxfiles(dirpath: &str) -> Vec<PathBuf> {
let entries = fs::read_dir(dirpath);
if entries.is_err() {
let stderr = std::io::stderr();
let mut stderr = stderr.lock();
AlertMessage::alert(&mut stderr, format!("{}", entries.unwrap_err())).ok();
return vec![];
pub struct App {
rt: Runtime,
rule_keys: Vec<String>,
}
impl App {
pub fn new() -> App {
return App {
rt: utils::create_tokio_runtime(),
rule_keys: Vec::new(),
};
}
let mut ret = vec![];
for e in entries.unwrap() {
if e.is_err() {
continue;
fn exec(&mut self) {
let analysis_start_time: DateTime<Local> = Local::now();
if !configs::CONFIG.read().unwrap().args.is_present("q") {
self.output_logo();
println!("");
self.output_eggs(&format!(
"{:02}/{:02}",
&analysis_start_time.month().to_owned(),
&analysis_start_time.day().to_owned()
));
}
if configs::CONFIG.read().unwrap().args.args.len() == 0 {
println!(
"{}",
configs::CONFIG.read().unwrap().args.usage().to_string()
);
return;
}
if let Some(filepath) = configs::CONFIG.read().unwrap().args.value_of("filepath") {
if !filepath.ends_with(".evtx") {
AlertMessage::alert(
&mut std::io::stderr().lock(),
"--filepath only accepts .evtx files.".to_owned(),
)
.ok();
return;
}
self.analysis_files(vec![PathBuf::from(filepath)]);
} else if let Some(directory) = configs::CONFIG.read().unwrap().args.value_of("directory") {
let evtx_files = self.collect_evtxfiles(&directory);
if evtx_files.len() == 0 {
AlertMessage::alert(
&mut std::io::stderr().lock(),
"No .evtx files were found.".to_owned(),
)
.ok();
return;
}
self.analysis_files(evtx_files);
} else if configs::CONFIG
.read()
.unwrap()
.args
.is_present("contributors")
{
self.print_contributors();
return;
}
let analysis_end_time: DateTime<Local> = Local::now();
let analysis_duration = analysis_end_time.signed_duration_since(analysis_start_time);
println!("Elapsed Time: {}", &analysis_duration.hhmmssxxx());
println!("");
}
fn collect_evtxfiles(&self, dirpath: &str) -> Vec<PathBuf> {
let entries = fs::read_dir(dirpath);
if entries.is_err() {
let stderr = std::io::stderr();
let mut stderr = stderr.lock();
AlertMessage::alert(&mut stderr, format!("{}", entries.unwrap_err())).ok();
return vec![];
}
let path = e.unwrap().path();
if path.is_dir() {
path.to_str().and_then(|path_str| {
let subdir_ret = collect_evtxfiles(path_str);
ret.extend(subdir_ret);
return Option::Some(());
});
} else {
let path_str = path.to_str().unwrap_or("");
if path_str.ends_with(".evtx") {
ret.push(path);
let mut ret = vec![];
for e in entries.unwrap() {
if e.is_err() {
continue;
}
let path = e.unwrap().path();
if path.is_dir() {
path.to_str().and_then(|path_str| {
let subdir_ret = self.collect_evtxfiles(path_str);
ret.extend(subdir_ret);
return Option::Some(());
});
} else {
let path_str = path.to_str().unwrap_or("");
if path_str.ends_with(".evtx") {
ret.push(path);
}
}
}
return ret;
}
fn print_contributors(&self) {
match fs::read_to_string("./contributors.txt") {
Ok(contents) => println!("{}", contents),
Err(err) => {
AlertMessage::alert(&mut std::io::stderr().lock(), format!("{}", err)).ok();
}
}
}
return ret;
}
fn analysis_files(&mut self, evtx_files: Vec<PathBuf>) {
let level = configs::CONFIG
.read()
.unwrap()
.args
.value_of("min-level")
.unwrap_or("informational")
.to_uppercase();
println!("Analyzing event files: {:?}", evtx_files.len());
fn print_contributors() {
match fs::read_to_string("./contributors.txt") {
Ok(contents) => println!("{}", contents),
Err(err) => {
AlertMessage::alert(&mut std::io::stderr().lock(), format!("{}", err)).ok();
let rule_files = detection::Detection::parse_rule_files(
level,
configs::CONFIG.read().unwrap().args.value_of("rules"),
&filter::exclude_ids(),
);
let mut pb = ProgressBar::new(evtx_files.len() as u64);
self.rule_keys = self.get_all_keys(&rule_files);
let mut detection = detection::Detection::new(rule_files);
for evtx_file in evtx_files {
if configs::CONFIG.read().unwrap().args.is_present("verbose") {
println!("Checking target evtx FilePath: {:?}", &evtx_file);
}
detection = self.analysis_file(evtx_file, detection);
pb.inc();
}
after_fact();
detection.print_unique_results();
}
}
fn analysis_files(evtx_files: Vec<PathBuf>) {
let level = configs::CONFIG
.read()
.unwrap()
.args
.value_of("min-level")
.unwrap_or("informational")
.to_uppercase();
println!("Analyzing event files: {:?}", evtx_files.len());
let rule_files = detection::Detection::parse_rule_files(
level,
configs::CONFIG.read().unwrap().args.value_of("rules"),
&filter::exclude_ids(),
);
let mut pb = ProgressBar::new(evtx_files.len() as u64);
let mut detection = detection::Detection::new(rule_files);
for evtx_file in evtx_files {
if configs::CONFIG.read().unwrap().args.is_present("verbose") {
println!("Checking target evtx FilePath: {:?}", &evtx_file);
// Windowsイベントログファイルを1ファイル分解析する。
fn analysis_file(
&self,
evtx_filepath: PathBuf,
mut detection: detection::Detection,
) -> detection::Detection {
let path = evtx_filepath.display();
let parser = self.evtx_to_jsons(evtx_filepath.clone());
if parser.is_none() {
return detection;
}
detection = analysis_file(evtx_file, detection);
pb.inc();
}
after_fact();
detection.print_unique_results();
}
// Windowsイベントログファイルを1ファイル分解析する。
fn analysis_file(
evtx_filepath: PathBuf,
mut detection: detection::Detection,
) -> detection::Detection {
let filepath_disp = evtx_filepath.display();
let parser = evtx_to_jsons(evtx_filepath.clone());
if parser.is_none() {
return detection;
}
let mut tl = Timeline::new();
let mut parser = parser.unwrap();
let mut records = parser.records_json_value();
let mut tl = Timeline::new();
let mut parser = parser.unwrap();
let mut records = parser.records_json_value();
let tokio_rt = utils::create_tokio_runtime();
loop {
let mut records_per_detect = vec![];
while records_per_detect.len() < MAX_DETECT_RECORDS {
// パースに失敗している場合、エラーメッセージを出力
let next_rec = records.next();
if next_rec.is_none() {
break;
}
let target_event_time = configs::TargetEventTime::new();
let record_result = next_rec.unwrap();
if record_result.is_err() {
let evtx_filepath = &path;
let errmsg = format!(
"Failed to parse event file. EventFile:{} Error:{}",
evtx_filepath,
record_result.unwrap_err()
);
AlertMessage::alert(&mut std::io::stderr().lock(), errmsg).ok();
continue;
}
loop {
let mut records_per_detect = vec![];
while records_per_detect.len() < MAX_DETECT_RECORDS {
// パースに失敗している場合、エラーメッセージを出力
let next_rec = records.next();
if next_rec.is_none() {
// target_eventids.txtでフィルタする。
let data = record_result.unwrap().data;
if self._is_target_event_id(&data) == false {
continue;
}
// EvtxRecordInfo構造体に変更
records_per_detect.push(data);
}
if records_per_detect.len() == 0 {
break;
}
let record_result = next_rec.unwrap();
if record_result.is_err() {
let evtx_filepath = &filepath_disp;
let errmsg = format!(
"Failed to parse event file. EventFile:{} Error:{}",
evtx_filepath,
record_result.unwrap_err()
);
AlertMessage::alert(&mut std::io::stderr().lock(), errmsg).ok();
continue;
let records_per_detect = self.rt.block_on(App::create_rec_infos(
records_per_detect,
&path,
self.rule_keys.clone(),
));
// // timeline機能の実行
tl.start(&records_per_detect);
// // ruleファイルの検知
detection = detection.start(&self.rt, records_per_detect);
}
detection.add_aggcondtion_msg();
tl.tm_stats_dsp_msg();
return detection;
}
async fn create_rec_infos(
records_per_detect: Vec<Value>,
path: &dyn Display,
rule_keys: Vec<String>,
) -> Vec<EvtxRecordInfo> {
let path = Arc::new(path.to_string());
let rule_keys = Arc::new(rule_keys);
let threads: Vec<JoinHandle<EvtxRecordInfo>> = records_per_detect
.into_iter()
.map(|rec| {
let arc_rule_keys = Arc::clone(&rule_keys);
let arc_path = Arc::clone(&path);
return spawn(async move {
let rec_info =
utils::create_rec_info(rec, arc_path.to_string(), &arc_rule_keys);
return rec_info;
});
})
.collect();
let mut ret = vec![];
for thread in threads.into_iter() {
ret.push(thread.await.unwrap());
}
return ret;
}
fn get_all_keys(&self, rules: &Vec<RuleNode>) -> Vec<String> {
let mut key_set = HashSet::new();
for rule in rules {
let keys = get_detection_keys(rule);
key_set.extend(keys);
}
let ret: Vec<String> = key_set.into_iter().collect();
return ret;
}
// target_eventids.txtの設定を元にフィルタする。
fn _is_target_event_id(&self, data: &Value) -> bool {
let eventid = utils::get_event_value(&utils::get_event_id_key(), data);
if eventid.is_none() {
return true;
}
return match eventid.unwrap() {
Value::String(s) => utils::is_target_event_id(s),
Value::Number(n) => utils::is_target_event_id(&n.to_string()),
_ => true, // レコードからEventIdが取得できない場合は、特にフィルタしない
};
}
fn evtx_to_jsons(&self, evtx_filepath: PathBuf) -> Option<EvtxParser<File>> {
match EvtxParser::from_path(evtx_filepath) {
Ok(evtx_parser) => {
// parserのデフォルト設定を変更
let mut parse_config = ParserSettings::default();
parse_config = parse_config.separate_json_attributes(true); // XMLのattributeをJSONに変換する時のルールを設定
parse_config = parse_config.num_threads(0); // 設定しないと遅かったので、設定しておく。
let evtx_parser = evtx_parser.with_configuration(parse_config);
return Option::Some(evtx_parser);
}
// target_eventids.txtでフィルタする。
let data = record_result.unwrap().data;
if _is_target_event_id(&data) == false {
continue;
Err(e) => {
eprintln!("{}", e);
return Option::None;
}
}
}
let eventtime = utils::get_event_value(&utils::get_event_time(), &data);
if eventtime.is_some() {
let time = utils::str_time_to_datetime(eventtime.unwrap().as_str().unwrap_or(""));
if !target_event_time.is_target(&time) {
continue;
}
fn _output_with_omikuji(&self, omikuji: Omikuji) {
let fp = &format!("art/omikuji/{}", omikuji);
let content = fs::read_to_string(fp).unwrap();
println!("{}", content);
}
/// output logo
fn output_logo(&self) {
let fp = &format!("art/logo.txt");
let content = fs::read_to_string(fp).unwrap_or("".to_owned());
println!("{}", content);
}
/// output easter egg arts
fn output_eggs(&self, exec_datestr: &str) {
let mut eggs: HashMap<&str, &str> = HashMap::new();
eggs.insert("01/01", "art/happynewyear.txt");
eggs.insert("02/22", "art/ninja.txt");
eggs.insert("08/08", "art/takoyaki.txt");
eggs.insert("12/25", "art/christmas.txt");
match eggs.get(exec_datestr) {
None => {}
Some(path) => {
let content = fs::read_to_string(path).unwrap_or("".to_owned());
println!("{}", content);
}
// EvtxRecordInfo構造体に変更
records_per_detect.push(_create_rec_info(data, &filepath_disp));
}
if records_per_detect.len() == 0 {
break;
}
// timeline機能の実行
tl.start(&records_per_detect);
// ruleファイルの検知
detection = detection.start(&tokio_rt, records_per_detect);
}
tokio_rt.shutdown_background();
detection.add_aggcondtion_msg();
tl.tm_stats_dsp_msg();
return detection;
}
// target_eventids.txtの設定を元にフィルタする。
fn _is_target_event_id(data: &Value) -> bool {
let eventid = utils::get_event_value(&utils::get_event_id_key(), data);
if eventid.is_none() {
return true;
}
return match eventid.unwrap() {
Value::String(s) => utils::is_target_event_id(s),
Value::Number(n) => utils::is_target_event_id(&n.to_string()),
_ => true, // レコードからEventIdが取得できない場合は、特にフィルタしない
};
}
// EvtxRecordInfoを作成します。
fn _create_rec_info(mut data: Value, path: &dyn Display) -> EvtxRecordInfo {
// 高速化のための処理
// RuleNodeでワイルドカードや正規表現のマッチング処理をする際には、
// Value(JSON)がstring型以外の場合はstringに変換して比較している。
// RuleNodeでマッチングする毎にstring変換していると、
// 1回の処理はそこまででもないが相当回数呼び出されれるとボトルネックになりうる。
// なので、よく使われるstring型ではない値を事前に変換しておくことで、
// string変換する回数を減らせる。
// 本当はやりたくないが...
match &data["Event"]["System"]["EventID"] {
Value::Number(n) => data["Event"]["System"]["EventID"] = Value::String(n.to_string()),
_ => (),
};
match &data["Event"]["EventData"]["LogonType"] {
Value::Number(n) => data["Event"]["EventData"]["LogonType"] = Value::String(n.to_string()),
_ => (),
}
match &data["Event"]["EventData"]["DestinationPort"] {
Value::Number(n) => {
data["Event"]["EventData"]["DestinationPort"] = Value::String(n.to_string())
}
_ => (),
}
// EvtxRecordInfoを作る
let data_str = data.to_string();
return EvtxRecordInfo::new(path.to_string(), data, data_str);
}
fn evtx_to_jsons(evtx_filepath: PathBuf) -> Option<EvtxParser<File>> {
match EvtxParser::from_path(evtx_filepath) {
Ok(evtx_parser) => {
// parserのデフォルト設定を変更
let mut parse_config = ParserSettings::default();
parse_config = parse_config.separate_json_attributes(true); // XMLのattributeをJSONに変換する時のルールを設定
parse_config = parse_config.num_threads(utils::get_thread_num()); // 設定しないと遅かったので、設定しておく。
let evtx_parser = evtx_parser.with_configuration(parse_config);
return Option::Some(evtx_parser);
}
Err(e) => {
eprintln!("{}", e);
return Option::None;
}
}
}
fn _output_with_omikuji(omikuji: Omikuji) {
let fp = &format!("art/omikuji/{}", omikuji);
let content = fs::read_to_string(fp).unwrap();
println!("{}", content);
}
/// output logo
fn output_logo() {
let fp = &format!("art/logo.txt");
let content = fs::read_to_string(fp).unwrap_or("".to_owned());
println!("{}", content);
}
/// output easter egg arts
fn output_eggs(exec_datestr: &str) {
let mut eggs: HashMap<&str, &str> = HashMap::new();
eggs.insert("01/01", "art/happynewyear.txt");
eggs.insert("02/22", "art/ninja.txt");
eggs.insert("08/08", "art/takoyaki.txt");
eggs.insert("12/25", "art/christmas.txt");
match eggs.get(exec_datestr) {
None => {}
Some(path) => {
let content = fs::read_to_string(path).unwrap_or("".to_owned());
println!("{}", content);
}
}
}
#[cfg(test)]
mod tests {
use crate::collect_evtxfiles;
use crate::App;
#[test]
fn test_collect_evtxfiles() {
let files = collect_evtxfiles("test_files/evtx");
let app = App::new();
let files = app.collect_evtxfiles("test_files/evtx");
assert_eq!(3, files.len());
files.iter().for_each(|file| {