Pivot Keyword List機能の追加 (#412)

* add get_pivot_keyword() func * change function name and call it's function * [WIP] support config file * compilete output * cargo fmt * [WIP] add test * add test * support -o option in pivot * add pivot mod * fix miss * pass test in pivot.rs * add comment * pass all test * add fast return * fix output * add test config file * review * rebase * cargo fmt * test pass * fix clippy in my commit * cargo fmt * little refactor * change file input logic and config format * [WIP] change output * [wip] change deta structure * change output & change data structure * pass test * add config * cargo fmt & clippy & rebase * fix cllipy * delete /rules/ in .gitignore * clean comment * clean * clean * fix rebase miss * fix rebase miss * fix clippy * file name output on -o to stdout * add pivot_keywords.txt to ./config * updated english * Documentation update * cargo fmt and clean * updated translate japanese * readme update * readme update Co-authored-by: DustInDark <nextsasasa@gmail.com> Co-authored-by: Tanaka Zakku <71482215+YamatoSecurity@users.noreply.github.com>
2022-04-05 21:17:23 +09:00
parent 545119bdfe
commit c8efa95447
13 changed files with 511 additions and 12 deletions
@@ -1,3 +1,5 @@
+use crate::detections::pivot::PivotKeyword;
+use crate::detections::pivot::PIVOT_KEYWORD;
 use crate::detections::print::AlertMessage;
 use crate::detections::utils;
 use chrono::{DateTime, Utc};
@@ -67,7 +69,7 @@ fn build_app<'a>() -> ArgMatches<'a> {

    let usages = "-d --directory=[DIRECTORY] 'Directory of multiple .evtx files.'
    -f --filepath=[FILEPATH] 'File path to one .evtx file.'
-    -r --rules=[RULEFILE/RULEDIRECTORY] 'Rule file or directory. (Default: ./rules)'
+    -r --rules=[RULEDIRECTORY/RULEFILE] 'Rule file or directory (default: ./rules)'
    -c --color 'Output with color. (Terminal needs to support True Color.)'
    -C --config=[RULECONFIGDIRECTORY] 'Rule config folder. (Default: ./rules/config)'
    -o --output=[CSV_TIMELINE] 'Save the timeline in CSV format. (Example: results.csv)'
@@ -86,6 +88,7 @@ fn build_app<'a>() -> ArgMatches<'a> {
    -s --statistics 'Prints statistics of event IDs.'
    -q --quiet 'Quiet mode. Do not display the launch banner.'
    -Q --quiet-errors 'Quiet errors mode. Do not save error logs.'
+    -p --pivot-keywords-list 'Create a list of pivot keywords.'
    --contributors 'Prints the list of contributors.'";
    App::new(&program)
        .about("Hayabusa: Aiming to be the world's greatest Windows event log analysis tool!")
@@ -268,6 +271,7 @@ impl Default for EventKeyAliasConfig {
 fn load_eventkey_alias(path: &str) -> EventKeyAliasConfig {
    let mut config = EventKeyAliasConfig::new();

+    // eventkey_aliasが読み込めなかったらエラーで終了とする。
    let read_result = utils::read_csv(path);
    if read_result.is_err() {
        AlertMessage::alert(
@@ -277,7 +281,7 @@ fn load_eventkey_alias(path: &str) -> EventKeyAliasConfig {
        .ok();
        return config;
    }
-    // eventkey_aliasが読み込めなかったらエラーで終了とする。
+
    read_result.unwrap().into_iter().for_each(|line| {
        if line.len() != 2 {
            return;
@@ -302,6 +306,40 @@ fn load_eventkey_alias(path: &str) -> EventKeyAliasConfig {
    config
 }

+///設定ファイルを読み込み、keyとfieldsのマップをPIVOT_KEYWORD大域変数にロードする。
+pub fn load_pivot_keywords(path: &str) {
+    let read_result = utils::read_txt(path);
+    if read_result.is_err() {
+        AlertMessage::alert(
+            &mut BufWriter::new(std::io::stderr().lock()),
+            read_result.as_ref().unwrap_err(),
+        )
+        .ok();
+    }
+
+    read_result.unwrap().into_iter().for_each(|line| {
+        let map: Vec<&str> = line.split('.').collect();
+        if map.len() != 2 {
+            return;
+        }
+
+        //存在しなければ、keyを作成
+        PIVOT_KEYWORD
+            .write()
+            .unwrap()
+            .entry(map[0].to_string())
+            .or_insert(PivotKeyword::new());
+
+        PIVOT_KEYWORD
+            .write()
+            .unwrap()
+            .get_mut(&map[0].to_string())
+            .unwrap()
+            .fields
+            .insert(map[1].to_string());
+    });
+}
+
 #[derive(Debug, Clone)]
 pub struct EventInfo {
    pub evttitle: String,
@@ -1,10 +1,12 @@
 extern crate csv;

 use crate::detections::configs;
+use crate::detections::pivot::insert_pivot_keyword;
 use crate::detections::print::AlertMessage;
 use crate::detections::print::DetectInfo;
 use crate::detections::print::ERROR_LOG_STACK;
 use crate::detections::print::MESSAGES;
+use crate::detections::print::PIVOT_KEYWORD_LIST_FLAG;
 use crate::detections::print::QUIET_ERRORS_FLAG;
 use crate::detections::print::STATISTICS_FLAG;
 use crate::detections::rule;
@@ -177,6 +179,12 @@ impl Detection {
            if !result {
                continue;
            }
+
+            if *PIVOT_KEYWORD_LIST_FLAG {
+                insert_pivot_keyword(&record_info.record);
+                continue;
+            }
+
            // aggregation conditionが存在しない場合はそのまま出力対応を行う
            if !agg_condition {
                Detection::insert_message(&rule, record_info);
@@ -1,5 +1,6 @@
 pub mod configs;
 pub mod detection;
+pub mod pivot;
 pub mod print;
 pub mod rule;
 pub mod utils;
@@ -0,0 +1,270 @@
+use hashbrown::HashMap;
+use hashbrown::HashSet;
+use lazy_static::lazy_static;
+use serde_json::Value;
+use std::sync::RwLock;
+
+use crate::detections::configs;
+use crate::detections::utils::get_serde_number_to_string;
+
+#[derive(Debug)]
+pub struct PivotKeyword {
+    pub keywords: HashSet<String>,
+    pub fields: HashSet<String>,
+}
+
+lazy_static! {
+    pub static ref PIVOT_KEYWORD: RwLock<HashMap<String, PivotKeyword>> =
+        RwLock::new(HashMap::new());
+}
+
+impl Default for PivotKeyword {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl PivotKeyword {
+    pub fn new() -> PivotKeyword {
+        PivotKeyword {
+            keywords: HashSet::new(),
+            fields: HashSet::new(),
+        }
+    }
+}
+
+///levelがlowより大きいレコードの場合、keywordがrecord内にみつかれば、
+///それをPIVOT_KEYWORD.keywordsに入れる。
+pub fn insert_pivot_keyword(event_record: &Value) {
+    //levelがlow異常なら続ける
+    let mut is_exist_event_key = false;
+    let mut tmp_event_record: &Value = event_record;
+    for s in ["Event", "System", "Level"] {
+        if let Some(record) = tmp_event_record.get(s) {
+            is_exist_event_key = true;
+            tmp_event_record = record;
+        }
+    }
+    if is_exist_event_key {
+        let hash_value = get_serde_number_to_string(tmp_event_record);
+
+        if hash_value.is_some() && hash_value.as_ref().unwrap() == "infomational"
+            || hash_value.as_ref().unwrap() == "undefined"
+            || hash_value.as_ref().unwrap() == "-"
+        {
+            return;
+        }
+    } else {
+        return;
+    }
+
+    for (_, pivot) in PIVOT_KEYWORD.write().unwrap().iter_mut() {
+        for field in &pivot.fields {
+            if let Some(array_str) = configs::EVENTKEY_ALIAS.get_event_key(&String::from(field)) {
+                let split: Vec<&str> = array_str.split('.').collect();
+                let mut is_exist_event_key = false;
+                let mut tmp_event_record: &Value = event_record;
+                for s in split {
+                    if let Some(record) = tmp_event_record.get(s) {
+                        is_exist_event_key = true;
+                        tmp_event_record = record;
+                    }
+                }
+                if is_exist_event_key {
+                    let hash_value = get_serde_number_to_string(tmp_event_record);
+
+                    if let Some(value) = hash_value {
+                        if value == "-" || value == "127.0.0.1" || value == "::1" {
+                            continue;
+                        }
+                        pivot.keywords.insert(value);
+                    };
+                }
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::detections::configs::load_pivot_keywords;
+    use crate::detections::pivot::insert_pivot_keyword;
+    use crate::detections::pivot::PIVOT_KEYWORD;
+    use serde_json;
+
+    //PIVOT_KEYWORDはグローバルなので、他の関数の影響も考慮する必要がある。
+    #[test]
+    fn insert_pivot_keyword_local_ip4() {
+        load_pivot_keywords("test_files/config/pivot_keywords.txt");
+        let record_json_str = r#"
+        {
+            "Event": { 
+                "System": {
+                    "Level": "high"
+                },
+                "EventData": {
+                    "IpAddress": "127.0.0.1"
+                }
+            }
+        }"#;
+        insert_pivot_keyword(&serde_json::from_str(record_json_str).unwrap());
+
+        assert!(!PIVOT_KEYWORD
+            .write()
+            .unwrap()
+            .get_mut("Ip Addresses")
+            .unwrap()
+            .keywords
+            .contains("127.0.0.1"));
+    }
+
+    #[test]
+    fn insert_pivot_keyword_ip4() {
+        load_pivot_keywords("test_files/config/pivot_keywords.txt");
+        let record_json_str = r#"
+        {
+            "Event": { 
+                "System": {
+                    "Level": "high"
+                },
+                "EventData": {
+                    "IpAddress": "10.0.0.1"
+                }
+            }
+        }"#;
+        insert_pivot_keyword(&serde_json::from_str(record_json_str).unwrap());
+
+        assert!(PIVOT_KEYWORD
+            .write()
+            .unwrap()
+            .get_mut("Ip Addresses")
+            .unwrap()
+            .keywords
+            .contains("10.0.0.1"));
+    }
+
+    #[test]
+    fn insert_pivot_keyword_ip_empty() {
+        load_pivot_keywords("test_files/config/pivot_keywords.txt");
+        let record_json_str = r#"
+        {
+            "Event": { 
+                "System": {
+                    "Level": "high"
+                },
+                "EventData": {
+                    "IpAddress": "-"
+                }
+            }
+        }"#;
+        insert_pivot_keyword(&serde_json::from_str(record_json_str).unwrap());
+
+        assert!(!PIVOT_KEYWORD
+            .write()
+            .unwrap()
+            .get_mut("Ip Addresses")
+            .unwrap()
+            .keywords
+            .contains("-"));
+    }
+
+    #[test]
+    fn insert_pivot_keyword_local_ip6() {
+        load_pivot_keywords("test_files/config/pivot_keywords.txt");
+        let record_json_str = r#"
+        {
+            "Event": { 
+                "System": {
+                    "Level": "high"
+                },
+                "EventData": {
+                    "IpAddress": "::1"
+                }
+            }
+        }"#;
+        insert_pivot_keyword(&serde_json::from_str(record_json_str).unwrap());
+
+        assert!(!PIVOT_KEYWORD
+            .write()
+            .unwrap()
+            .get_mut("Ip Addresses")
+            .unwrap()
+            .keywords
+            .contains("::1"));
+    }
+
+    #[test]
+    fn insert_pivot_keyword_level_infomational() {
+        load_pivot_keywords("test_files/config/pivot_keywords.txt");
+        let record_json_str = r#"
+        {
+            "Event": { 
+                "System": {
+                    "Level": "infomational"
+                },
+                "EventData": {
+                    "IpAddress": "10.0.0.2"
+                }
+            }
+        }"#;
+        insert_pivot_keyword(&serde_json::from_str(record_json_str).unwrap());
+
+        assert!(!PIVOT_KEYWORD
+            .write()
+            .unwrap()
+            .get_mut("Ip Addresses")
+            .unwrap()
+            .keywords
+            .contains("10.0.0.2"));
+    }
+
+    #[test]
+    fn insert_pivot_keyword_level_low() {
+        load_pivot_keywords("test_files/config/pivot_keywords.txt");
+        let record_json_str = r#"
+        {
+            "Event": { 
+                "System": {
+                    "Level": "low"
+                },
+                "EventData": {
+                    "IpAddress": "10.0.0.1"
+                }
+            }
+        }"#;
+        insert_pivot_keyword(&serde_json::from_str(record_json_str).unwrap());
+
+        assert!(PIVOT_KEYWORD
+            .write()
+            .unwrap()
+            .get_mut("Ip Addresses")
+            .unwrap()
+            .keywords
+            .contains("10.0.0.1"));
+    }
+
+    #[test]
+    fn insert_pivot_keyword_level_none() {
+        load_pivot_keywords("test_files/config/pivot_keywords.txt");
+        let record_json_str = r#"
+        {
+            "Event": { 
+                "System": {
+                    "Level": "-"
+                },
+                "EventData": {
+                    "IpAddress": "10.0.0.3"
+                }
+            }
+        }"#;
+        insert_pivot_keyword(&serde_json::from_str(record_json_str).unwrap());
+
+        assert!(!PIVOT_KEYWORD
+            .write()
+            .unwrap()
+            .get_mut("Ip Addresses")
+            .unwrap()
+            .keywords
+            .contains("10.0.0.3"));
+    }
+}
@@ -53,6 +53,11 @@ lazy_static! {
        .unwrap()
        .args
        .is_present("statistics");
+    pub static ref PIVOT_KEYWORD_LIST_FLAG: bool = configs::CONFIG
+        .read()
+        .unwrap()
+        .args
+        .is_present("pivot-keywords-list");
 }

 impl Default for Message {
@@ -87,7 +87,7 @@ pub fn read_csv(filename: &str) -> Result<Vec<Vec<String>>, String> {
        return Result::Err(e.to_string());
    }

-    let mut rdr = csv::Reader::from_reader(contents.as_bytes());
+    let mut rdr = csv::ReaderBuilder::new().from_reader(contents.as_bytes());
    rdr.records().for_each(|r| {
        if r.is_err() {
            return;