Feature/#216 output allfields csvnewcolumn (#469)

* refactoring

* refactoring

* under constructing

* underconstructing

* under construction

* underconstructing

* fix existing testcase

* finish implement

* fmt

* add option

* change name

* fix control code bug

* fix disp

* change format and fix testcase

* fix help

src/afterfact.rs

@@ -21,9 +21,11 @@ pub struct CsvFormat<'a> {
     computer: &'a str,
     event_i_d: &'a str,
     level: &'a str,
+    mitre_attack: &'a str,
     rule_title: &'a str,
     details: &'a str,
-    mitre_attack: &'a str,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    record_information: Option<&'a str>,
     rule_path: &'a str,
     file_path: &'a str,
 }
@@ -37,6 +39,8 @@ pub struct DisplayFormat<'a> {
     pub level: &'a str,
     pub rule_title: &'a str,
     pub details: &'a str,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub record_information: Option<&'a str>,
 }
 
 /// level_color.txtファイルを読み込み対応する文字色のマッピングを返却する関数
@@ -139,88 +143,44 @@ fn emit_csv<W: std::io::Write>(
     for (time, detect_infos) in messages.iter() {
         for detect_info in detect_infos {
             if displayflag {
-                if color_map.is_some() {
-                    let output_color =
-                        _get_output_color(color_map.as_ref().unwrap(), &detect_info.level);
-                    wtr.serialize(DisplayFormat {
-                        timestamp: &format!(
-                            "{} ",
-                            &format_time(time).truecolor(
-                                output_color[0],
-                                output_color[1],
-                                output_color[2]
-                            )
-                        ),
-                        level: &format!(
-                            " {} ",
-                            &detect_info.level.truecolor(
-                                output_color[0],
-                                output_color[1],
-                                output_color[2]
-                            )
-                        ),
-                        computer: &format!(
-                            " {} ",
-                            &detect_info.computername.truecolor(
-                                output_color[0],
-                                output_color[1],
-                                output_color[2]
-                            )
-                        ),
-                        event_i_d: &format!(
-                            " {} ",
-                            &detect_info.eventid.truecolor(
-                                output_color[0],
-                                output_color[1],
-                                output_color[2]
-                            )
-                        ),
-                        rule_title: &format!(
-                            " {} ",
-                            &detect_info.alert.truecolor(
-                                output_color[0],
-                                output_color[1],
-                                output_color[2]
-                            )
-                        ),
-                        details: &format!(
-                            " {}",
-                            &detect_info.detail.truecolor(
-                                output_color[0],
-                                output_color[1],
-                                output_color[2]
-                            )
-                        ),
-                    })?;
-                } else {
-                    wtr.serialize(DisplayFormat {
-                        timestamp: &format!("{} ", &format_time(time)),
-                        level: &format!(" {} ", &detect_info.level),
-                        computer: &format!(" {} ", &detect_info.computername),
-                        event_i_d: &format!(" {} ", &detect_info.eventid),
-                        rule_title: &format!(" {} ", &detect_info.alert),
-                        details: &format!(
-                            " {}",
-                            &detect_info
+                let colors = color_map
+                    .as_ref()
+                    .map(|cl_mp| _get_output_color(cl_mp, &detect_info.level));
+                let colors = colors.as_ref();
+
+                let recinfo = detect_info
+                    .record_information
+                    .as_ref()
+                    .map(|recinfo| _format_cell(recinfo, ColPos::Last, colors));
+                let details = detect_info
                     .detail
                     .chars()
                     .filter(|&c| !c.is_control())
-                                .collect::<String>()
-                        ),
-                    })?;
-                }
+                    .collect::<String>();
+
+                let dispformat = DisplayFormat {
+                    timestamp: &_format_cell(&format_time(time), ColPos::First, colors),
+                    level: &_format_cell(&detect_info.level, ColPos::Other, colors),
+                    computer: &_format_cell(&detect_info.computername, ColPos::Other, colors),
+                    event_i_d: &_format_cell(&detect_info.eventid, ColPos::Other, colors),
+                    rule_title: &_format_cell(&detect_info.alert, ColPos::Other, colors),
+                    details: &_format_cell(&details, ColPos::Other, colors),
+                    record_information: recinfo.as_deref(),
+                };
+                wtr.serialize(dispformat)?;
             } else {
                 // csv出力時フォーマット
                 wtr.serialize(CsvFormat {
                     timestamp: &format_time(time),
-                    file_path: &detect_info.filepath,
-                    rule_path: &detect_info.rulepath,
                     level: &detect_info.level,
                     computer: &detect_info.computername,
                     event_i_d: &detect_info.eventid,
+                    mitre_attack: &detect_info.tag_info,
                     rule_title: &detect_info.alert,
                     details: &detect_info.detail,
-                    mitre_attack: &detect_info.tag_info,
+                    record_information: detect_info.record_information.as_deref(),
+                    file_path: &detect_info.filepath,
+                    rule_path: &detect_info.rulepath,
                 })?;
             }
             let level_suffix = *configs::LEVELMAP
@@ -252,6 +212,29 @@ fn emit_csv<W: std::io::Write>(
     Ok(())
 }
 
+enum ColPos {
+    First, // 先頭
+    Last,  // 最後
+    Other, // それ以外
+}
+
+fn _format_cellpos(column: ColPos, colval: &str) -> String {
+    return match column {
+        ColPos::First => format!("{} ", colval),
+        ColPos::Last => format!(" {}", colval),
+        ColPos::Other => format!(" {} ", colval),
+    };
+}
+
+fn _format_cell(word: &str, column: ColPos, output_color: Option<&Vec<u8>>) -> String {
+    if let Some(color) = output_color {
+        let colval = format!("{}", word.truecolor(color[0], color[1], color[2]));
+        _format_cellpos(column, &colval)
+    } else {
+        _format_cellpos(column, word)
+    }
+}
+
 /// 与えられたユニークな検知数と全体の検知数の情報(レベル別と総計)を元に結果文を標準出力に表示する関数
 fn _print_unique_results(
     mut counts_by_level: Vec<u128>,
@@ -358,6 +341,7 @@ mod tests {
         let test_eventid = "1111";
         let output = "pokepoke";
         let test_attack = "execution/txxxx.yyy";
+        let test_recinfo = "record_infoinfo11";
         {
             let mut messages = print::MESSAGES.lock().unwrap();
             messages.clear();
@@ -388,6 +372,7 @@ mod tests {
                     alert: test_title.to_string(),
                     detail: String::default(),
                     tag_info: test_attack.to_string(),
+                    record_information: Option::Some(test_recinfo.to_string()),
                 },
             );
         }
@@ -396,7 +381,7 @@ mod tests {
             .unwrap();
         let expect_tz = expect_time.with_timezone(&Local);
         let expect =
-            "Timestamp,Computer,EventID,Level,RuleTitle,Details,MitreAttack,RulePath,FilePath\n"
+            "Timestamp,Computer,EventID,Level,MitreAttack,RuleTitle,Details,RecordInformation,RulePath,FilePath\n"
                 .to_string()
                 + &expect_tz
                     .clone()
@@ -409,11 +394,13 @@ mod tests {
                 + ","
                 + test_level
                 + ","
+                + test_attack
+                + ","
                 + test_title
                 + ","
                 + output
                 + ","
-                + test_attack
+                + test_recinfo
                 + ","
                 + testrulepath
                 + ","
@@ -470,6 +457,7 @@ mod tests {
                     alert: test_title.to_string(),
                     detail: String::default(),
                     tag_info: test_attack.to_string(),
+                    record_information: Option::Some(String::default()),
                 },
             );
             messages.debug();
@@ -478,7 +466,8 @@ mod tests {
             .datetime_from_str("1996-02-27T01:05:01Z", "%Y-%m-%dT%H:%M:%SZ")
             .unwrap();
         let expect_tz = expect_time.with_timezone(&Local);
-        let expect_header = "Timestamp|Computer|EventID|Level|RuleTitle|Details\n";
+        let expect_header =
+            "Timestamp|Computer|EventID|Level|RuleTitle|Details|RecordInformation\n";
         let expect_colored = expect_header.to_string()
             + &get_white_color_string(
                 &expect_tz
@@ -496,6 +485,8 @@ mod tests {
             + &get_white_color_string(test_title)
             + " | "
             + &get_white_color_string(output)
+            + " | "
+            + &get_white_color_string("")
             + "\n";
         let expect_nocoloed = expect_header.to_string()
             + &expect_tz
@@ -512,6 +503,8 @@ mod tests {
             + test_title
             + " | "
             + output
+            + " | "
+            + ""
             + "\n";
 
         let mut file: Box<dyn io::Write> =

src/detections/configs.rs

@@ -69,6 +69,7 @@ fn build_app<'a>() -> ArgMatches<'a> {
 
     let usages = "-d --directory=[DIRECTORY] 'Directory of multiple .evtx files.'
     -f --filepath=[FILEPATH] 'File path to one .evtx file.'
+    -F --full-data 'Print all field information.'
     -r --rules=[RULEDIRECTORY/RULEFILE] 'Rule file or directory (default: ./rules)'
     -c --color 'Output with color. (Terminal needs to support True Color.)'
     -C --config=[RULECONFIGDIRECTORY] 'Rule config folder. (Default: ./rules/config)'

src/detections/detection.rs

@@ -32,6 +32,7 @@ pub struct EvtxRecordInfo {
     pub record: Value,         // 1レコード分のデータをJSON形式にシリアライズしたもの
     pub data_string: String,
     pub key_2_value: hashbrown::HashMap<String, String>,
+    pub record_information: Option<String>,
 }
 
 impl EvtxRecordInfo {
@@ -204,24 +205,29 @@ impl Detection {
             .filter_map(|info| TAGS_CONFIG.get(info.as_str().unwrap_or(&String::default())))
             .map(|str| str.to_owned())
             .collect();
-        MESSAGES.lock().unwrap().insert(
-            &record_info.record,
-            rule.yaml["details"].as_str().unwrap_or("").to_string(),
-            DetectInfo {
+
+        let recinfo = record_info
+            .record_information
+            .as_ref()
+            .map(|recinfo| recinfo.to_string());
+        let detect_info = DetectInfo {
             filepath: record_info.evtx_filepath.to_string(),
             rulepath: rule.rulepath.to_string(),
             level: rule.yaml["level"].as_str().unwrap_or("-").to_string(),
             computername: record_info.record["Event"]["System"]["Computer"]
                 .to_string()
                 .replace('\"', ""),
-                eventid: get_serde_number_to_string(
-                    &record_info.record["Event"]["System"]["EventID"],
-                )
+            eventid: get_serde_number_to_string(&record_info.record["Event"]["System"]["EventID"])
                 .unwrap_or_else(|| "-".to_owned()),
             alert: rule.yaml["title"].as_str().unwrap_or("").to_string(),
             detail: String::default(),
             tag_info: tag_info.join(" | "),
-            },
+            record_information: recinfo,
+        };
+        MESSAGES.lock().unwrap().insert(
+            &record_info.record,
+            rule.yaml["details"].as_str().unwrap_or("").to_string(),
+            detect_info,
         );
     }
 
@@ -234,8 +240,12 @@ impl Detection {
             .map(|info| info.as_str().unwrap_or("").replace("attack.", ""))
             .collect();
         let output = Detection::create_count_output(rule, &agg_result);
-        MESSAGES.lock().unwrap().insert_message(
-            DetectInfo {
+        let rec_info = if configs::CONFIG.read().unwrap().args.is_present("full-data") {
+            Option::Some(String::default())
+        } else {
+            Option::None
+        };
+        let detect_info = DetectInfo {
             filepath: "-".to_owned(),
             rulepath: rule.rulepath.to_owned(),
             level: rule.yaml["level"].as_str().unwrap_or("").to_owned(),
@@ -243,10 +253,14 @@ impl Detection {
             eventid: "-".to_owned(),
             alert: rule.yaml["title"].as_str().unwrap_or("").to_owned(),
             detail: output,
+            record_information: rec_info,
             tag_info: tag_info.join(" : "),
-            },
-            agg_result.start_timedate,
-        )
+        };
+
+        MESSAGES
+            .lock()
+            .unwrap()
+            .insert_message(detect_info, agg_result.start_timedate)
     }
 
     ///aggregation conditionのcount部分の検知出力文の文字列を返す関数
@@ -509,4 +523,7 @@ mod tests {
             expected_output
         );
     }
+
+    #[test]
+    fn test_create_fields_value() {}
 }

src/detections/print.rs

@@ -31,6 +31,7 @@ pub struct DetectInfo {
     pub alert: String,
     pub detail: String,
     pub tag_info: String,
+    pub record_information: Option<String>,
 }
 
 pub struct AlertMessage {}
@@ -285,6 +286,7 @@ mod tests {
                 alert: "test1".to_string(),
                 detail: String::default(),
                 tag_info: "txxx.001".to_string(),
+                record_information: Option::Some("record_information1".to_string()),
             },
         );
 
@@ -315,6 +317,7 @@ mod tests {
                 alert: "test2".to_string(),
                 detail: String::default(),
                 tag_info: "txxx.002".to_string(),
+                record_information: Option::Some("record_information2".to_string()),
             },
         );
 
@@ -345,6 +348,7 @@ mod tests {
                 alert: "test3".to_string(),
                 detail: String::default(),
                 tag_info: "txxx.003".to_string(),
+                record_information: Option::Some("record_information3".to_string()),
             },
         );
 
@@ -370,12 +374,13 @@ mod tests {
                 alert: "test4".to_string(),
                 detail: String::default(),
                 tag_info: "txxx.004".to_string(),
+                record_information: Option::Some("record_information4".to_string()),
             },
         );
 
         let display = format!("{}", format_args!("{:?}", message));
         println!("display::::{}", display);
-        let expect = "Message { map: {1970-01-01T00:00:00Z: [DetectInfo { filepath: \"a\", rulepath: \"test_rule4\", level: \"medium\", computername: \"testcomputer4\", eventid: \"4\", alert: \"test4\", detail: \"CommandLine4: hoge\", tag_info: \"txxx.004\" }], 1996-02-27T01:05:01Z: [DetectInfo { filepath: \"a\", rulepath: \"test_rule\", level: \"high\", computername: \"testcomputer1\", eventid: \"1\", alert: \"test1\", detail: \"CommandLine1: hoge\", tag_info: \"txxx.001\" }, DetectInfo { filepath: \"a\", rulepath: \"test_rule2\", level: \"high\", computername: \"testcomputer2\", eventid: \"2\", alert: \"test2\", detail: \"CommandLine2: hoge\", tag_info: \"txxx.002\" }], 2000-01-21T09:06:01Z: [DetectInfo { filepath: \"a\", rulepath: \"test_rule3\", level: \"high\", computername: \"testcomputer3\", eventid: \"3\", alert: \"test3\", detail: \"CommandLine3: hoge\", tag_info: \"txxx.003\" }]} }";
+        let expect = "Message { map: {1970-01-01T00:00:00Z: [DetectInfo { filepath: \"a\", rulepath: \"test_rule4\", level: \"medium\", computername: \"testcomputer4\", eventid: \"4\", alert: \"test4\", detail: \"CommandLine4: hoge\", tag_info: \"txxx.004\", record_information: Some(\"record_information4\") }], 1996-02-27T01:05:01Z: [DetectInfo { filepath: \"a\", rulepath: \"test_rule\", level: \"high\", computername: \"testcomputer1\", eventid: \"1\", alert: \"test1\", detail: \"CommandLine1: hoge\", tag_info: \"txxx.001\", record_information: Some(\"record_information1\") }, DetectInfo { filepath: \"a\", rulepath: \"test_rule2\", level: \"high\", computername: \"testcomputer2\", eventid: \"2\", alert: \"test2\", detail: \"CommandLine2: hoge\", tag_info: \"txxx.002\", record_information: Some(\"record_information2\") }], 2000-01-21T09:06:01Z: [DetectInfo { filepath: \"a\", rulepath: \"test_rule3\", level: \"high\", computername: \"testcomputer3\", eventid: \"3\", alert: \"test3\", detail: \"CommandLine3: hoge\", tag_info: \"txxx.003\", record_information: Some(\"record_information3\") }]} }";
         assert_eq!(display, expect);
     }
 

src/detections/utils.rs

@@ -10,11 +10,13 @@ use tokio::runtime::Runtime;
 use chrono::{DateTime, TimeZone, Utc};
 use regex::Regex;
 use serde_json::Value;
+use std::cmp::Ordering;
 use std::fs::File;
 use std::io::prelude::*;
 use std::io::{BufRead, BufReader};
 use std::str;
 use std::string::String;
+use std::vec;
 
 use super::detection::EvtxRecordInfo;
 
@@ -199,15 +201,6 @@ pub fn create_tokio_runtime() -> Runtime {
 
 // EvtxRecordInfoを作成します。
 pub fn create_rec_info(data: Value, path: String, keys: &[String]) -> EvtxRecordInfo {
-    // EvtxRecordInfoを作る
-    let data_str = data.to_string();
-    let mut rec = EvtxRecordInfo {
-        evtx_filepath: path,
-        record: data,
-        data_string: data_str,
-        key_2_value: hashbrown::HashMap::new(),
-    };
-
     // 高速化のための処理
 
     // 例えば、Value型から"Event.System.EventID"の値を取得しようとすると、value["Event"]["System"]["EventID"]のように3回アクセスする必要がある。
@@ -215,8 +208,9 @@ pub fn create_rec_info(data: Value, path: String, keys: &[String]) -> EvtxRecord
     // これなら、"Event.System.EventID"というキーを1回指定するだけで値を取得できるようになるので、高速化されるはず。
     // あと、serde_jsonのValueからvalue["Event"]みたいな感じで値を取得する処理がなんか遅いので、そういう意味でも早くなるかも
     // それと、serde_jsonでは内部的に標準ライブラリのhashmapを使用しているが、hashbrownを使った方が早くなるらしい。
+    let mut key_2_values = hashbrown::HashMap::new();
     for key in keys {
-        let val = get_event_value(key, &rec.record);
+        let val = get_event_value(key, &data);
         if val.is_none() {
             continue;
         }
@@ -226,10 +220,110 @@ pub fn create_rec_info(data: Value, path: String, keys: &[String]) -> EvtxRecord
             continue;
         }
 
-        rec.key_2_value.insert(key.trim().to_string(), val.unwrap());
+        key_2_values.insert(key.to_string(), val.unwrap());
     }
 
-    rec
+    // EvtxRecordInfoを作る
+    let data_str = data.to_string();
+    let rec_info = if configs::CONFIG.read().unwrap().args.is_present("full-data") {
+        Option::Some(create_recordinfos(&data))
+    } else {
+        Option::None
+    };
+    EvtxRecordInfo {
+        evtx_filepath: path,
+        record: data,
+        data_string: data_str,
+        key_2_value: key_2_values,
+        record_information: rec_info,
+    }
+}
+
+/**
+ * CSVのrecord infoカラムに出力する文字列を作る
+ */
+fn create_recordinfos(record: &Value) -> String {
+    let mut output = vec![];
+    _collect_recordinfo(&mut vec![], "", record, &mut output);
+
+    // 同じレコードなら毎回同じ出力になるようにソートしておく
+    output.sort_by(|(left, left_data), (right, right_data)| {
+        let ord = left.cmp(right);
+        if ord == Ordering::Equal {
+            left_data.cmp(right_data)
+        } else {
+            ord
+        }
+    });
+
+    let summary: Vec<String> = output
+        .iter()
+        .map(|(key, value)| {
+            return format!("{}:{}", key, value);
+        })
+        .collect();
+
+    // 標準出力する時はセルがハイプ区切りになるので、パイプ区切りにしない
+    if configs::CONFIG.read().unwrap().args.is_present("output") {
+        summary.join(" | ")
+    } else {
+        summary.join(" ")
+    }
+}
+
+/**
+ * CSVのfieldsカラムに出力する要素を全て収集する
+ */
+fn _collect_recordinfo<'a>(
+    keys: &mut Vec<&'a str>,
+    parent_key: &'a str,
+    value: &'a Value,
+    output: &mut Vec<(String, String)>,
+) {
+    match value {
+        Value::Array(ary) => {
+            for sub_value in ary {
+                _collect_recordinfo(keys, parent_key, sub_value, output);
+            }
+        }
+        Value::Object(obj) => {
+            // lifetimeの関係でちょっと変な実装になっている
+            if !parent_key.is_empty() {
+                keys.push(parent_key);
+            }
+            for (key, value) in obj {
+                // 属性は出力しない
+                if key.ends_with("_attributes") {
+                    continue;
+                }
+                // Event.Systemは出力しない
+                if key.eq("System") && keys.get(0).unwrap_or(&"").eq(&"Event") {
+                    continue;
+                }
+
+                _collect_recordinfo(keys, key, value, output);
+            }
+            if !parent_key.is_empty() {
+                keys.pop();
+            }
+        }
+        Value::Null => (),
+        _ => {
+            // 一番子の要素の値しか収集しない
+            let strval = value_to_string(value);
+            if let Some(strval) = strval {
+                let strval = strval.trim().chars().fold(String::default(), |mut acc, c| {
+                    if c.is_control() || c.is_ascii_whitespace() {
+                        acc.push(' ');
+                    } else {
+                        acc.push(c);
+                    };
+                    acc
+                });
+                output.push((parent_key.to_string(), strval));
+            }
+        }
+    }
 }
 
 #[cfg(test)]
@@ -238,6 +332,66 @@ mod tests {
     use regex::Regex;
     use serde_json::Value;
 
+    #[test]
+    fn test_create_recordinfos() {
+        let record_json_str = r#"
+        {
+            "Event": {
+                "System": {"EventID": 4103, "Channel": "PowerShell", "Computer":"DESKTOP-ICHIICHI"},
+                "UserData": {"User": "u1", "AccessMask": "%%1369", "Process":"lsass.exe"},
+                "UserData_attributes": {"xmlns": "http://schemas.microsoft.com/win/2004/08/events/event"}
+            },
+            "Event_attributes": {"xmlns": "http://schemas.microsoft.com/win/2004/08/events/event"}
+        }"#;
+
+        match serde_json::from_str(record_json_str) {
+            Ok(record) => {
+                let ret = utils::create_recordinfos(&record);
+                // Systemは除外される/属性(_attributesも除外される)/key順に並ぶ
+                let expected = "AccessMask:%%1369 Process:lsass.exe User:u1".to_string();
+                assert_eq!(ret, expected);
+            }
+            Err(_) => {
+                panic!("Failed to parse json record.");
+            }
+        }
+    }
+
+    #[test]
+    fn test_create_recordinfos2() {
+        // EventDataの特殊ケース
+        let record_json_str = r#"
+        {
+            "Event": {
+                "System": {"EventID": 4103, "Channel": "PowerShell", "Computer":"DESKTOP-ICHIICHI"},
+                "EventData": {
+                    "Binary": "hogehoge",
+                    "Data":[
+                        "Data1",
+                        "DataData2",
+                        "",
+                        "DataDataData3"
+                    ]
+                },
+                "EventData_attributes": {"xmlns": "http://schemas.microsoft.com/win/2004/08/events/event"}
+            },
+            "Event_attributes": {"xmlns": "http://schemas.microsoft.com/win/2004/08/events/event"}
+        }"#;
+
+        match serde_json::from_str(record_json_str) {
+            Ok(record) => {
+                let ret = utils::create_recordinfos(&record);
+                // Systemは除外される/属性(_attributesも除外される)/key順に並ぶ
+                let expected = "Binary:hogehoge Data: Data:Data1 Data:DataData2 Data:DataDataData3"
+                    .to_string();
+                assert_eq!(ret, expected);
+            }
+            Err(_) => {
+                panic!("Failed to parse json record.");
+            }
+        }
+    }
+
     #[test]
     fn test_check_regex() {
         let regexes: Vec<Regex> =