Merge pull request #677 from Yamato-Security/675-condense-results-summary-with-tables

Condensed detection count output format in the results summary
2022-08-29 11:00:54 +09:00
parent d90f901696 5c129162d2
commit d0e227702b
6 changed files with 223 additions and 76 deletions
--- a/CHANGELOG-Japanese.md
+++ b/CHANGELOG-Japanese.md
@@ -10,6 +10,7 @@

 - 結果概要に各レベルで検知した上位5つのルールを表示するようにした。 (#667) (@hitenkoku)
 - 結果概要を出力しないようにするために `--no-summary` オプションを追加した。 (#672) (@hitenkoku)
+- 結果概要の表示を短縮させた。 (#675) (@hitenkoku)

 **バグ修正:**

--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@

 - Added top alerts to results summary. (#667) (@hitenkoku)
 - Added `--no-summary` option to not display the results summary. (#672) (@hitenkoku)
+- Made the results summary more compact. (#675) (@hitenkoku)

 **Bug Fixes:**

--- a/Cargo.lock
+++ b/Cargo.lock
@@ -254,6 +254,18 @@ dependencies = [
 "os_str_bytes",
 ]

+[[package]]
+name = "comfy-table"
+version = "6.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85914173c2f558d61613bfbbf1911f14e630895087a7ed2fafc0f5319e1536e7"
+dependencies = [
+ "crossterm",
+ "strum",
+ "strum_macros",
+ "unicode-width",
+]
+
 [[package]]
 name = "console"
 version = "0.15.1"
@@ -334,6 +346,31 @@ dependencies = [
 "once_cell",
 ]

+[[package]]
+name = "crossterm"
+version = "0.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67"
+dependencies = [
+ "bitflags",
+ "crossterm_winapi",
+ "libc",
+ "mio",
+ "parking_lot",
+ "signal-hook",
+ "signal-hook-mio",
+ "winapi",
+]
+
+[[package]]
+name = "crossterm_winapi"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "csv"
 version = "1.1.6"
@@ -554,7 +591,7 @@ dependencies = [
 "indoc",
 "jemallocator",
 "log",
- "quick-xml",
+ "quick-xml 0.23.0",
 "rayon",
 "rpmalloc",
 "serde",
@@ -713,6 +750,7 @@ dependencies = [
 "bytesize",
 "chrono",
 "clap 3.2.17",
+ "comfy-table",
 "crossbeam-utils",
 "csv",
 "dashmap",
@@ -735,7 +773,7 @@ dependencies = [
 "openssl",
 "pbr",
 "prettytable-rs",
- "quick-xml",
+ "quick-xml 0.24.0",
 "rand",
 "regex",
 "serde",
@@ -1067,9 +1105,9 @@ checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d"

 [[package]]
 name = "lock_api"
-version = "0.4.7"
+version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53"
+checksum = "9f80bf5aacaf25cbfc8210d1cfb718f2bf3b11c4c54e5afe36c236853a8ec390"
 dependencies = [
 "autocfg",
 "scopeguard",
@@ -1410,6 +1448,15 @@ name = "quick-xml"
 version = "0.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9279fbdacaad3baf559d8cabe0acc3d06e30ea14931af31af79578ac0946decc"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "quick-xml"
+version = "0.24.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "678404d55890514fa1c01fe98cf280b674db93944fdcb70310dd3be1d0d63be7"
 dependencies = [
 "memchr",
 "serde",
@@ -1626,18 +1673,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"

 [[package]]
 name = "serde"
-version = "1.0.143"
+version = "1.0.144"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53e8e5d5b70924f74ff5c6d64d9a5acd91422117c60f48c4e07855238a254553"
+checksum = "0f747710de3dcd43b88c9168773254e809d8ddbdf9653b84e2554ab219f17860"
 dependencies = [
 "serde_derive",
 ]

 [[package]]
 name = "serde_derive"
-version = "1.0.143"
+version = "1.0.144"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3d8e8de557aee63c26b85b947f5e59b690d0454c753f3adeb5cd7835ab88391"
+checksum = "94ed3a816fb1d101812f83e789f888322c34e291f894f19590dc310963e87a00"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -1646,9 +1693,9 @@ dependencies = [

 [[package]]
 name = "serde_json"
-version = "1.0.83"
+version = "1.0.85"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38dd04e3c8279e75b31ef29dbdceebfe5ad89f4d0937213c53f7d49d01b3d5a7"
+checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44"
 dependencies = [
 "itoa 1.0.3",
 "ryu",
@@ -1670,6 +1717,27 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012"

+[[package]]
+name = "signal-hook"
+version = "0.3.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d"
+dependencies = [
+ "libc",
+ "signal-hook-registry",
+]
+
+[[package]]
+name = "signal-hook-mio"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af"
+dependencies = [
+ "libc",
+ "mio",
+ "signal-hook",
+]
+
 [[package]]
 name = "signal-hook-registry"
 version = "1.4.0"
@@ -1713,9 +1781,9 @@ checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1"

 [[package]]
 name = "socket2"
-version = "0.4.4"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0"
+checksum = "10c98bba371b9b22a71a9414e420f92ddeb2369239af08200816169d5e2dd7aa"
 dependencies = [
 "libc",
 "winapi",
@@ -1797,6 +1865,25 @@ version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"

+[[package]]
+name = "strum"
+version = "0.24.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f"
+
+[[package]]
+name = "strum_macros"
+version = "0.24.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "rustversion",
+ "syn",
+]
+
 [[package]]
 name = "syn"
 version = "1.0.99"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -38,6 +38,7 @@ hyper = "0.14.*"
 lock_api = "0.4.*"
 crossbeam-utils = "0.8.*"
 num-format = "*"
+comfy-table = "6.*"

 [build-dependencies]
 static_vcruntime = "2.*"
--- a/screenshots/HayabusaResultsSummary.png
+++ b/screenshots/HayabusaResultsSummary.png
--- a/src/afterfact.rs
+++ b/src/afterfact.rs
@@ -7,12 +7,15 @@ use crate::detections::utils::{get_writable_color, write_color_buffer};
 use crate::options::profile::PROFILES;
 use bytesize::ByteSize;
 use chrono::{DateTime, Local, TimeZone, Utc};
+use comfy_table::modifiers::UTF8_ROUND_CORNERS;
+use comfy_table::presets::UTF8_FULL;
 use csv::QuoteStyle;
 use itertools::Itertools;
 use krapslog::{build_sparkline, build_time_markers};
 use lazy_static::lazy_static;
 use linked_hash_map::LinkedHashMap;

+use comfy_table::*;
 use hashbrown::{HashMap, HashSet};
 use num_format::{Locale, ToFormattedString};
 use std::cmp::min;
@@ -29,17 +32,22 @@ use termcolor::{BufferWriter, Color, ColorChoice, ColorSpec, WriteColor};
 use terminal_size::Width;

 lazy_static! {
-    pub static ref OUTPUT_COLOR: HashMap<String, Color> = set_output_color();
+    pub static ref OUTPUT_COLOR: HashMap<String, Colors> = set_output_color();
+}
+
+pub struct Colors {
+    pub output_color: termcolor::Color,
+    pub table_color: comfy_table::Color,
 }

 /// level_color.txtファイルを読み込み対応する文字色のマッピングを返却する関数
-pub fn set_output_color() -> HashMap<String, Color> {
+pub fn set_output_color() -> HashMap<String, Colors> {
    let read_result = utils::read_csv(
        utils::check_setting_path(&CURRENT_EXE_PATH.to_path_buf(), "config/level_color.txt")
            .to_str()
            .unwrap(),
    );
-    let mut color_map: HashMap<String, Color> = HashMap::new();
+    let mut color_map: HashMap<String, Colors> = HashMap::new();
    if configs::CONFIG.read().unwrap().args.no_color {
        return color_map;
    }
@@ -69,16 +77,34 @@ pub fn set_output_color() -> HashMap<String, Color> {
        }
        color_map.insert(
            level.to_lowercase(),
-            Color::Rgb(color_code[0], color_code[1], color_code[2]),
+            Colors {
+                output_color: termcolor::Color::Rgb(color_code[0], color_code[1], color_code[2]),
+                table_color: comfy_table::Color::Rgb {
+                    r: color_code[0],
+                    g: color_code[1],
+                    b: color_code[2],
+                },
+            },
        );
    });
    color_map
 }

-fn _get_output_color(color_map: &HashMap<String, Color>, level: &str) -> Option<Color> {
+fn _get_output_color(color_map: &HashMap<String, Colors>, level: &str) -> Option<Color> {
    let mut color = None;
    if let Some(c) = color_map.get(&level.to_lowercase()) {
-        color = Some(c.to_owned());
+        color = Some(c.output_color.to_owned());
+    }
+    color
+}
+
+fn _get_table_color(
+    color_map: &HashMap<String, Colors>,
+    level: &str,
+) -> Option<comfy_table::Color> {
+    let mut color = None;
+    if let Some(c) = color_map.get(&level.to_lowercase()) {
+        color = Some(c.table_color.to_owned());
    }
    color
 }
@@ -166,7 +192,7 @@ pub fn after_fact(all_record_cnt: usize) {
 fn emit_csv<W: std::io::Write>(
    writer: &mut W,
    displayflag: bool,
-    color_map: HashMap<String, Color>,
+    color_map: HashMap<String, Colors>,
    all_record_cnt: u128,
 ) -> io::Result<()> {
    let disp_wtr = BufferWriter::stdout(ColorChoice::Always);
@@ -356,17 +382,9 @@ fn emit_csv<W: std::io::Write>(
            &disp_wtr,
            get_writable_color(None),
            &format!(
-                "Total events: {}",
-                all_record_cnt.to_formatted_string(&Locale::en)
-            ),
-            true,
-        )
-        .ok();
-        write_color_buffer(
-            &disp_wtr,
-            get_writable_color(None),
-            &format!(
-                "Data reduction: {} events ({:.2}%)",
+                "Detected events / Total events: {} / {} (reduced {} events ({:.2}%))",
+                (all_record_cnt - reducted_record_cnt).to_formatted_string(&Locale::en),
+                all_record_cnt.to_formatted_string(&Locale::en),
                reducted_record_cnt.to_formatted_string(&Locale::en),
                reducted_percent
            ),
@@ -377,15 +395,8 @@ fn emit_csv<W: std::io::Write>(

        _print_unique_results(
            total_detect_counts_by_level,
-            "Total".to_string(),
-            "detections".to_string(),
-            &color_map,
-        );
-        println!();
-
-        _print_unique_results(
            unique_detect_counts_by_level,
-            "Unique".to_string(),
+            "Total | Unique".to_string(),
            "detections".to_string(),
            &color_map,
        );
@@ -393,11 +404,13 @@ fn emit_csv<W: std::io::Write>(

        _print_detection_summary_by_date(detect_counts_by_date_and_level, &color_map);
        println!();
+        println!();

        _print_detection_summary_by_computer(detect_counts_by_computer_and_level, &color_map);
        println!();

-        _print_detection_summary_by_rule(detect_counts_by_rule_and_level, &color_map);
+        _print_detection_summary_tables(detect_counts_by_rule_and_level, &color_map);
+        println!();
    }

    Ok(())
@@ -451,26 +464,30 @@ fn _format_cellpos(colval: &str, column: ColPos) -> String {
    }
 }

-/// output info which unique detection count and all detection count information(devided by level and total) to stdout.
+/// output info which unique detection count and all detection count information(separated by level and total) to stdout.
 fn _print_unique_results(
    mut counts_by_level: Vec<u128>,
+    mut unique_counts_by_level: Vec<u128>,
    head_word: String,
    tail_word: String,
-    color_map: &HashMap<String, Color>,
+    color_map: &HashMap<String, Colors>,
 ) {
    // the order in which are registered and the order of levels to be displayed are reversed
    counts_by_level.reverse();
+    unique_counts_by_level.reverse();

    let total_count = counts_by_level.iter().sum::<u128>();
+    let unique_total_count = unique_counts_by_level.iter().sum::<u128>();
    // output total results
    write_color_buffer(
        &BufferWriter::stdout(ColorChoice::Always),
        None,
        &format!(
-            "{} {}: {}",
+            "{} {}: {} | {}",
            head_word,
            tail_word,
            total_count.to_formatted_string(&Locale::en),
+            unique_total_count.to_formatted_string(&Locale::en)
        ),
        true,
    )
@@ -485,13 +502,20 @@ fn _print_unique_results(
        } else {
            (counts_by_level[i] as f64) / (total_count as f64) * 100.0
        };
+        let unique_percent = if unique_total_count == 0 {
+            0 as f64
+        } else {
+            (unique_counts_by_level[i] as f64) / (unique_total_count as f64) * 100.0
+        };
        let output_raw_str = format!(
-            "{} {} {}: {} ({:.2}%)",
+            "{} {} {}: {} ({:.2}%) | {} ({:.2}%)",
            head_word,
            level_name,
            tail_word,
            counts_by_level[i].to_formatted_string(&Locale::en),
-            percent
+            percent,
+            unique_counts_by_level[i].to_formatted_string(&Locale::en),
+            unique_percent
        );
        write_color_buffer(
            &BufferWriter::stdout(ColorChoice::Always),
@@ -506,13 +530,15 @@ fn _print_unique_results(
 /// 各レベル毎で最も高い検知数を出した日付を出力する
 fn _print_detection_summary_by_date(
    detect_counts_by_date: HashMap<String, HashMap<String, u128>>,
-    color_map: &HashMap<String, Color>,
+    color_map: &HashMap<String, Colors>,
 ) {
    let buf_wtr = BufferWriter::stdout(ColorChoice::Always);
    let mut wtr = buf_wtr.buffer();
    wtr.set_color(ColorSpec::new().set_fg(None)).ok();

-    for level in LEVEL_ABBR.values() {
+    writeln!(wtr, "Dates with most total detections:").ok();
+
+    for (idx, level) in LEVEL_ABBR.values().enumerate() {
        // output_levelsはlevelsからundefinedを除外した配列であり、各要素は必ず初期化されているのでSomeであることが保証されているのでunwrapをそのまま実施
        let detections_by_day = detect_counts_by_date.get(level).unwrap();
        let mut max_detect_str = String::default();
@@ -533,13 +559,18 @@ fn _print_detection_summary_by_date(
        if !exist_max_data {
            max_detect_str = "n/a".to_string();
        }
-        writeln!(
+        write!(
            wtr,
-            "Date with most total {} detections: {}",
+            "{}: {}",
            LEVEL_FULL.get(level.as_str()).unwrap(),
            &max_detect_str
        )
        .ok();
+        if idx != LEVEL_ABBR.len() - 1 {
+            wtr.set_color(ColorSpec::new().set_fg(None)).ok();
+
+            write!(wtr, ", ").ok();
+        }
    }
    buf_wtr.print(&wtr).ok();
 }
@@ -547,12 +578,13 @@ fn _print_detection_summary_by_date(
 /// 各レベル毎で最も高い検知数を出した日付を出力する
 fn _print_detection_summary_by_computer(
    detect_counts_by_computer: HashMap<String, HashMap<String, i128>>,
-    color_map: &HashMap<String, Color>,
+    color_map: &HashMap<String, Colors>,
 ) {
    let buf_wtr = BufferWriter::stdout(ColorChoice::Always);
    let mut wtr = buf_wtr.buffer();
    wtr.set_color(ColorSpec::new().set_fg(None)).ok();

+    writeln!(wtr, "Top 5 computers with most unique detections:").ok();
    for level in LEVEL_ABBR.values() {
        // output_levelsはlevelsからundefinedを除外した配列であり、各要素は必ず初期化されているのでSomeであることが保証されているのでunwrapをそのまま実施
        let detections_by_computer = detect_counts_by_computer.get(level).unwrap();
@@ -585,7 +617,7 @@ fn _print_detection_summary_by_computer(
        .ok();
        writeln!(
            wtr,
-            "Top 5 computers with most unique {} detections: {}",
+            "{}: {}",
            LEVEL_FULL.get(level.as_str()).unwrap(),
            &result_str
        )
@@ -594,53 +626,78 @@ fn _print_detection_summary_by_computer(
    buf_wtr.print(&wtr).ok();
 }

-/// 各レベルごとで検出数が多かったルールのタイトルを出力する関数
-fn _print_detection_summary_by_rule(
+/// 各レベルごとで検出数が多かったルールと日ごとの検知数を表形式で出力する関数
+fn _print_detection_summary_tables(
    detect_counts_by_rule_and_level: HashMap<String, HashMap<String, i128>>,
-    color_map: &HashMap<String, Color>,
+    color_map: &HashMap<String, Colors>,
 ) {
    let buf_wtr = BufferWriter::stdout(ColorChoice::Always);
    let mut wtr = buf_wtr.buffer();
    wtr.set_color(ColorSpec::new().set_fg(None)).ok();
-    let level_cnt = detect_counts_by_rule_and_level.len();
-    for (idx, level) in LEVEL_ABBR.values().enumerate() {
+    let mut output = vec![];
+    let mut col_color = vec![];
+    for level in LEVEL_ABBR.values() {
+        let mut col_output: Vec<String> = vec![];
+        col_output.push(format!(
+            "Top {} alerts:",
+            LEVEL_FULL.get(level.as_str()).unwrap()
+        ));
+
+        col_color.push(_get_table_color(
+            color_map,
+            LEVEL_FULL.get(level.as_str()).unwrap(),
+        ));
+
        // output_levelsはlevelsからundefinedを除外した配列であり、各要素は必ず初期化されているのでSomeであることが保証されているのでunwrapをそのまま実施
        let detections_by_computer = detect_counts_by_rule_and_level.get(level).unwrap();
-        let mut result_vec: Vec<String> = Vec::new();
        let mut sorted_detections: Vec<(&String, &i128)> = detections_by_computer.iter().collect();

        sorted_detections.sort_by(|a, b| (-a.1).cmp(&(-b.1)));

        for x in sorted_detections.iter().take(5) {
-            result_vec.push(format!(
+            col_output.push(format!(
                "{} ({})",
                x.0,
                x.1.to_formatted_string(&Locale::en)
            ));
        }
-        let result_str = if result_vec.is_empty() {
-            "None".to_string()
+        let na_cnt = if sorted_detections.len() > 5 {
+            0
        } else {
-            result_vec.join("\n")
+            5 - sorted_detections.len()
        };
-
-        wtr.set_color(ColorSpec::new().set_fg(_get_output_color(
-            color_map,
-            LEVEL_FULL.get(level.as_str()).unwrap(),
-        )))
-        .ok();
-        writeln!(
-            wtr,
-            "Top {} alerts:\n{}",
-            LEVEL_FULL.get(level.as_str()).unwrap(),
-            &result_str
-        )
-        .ok();
-        if idx != level_cnt - 1 {
-            writeln!(wtr).ok();
+        for _x in 0..na_cnt {
+            col_output.push("N/A".to_string());
        }
+        output.push(col_output);
    }
-    buf_wtr.print(&wtr).ok();
+
+    let mut tb = Table::new();
+    tb.load_preset(UTF8_FULL)
+        .apply_modifier(UTF8_ROUND_CORNERS)
+        .set_content_arrangement(ContentArrangement::Dynamic)
+        .set_width(500);
+    for x in 0..2 {
+        tb.add_row(vec![
+            Cell::new(&output[2 * x][0]).fg(col_color[2 * x].unwrap_or(comfy_table::Color::Reset)),
+            Cell::new(&output[2 * x + 1][0])
+                .fg(col_color[2 * x + 1].unwrap_or(comfy_table::Color::Reset)),
+        ]);
+
+        tb.add_row(vec![
+            Cell::new(&output[2 * x][1..].join("\n"))
+                .fg(col_color[2 * x].unwrap_or(comfy_table::Color::Reset)),
+            Cell::new(&output[2 * x + 1][1..].join("\n"))
+                .fg(col_color[2 * x + 1].unwrap_or(comfy_table::Color::Reset)),
+        ]);
+    }
+    tb.add_row(vec![
+        Cell::new(&output[4][0]).fg(col_color[4].unwrap_or(comfy_table::Color::Reset))
+    ]);
+    tb.add_row(vec![
+        Cell::new(&output[4][1..].join("\n")).fg(col_color[4].unwrap_or(comfy_table::Color::Reset))
+    ]);
+    println!("{tb}");
 }

 /// get timestamp to input datetime.