Display total event count and data reduction (#539)

* added reduction percent and all records cnt #538

* version updated v1.3.0-dev

* added events word

* added side margin to sparkline #533

* fixed centering

* change margin from 5 to 3

* readme warning typo fix

Co-authored-by: Tanaka Zakku <71482215+YamatoSecurity@users.noreply.github.com>

src/afterfact.rs

@@ -101,7 +101,12 @@ fn _get_output_color(color_map: &HashMap<String, Color>, level: &str) -> Option<
 }
 
 /// print timeline histogram
-fn _print_timeline_hist(timestamps: Vec<i64>, marker_count: usize, length: usize) {
+fn _print_timeline_hist(
+    timestamps: Vec<i64>,
+    marker_count: usize,
+    length: usize,
+    side_margin_size: usize,
+) {
     if timestamps.is_empty() {
         return;
     }
@@ -116,15 +121,27 @@ fn _print_timeline_hist(timestamps: Vec<i64>, marker_count: usize, length: usize
     writeln!(wtr, "{}", title).ok();
     writeln!(wtr).ok();
 
-    let (header, footer) = build_time_markers(&timestamps, marker_count, length);
-    let sparkline = build_sparkline(&timestamps, length);
-    writeln!(wtr, "{}", header).ok();
-    writeln!(wtr, "{}", sparkline.unwrap_or_default()).ok();
-    writeln!(wtr, "{}", footer).ok();
+    let (header_raw, footer_raw) =
+        build_time_markers(&timestamps, marker_count, length - (side_margin_size * 2));
+    let sparkline = build_sparkline(&timestamps, length - (side_margin_size * 2));
+    for header_str in header_raw.lines() {
+        writeln!(wtr, "{}{}", " ".repeat(side_margin_size - 1), header_str).ok();
+    }
+    writeln!(
+        wtr,
+        "{}{}",
+        " ".repeat(side_margin_size - 1),
+        sparkline.unwrap_or_default()
+    )
+    .ok();
+    for footer_str in footer_raw.lines() {
+        writeln!(wtr, "{}{}", " ".repeat(side_margin_size - 1), footer_str).ok();
+    }
+
     buf_wtr.print(&wtr).ok();
 }
 
-pub fn after_fact() {
+pub fn after_fact(all_record_cnt: usize) {
     let fn_emit_csv_err = |err: Box<dyn Error>| {
         AlertMessage::alert(
             &mut BufWriter::new(std::io::stderr().lock()),
@@ -155,7 +172,7 @@ pub fn after_fact() {
             Box::new(BufWriter::new(io::stdout()))
         };
     let color_map = set_output_color();
-    if let Err(err) = emit_csv(&mut target, displayflag, color_map) {
+    if let Err(err) = emit_csv(&mut target, displayflag, color_map, all_record_cnt as u128) {
         fn_emit_csv_err(Box::new(err));
     }
 }
@@ -164,6 +181,7 @@ fn emit_csv<W: std::io::Write>(
     writer: &mut W,
     displayflag: bool,
     color_map: HashMap<String, Color>,
+    all_record_cnt: u128,
 ) -> io::Result<()> {
     let disp_wtr = BufferWriter::stdout(ColorChoice::Always);
     let mut disp_wtr_buf = disp_wtr.buffer();
@@ -252,12 +270,26 @@ fn emit_csv<W: std::io::Write>(
         wtr.flush()?;
     }
     println!();
+
     let size = terminal_size();
     let terminal_width = match size {
         Some((Width(w), _)) => w as usize,
         None => 100,
     };
-    _print_timeline_hist(timestamps, 10, terminal_width);
+    _print_timeline_hist(timestamps, 10, terminal_width, 3);
+    println!();
+    let reducted_record_cnt: u128 =
+        all_record_cnt - total_detect_counts_by_level.iter().sum::<u128>();
+    let reducted_percent = if all_record_cnt == 0 {
+        0 as f64
+    } else {
+        (reducted_record_cnt as f64) / (all_record_cnt as f64) * 100.0
+    };
+    println!("Total events: {}", all_record_cnt);
+    println!(
+        "Data reduction: {} events ({:.2}%)",
+        reducted_record_cnt, reducted_percent
+    );
     println!();
     _print_unique_results(
         total_detect_counts_by_level,
@@ -488,7 +520,7 @@ mod tests {
                 + test_filepath
                 + "\n";
         let mut file: Box<dyn io::Write> = Box::new(File::create("./test_emit_csv.csv").unwrap());
-        assert!(emit_csv(&mut file, false, HashMap::default()).is_ok());
+        assert!(emit_csv(&mut file, false, HashMap::default(), 1).is_ok());
         match read_to_string("./test_emit_csv.csv") {
             Err(_) => panic!("Failed to open file."),
             Ok(s) => {

src/detections/configs.rs

@@ -98,7 +98,7 @@ fn build_app<'a>() -> ArgMatches<'a> {
     --contributors 'Prints the list of contributors.'";
     App::new(&program)
         .about("Hayabusa: Aiming to be the world's greatest Windows event log analysis tool!")
-        .version("1.2.2")
+        .version("1.3.0-dev")
         .author("Yamato Security (https://github.com/Yamato-Security/hayabusa) @SecurityYamato")
         .setting(AppSettings::VersionlessSubcommands)
         .arg(

src/main.rs

@@ -457,16 +457,19 @@ impl App {
         pb.show_speed = false;
         self.rule_keys = self.get_all_keys(&rule_files);
         let mut detection = detection::Detection::new(rule_files);
+        let mut total_records: usize = 0;
         for evtx_file in evtx_files {
             if configs::CONFIG.read().unwrap().args.is_present("verbose") {
                 println!("Checking target evtx FilePath: {:?}", &evtx_file);
             }
-            detection = self.analysis_file(evtx_file, detection);
+            let cnt_tmp: usize;
+            (detection, cnt_tmp) = self.analysis_file(evtx_file, detection);
+            total_records += cnt_tmp;
             pb.inc();
         }
         detection.add_aggcondition_msges(&self.rt);
         if !(*STATISTICS_FLAG || *LOGONSUMMARY_FLAG || *PIVOT_KEYWORD_LIST_FLAG) {
-            after_fact();
+            after_fact(total_records);
         }
     }
 
@@ -475,11 +478,12 @@ impl App {
         &self,
         evtx_filepath: PathBuf,
         mut detection: detection::Detection,
-    ) -> detection::Detection {
+    ) -> (detection::Detection, usize) {
         let path = evtx_filepath.display();
         let parser = self.evtx_to_jsons(evtx_filepath.clone());
+        let mut record_cnt = 0;
         if parser.is_none() {
-            return detection;
+            return (detection, record_cnt);
         }
 
         let mut tl = Timeline::new();
@@ -529,6 +533,8 @@ impl App {
                 break;
             }
 
+            record_cnt += records_per_detect.len();
+
             let records_per_detect = self.rt.block_on(App::create_rec_infos(
                 records_per_detect,
                 &path,
@@ -547,7 +553,7 @@ impl App {
         tl.tm_stats_dsp_msg();
         tl.tm_logon_stats_dsp_msg();
 
-        detection
+        (detection, record_cnt)
     }
 
     async fn create_rec_infos(