Display total event count and data reduction (#539)

* added reduction percent and all records cnt #538

* version updated v1.3.0-dev

* added events word

* added side margin to sparkline #533

* fixed centering

* change margin from 5 to 3

* readme warning typo fix

Co-authored-by: Tanaka Zakku <71482215+YamatoSecurity@users.noreply.github.com>

Cargo.lock

@@ -916,9 +916,9 @@ dependencies = [
 
 [[package]]
 name = "hayabusa"
-version = "1.2.2"
+version = "1.3.0-dev"
 dependencies = [
- "base64 0.13.0",
+ "base64 0.10.1",
  "chrono",
  "clap 2.34.0",
  "csv",

README.md

@@ -161,7 +161,7 @@ Please download the latest stable version of hayabusa with compiled binaries or
 
 You can `git clone` the repository with the following command and compile binary from source code:
 
-** Warning:** The main branch of the repository is for development purposes so you may be able to access new features not yet officially released, however, there may be bugs so consider it unstable.
+**Warning:** The main branch of the repository is for development purposes so you may be able to access new features not yet officially released, however, there may be bugs so consider it unstable.
 
 ```bash
 git clone https://github.com/Yamato-Security/hayabusa.git --recursive

src/afterfact.rs

@@ -101,7 +101,12 @@ fn _get_output_color(color_map: &HashMap<String, Color>, level: &str) -> Option<
 }
 
 /// print timeline histogram
-fn _print_timeline_hist(timestamps: Vec<i64>, marker_count: usize, length: usize) {
+fn _print_timeline_hist(
+    timestamps: Vec<i64>,
+    marker_count: usize,
+    length: usize,
+    side_margin_size: usize,
+) {
     if timestamps.is_empty() {
         return;
     }
@@ -116,15 +121,27 @@ fn _print_timeline_hist(timestamps: Vec<i64>, marker_count: usize, length: usize
     writeln!(wtr, "{}", title).ok();
     writeln!(wtr).ok();
 
-    let (header, footer) = build_time_markers(&timestamps, marker_count, length);
-    let sparkline = build_sparkline(&timestamps, length);
-    writeln!(wtr, "{}", header).ok();
-    writeln!(wtr, "{}", sparkline.unwrap_or_default()).ok();
-    writeln!(wtr, "{}", footer).ok();
+    let (header_raw, footer_raw) =
+        build_time_markers(&timestamps, marker_count, length - (side_margin_size * 2));
+    let sparkline = build_sparkline(&timestamps, length - (side_margin_size * 2));
+    for header_str in header_raw.lines() {
+        writeln!(wtr, "{}{}", " ".repeat(side_margin_size - 1), header_str).ok();
+    }
+    writeln!(
+        wtr,
+        "{}{}",
+        " ".repeat(side_margin_size - 1),
+        sparkline.unwrap_or_default()
+    )
+    .ok();
+    for footer_str in footer_raw.lines() {
+        writeln!(wtr, "{}{}", " ".repeat(side_margin_size - 1), footer_str).ok();
+    }
+
     buf_wtr.print(&wtr).ok();
 }
 
-pub fn after_fact() {
+pub fn after_fact(all_record_cnt: usize) {
     let fn_emit_csv_err = |err: Box<dyn Error>| {
         AlertMessage::alert(
             &mut BufWriter::new(std::io::stderr().lock()),
@@ -155,7 +172,7 @@ pub fn after_fact() {
             Box::new(BufWriter::new(io::stdout()))
         };
     let color_map = set_output_color();
-    if let Err(err) = emit_csv(&mut target, displayflag, color_map) {
+    if let Err(err) = emit_csv(&mut target, displayflag, color_map, all_record_cnt as u128) {
         fn_emit_csv_err(Box::new(err));
     }
 }
@@ -164,6 +181,7 @@ fn emit_csv<W: std::io::Write>(
     writer: &mut W,
     displayflag: bool,
     color_map: HashMap<String, Color>,
+    all_record_cnt: u128,
 ) -> io::Result<()> {
     let disp_wtr = BufferWriter::stdout(ColorChoice::Always);
     let mut disp_wtr_buf = disp_wtr.buffer();
@@ -252,12 +270,26 @@ fn emit_csv<W: std::io::Write>(
         wtr.flush()?;
     }
     println!();
+
     let size = terminal_size();
     let terminal_width = match size {
         Some((Width(w), _)) => w as usize,
         None => 100,
     };
-    _print_timeline_hist(timestamps, 10, terminal_width);
+    _print_timeline_hist(timestamps, 10, terminal_width, 3);
+    println!();
+    let reducted_record_cnt: u128 =
+        all_record_cnt - total_detect_counts_by_level.iter().sum::<u128>();
+    let reducted_percent = if all_record_cnt == 0 {
+        0 as f64
+    } else {
+        (reducted_record_cnt as f64) / (all_record_cnt as f64) * 100.0
+    };
+    println!("Total events: {}", all_record_cnt);
+    println!(
+        "Data reduction: {} events ({:.2}%)",
+        reducted_record_cnt, reducted_percent
+    );
     println!();
     _print_unique_results(
         total_detect_counts_by_level,
@@ -488,7 +520,7 @@ mod tests {
                 + test_filepath
                 + "\n";
         let mut file: Box<dyn io::Write> = Box::new(File::create("./test_emit_csv.csv").unwrap());
-        assert!(emit_csv(&mut file, false, HashMap::default()).is_ok());
+        assert!(emit_csv(&mut file, false, HashMap::default(), 1).is_ok());
         match read_to_string("./test_emit_csv.csv") {
             Err(_) => panic!("Failed to open file."),
             Ok(s) => {

src/detections/configs.rs

@@ -98,7 +98,7 @@ fn build_app<'a>() -> ArgMatches<'a> {
     --contributors 'Prints the list of contributors.'";
     App::new(&program)
         .about("Hayabusa: Aiming to be the world's greatest Windows event log analysis tool!")
-        .version("1.2.2")
+        .version("1.3.0-dev")
         .author("Yamato Security (https://github.com/Yamato-Security/hayabusa) @SecurityYamato")
         .setting(AppSettings::VersionlessSubcommands)
         .arg(

src/main.rs

@@ -457,16 +457,19 @@ impl App {
         pb.show_speed = false;
         self.rule_keys = self.get_all_keys(&rule_files);
         let mut detection = detection::Detection::new(rule_files);
+        let mut total_records: usize = 0;
         for evtx_file in evtx_files {
             if configs::CONFIG.read().unwrap().args.is_present("verbose") {
                 println!("Checking target evtx FilePath: {:?}", &evtx_file);
             }
-            detection = self.analysis_file(evtx_file, detection);
+            let cnt_tmp: usize;
+            (detection, cnt_tmp) = self.analysis_file(evtx_file, detection);
+            total_records += cnt_tmp;
             pb.inc();
         }
         detection.add_aggcondition_msges(&self.rt);
         if !(*STATISTICS_FLAG || *LOGONSUMMARY_FLAG || *PIVOT_KEYWORD_LIST_FLAG) {
-            after_fact();
+            after_fact(total_records);
         }
     }
 
@@ -475,11 +478,12 @@ impl App {
         &self,
         evtx_filepath: PathBuf,
         mut detection: detection::Detection,
-    ) -> detection::Detection {
+    ) -> (detection::Detection, usize) {
         let path = evtx_filepath.display();
         let parser = self.evtx_to_jsons(evtx_filepath.clone());
+        let mut record_cnt = 0;
         if parser.is_none() {
-            return detection;
+            return (detection, record_cnt);
         }
 
         let mut tl = Timeline::new();
@@ -529,6 +533,8 @@ impl App {
                 break;
             }
 
+            record_cnt += records_per_detect.len();
+
             let records_per_detect = self.rt.block_on(App::create_rec_infos(
                 records_per_detect,
                 &path,
@@ -547,7 +553,7 @@ impl App {
         tl.tm_stats_dsp_msg();
         tl.tm_logon_stats_dsp_msg();
 
-        detection
+        (detection, record_cnt)
     }
 
     async fn create_rec_infos(