From 497c45f8a20705435372621d7df0b27ab972db03 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 12 Aug 2022 15:38:30 +0900 Subject: [PATCH 01/40] added output feature to json #654 --- src/afterfact.rs | 91 +++++++++++++++++++++++++++++++++++++-- src/detections/configs.rs | 3 ++ 2 files changed, 91 insertions(+), 3 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 7be9dec8..20818919 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -7,11 +7,12 @@ use crate::detections::utils::{get_writable_color, write_color_buffer}; use crate::options::profile::PROFILES; use bytesize::ByteSize; use chrono::{DateTime, Local, TimeZone, Utc}; -use csv::QuoteStyle; +use csv::{QuoteStyle, WriterBuilder}; use itertools::Itertools; use krapslog::{build_sparkline, build_time_markers}; use lazy_static::lazy_static; use linked_hash_map::LinkedHashMap; +use std::str::FromStr; use hashbrown::{HashMap, HashSet}; use num_format::{Locale, ToFormattedString}; @@ -171,7 +172,16 @@ fn emit_csv( ) -> io::Result<()> { let disp_wtr = BufferWriter::stdout(ColorChoice::Always); let mut disp_wtr_buf = disp_wtr.buffer(); - let mut wtr = csv::WriterBuilder::new().from_writer(writer); + + let mut wtr = if configs::CONFIG.read().unwrap().args.json_timeline { + WriterBuilder::new() + .delimiter(b'\n') + .double_quote(true) + .quote_style(QuoteStyle::Never) + .from_writer(writer) + } else { + WriterBuilder::new().from_writer(writer) + }; disp_wtr_buf.set_color(ColorSpec::new().set_fg(None)).ok(); @@ -197,6 +207,9 @@ fn emit_csv( let mut timestamps: Vec = Vec::new(); let mut plus_header = true; let mut detected_record_idset: HashSet = HashSet::new(); + if configs::CONFIG.read().unwrap().args.json_timeline { + wtr.write_field("[")?; + } for time in message::MESSAGES.clone().into_read_only().keys().sorted() { let multi = message::MESSAGES.get(time).unwrap(); let (_, detect_infos) = multi.pair(); @@ -229,6 +242,10 @@ fn emit_csv( false, ) .ok(); + } else if configs::CONFIG.read().unwrap().args.json_timeline { + wtr.write_field(" {")?; + wtr.write_field(&output_json_str(&detect_info.ext_field))?; + wtr.write_field(" },")?; } else { // csv output format if plus_header { @@ -282,6 +299,9 @@ fn emit_csv( .insert(detect_info.level.to_lowercase(), detect_counts_by_date); } } + if configs::CONFIG.read().unwrap().args.json_timeline { + wtr.write_field("]")?; + } if displayflag { println!(); } else { @@ -407,7 +427,7 @@ fn _get_serialized_disp_output(data: &LinkedHashMap, header: boo } } } - let mut disp_serializer = csv::WriterBuilder::new() + let mut disp_serializer = WriterBuilder::new() .double_quote(false) .quote_style(QuoteStyle::Never) .delimiter(b'|') @@ -576,6 +596,71 @@ fn _get_timestamp(time: &DateTime) -> i64 { } } +/// json出力の際に配列として対応させるdetails,MitreTactics,MitreTags,OtherTagsに該当する場合に配列を返す関数 +fn _get_json_vec(target_alias_context: &str, target_data: &String) -> Vec { + if target_alias_context.contains("%MitreTactics%") + || target_alias_context.contains("%OtherTags%") + || target_alias_context.contains("%MitreTags%") + { + let ret: Vec = target_data + .to_owned() + .split(" : ") + .map(|x| x.to_string()) + .collect(); + ret + } else if target_alias_context.contains("%Details%") { + let ret: Vec = target_data + .to_owned() + .split(" | ") + .map(|x| x.to_string()) + .collect(); + if target_data == &ret[0] && !target_data.contains(": ") { + vec![] + } else { + ret + } + } else { + vec![] + } +} + +fn output_json_str(ext_field: &LinkedHashMap) -> String { + let mut target: Vec = vec![]; + let profile = PROFILES.clone().unwrap_or_default(); + for (k, v) in ext_field.iter() { + let output_value_fmt = profile.get(k).unwrap(); + let vec_data = _get_json_vec(output_value_fmt, v); + if vec_data.is_empty() { + if let Ok(i) = i64::from_str(v) { + target.push(format!(" \"{}\": {}", k, i)); + } else if let Ok(b) = bool::from_str(v) { + target.push(format!(" \"{}\": {}", k, b)); + } else { + target.push(format!(" \"{}\": \"{}\"", k, v)); + } + } else if output_value_fmt.contains("%Details%") { + let mut latest_valid_index = 0; + for (idx, detail_contents) in vec_data.iter().enumerate() { + let val: Vec<&str> = detail_contents.split(": ").collect(); + println!("{} | {} | {}", idx, latest_valid_index, target.len()); + if val.len() != 1 { + target.push(format!(" \"{}\": {:?}", val[0], val[1..].join(""))); + latest_valid_index = idx; + } else { + let prev_idx = if latest_valid_index == 0 { + latest_valid_index + } else { + target.len() - 1 + }; + let prev = &target[prev_idx]; + target[prev_idx] = format!("{} | {}", prev, val[0]); + } + } + } + } + target.join(",\n") +} + #[cfg(test)] mod tests { use crate::afterfact::_get_serialized_disp_output; diff --git a/src/detections/configs.rs b/src/detections/configs.rs index a0ac1b74..22660b04 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -218,6 +218,9 @@ pub struct Config { /// Set default output profile #[clap(long = "set-default-profile")] pub set_default_profile: Option, + + #[clap(short = 'j', long = "profile")] + pub json_timeline: bool, } impl ConfigReader<'_> { From a8e66e565dc2c270930bf04192a2ba20835d49f7 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 12 Aug 2022 15:42:43 +0900 Subject: [PATCH 02/40] updated changelog #654 --- CHANGELOG-Japanese.md | 1 + CHANGELOG.md | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG-Japanese.md b/CHANGELOG-Japanese.md index f0042e46..2a995a1e 100644 --- a/CHANGELOG-Japanese.md +++ b/CHANGELOG-Japanese.md @@ -6,6 +6,7 @@ - `config/profiles.yaml`と`config/default_profile.yaml`の設定ファイルで、出力内容をカスタマイズできる。 (#165) (@hitenkoku) - 対象のフィールドがレコード内に存在しないことを確認する `null` キーワードに対応した。 (#643) (@hitenkoku) +- 解析結果をJSONに出力する機能を追加した (`-j` と `--json-timeline` )。 (#654) (@hitenkoku) **改善:** diff --git a/CHANGELOG.md b/CHANGELOG.md index a14081c8..eb0a56a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Customizable output of fields defined at `config/profiles.yaml` and `config/default_profile.yaml`. (#165) (@hitenkoku) - Implemented the `null` keyword for rule detection. It is used to check if a target field exists or not. (#643) (@hitenkoku) +- Added output to JSON option (`-j` and `--json-timeline` ) (#654) (@hitenkoku) **Enhancements:** From 152732611838acc1d7f0449ec6aa829498e037bb Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 12 Aug 2022 15:44:17 +0900 Subject: [PATCH 03/40] fixed json-timeline long option --- src/detections/configs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index 22660b04..738ad430 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -219,7 +219,7 @@ pub struct Config { #[clap(long = "set-default-profile")] pub set_default_profile: Option, - #[clap(short = 'j', long = "profile")] + #[clap(short = 'j', long = "json-timeline")] pub json_timeline: bool, } From be0b69a53570b15929efec78f62c64d2b7e5fdba Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 12 Aug 2022 16:11:29 +0900 Subject: [PATCH 04/40] added json-timeline help_heading and help_str --- src/detections/configs.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index 77a0d4f9..43929d2e 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -224,7 +224,8 @@ pub struct Config { #[clap(help_heading = Some("OTHER-ACTIONS"), long = "set-default-profile", value_name = "PROFILE")] pub set_default_profile: Option, - #[clap(short = 'j', long = "json-timeline")] + /// Output result in JSON format + #[clap(help_heading = Some("OTHER-ACTIONS"), short = 'j', long = "json-timeline")] pub json_timeline: bool, } From 4441d52a085ddf7f61201a7f8cdd10feabd643aa Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 12 Aug 2022 16:27:41 +0900 Subject: [PATCH 05/40] to avoid confuse of output option treatment, changed type json-timeline option --- src/afterfact.rs | 19 +++++++++++++++---- src/detections/configs.rs | 4 ++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 985ca493..82287678 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -153,6 +153,15 @@ pub fn after_fact(all_record_cnt: usize) { process::exit(1); } } + } else if let Some(json_path) = &configs::CONFIG.read().unwrap().args.json_timeline { + // output to file + match File::create(json_path) { + Ok(file) => Box::new(BufWriter::new(file)), + Err(err) => { + AlertMessage::alert(&format!("Failed to open file. {}", err)).ok(); + process::exit(1); + } + } } else { displayflag = true; // stdoutput (termcolor crate color output is not csv writer) @@ -172,8 +181,9 @@ fn emit_csv( ) -> io::Result<()> { let disp_wtr = BufferWriter::stdout(ColorChoice::Always); let mut disp_wtr_buf = disp_wtr.buffer(); + let json_output_flag = configs::CONFIG.read().unwrap().args.json_timeline.is_some(); - let mut wtr = if configs::CONFIG.read().unwrap().args.json_timeline { + let mut wtr = if json_output_flag { WriterBuilder::new() .delimiter(b'\n') .double_quote(true) @@ -207,7 +217,7 @@ fn emit_csv( let mut timestamps: Vec = Vec::new(); let mut plus_header = true; let mut detected_record_idset: HashSet = HashSet::new(); - if configs::CONFIG.read().unwrap().args.json_timeline { + if json_output_flag { wtr.write_field("[")?; } for time in message::MESSAGES.clone().into_read_only().keys().sorted() { @@ -242,7 +252,7 @@ fn emit_csv( false, ) .ok(); - } else if configs::CONFIG.read().unwrap().args.json_timeline { + } else if json_output_flag { wtr.write_field(" {")?; wtr.write_field(&output_json_str(&detect_info.ext_field))?; wtr.write_field(" },")?; @@ -299,9 +309,10 @@ fn emit_csv( .insert(detect_info.level.to_lowercase(), detect_counts_by_date); } } - if configs::CONFIG.read().unwrap().args.json_timeline { + if json_output_flag { wtr.write_field("]")?; } + if displayflag { println!(); } else { diff --git a/src/detections/configs.rs b/src/detections/configs.rs index 43929d2e..02c1dab8 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -225,8 +225,8 @@ pub struct Config { pub set_default_profile: Option, /// Output result in JSON format - #[clap(help_heading = Some("OTHER-ACTIONS"), short = 'j', long = "json-timeline")] - pub json_timeline: bool, + #[clap(help_heading = Some("OTHER-ACTIONS"), short = 'j', long = "json-timeline", value_name = "FILE")] + pub json_timeline: Option, } impl ConfigReader<'_> { From dfe70a686dffd40edb5910255089e7ae93adfae5 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 12 Aug 2022 16:29:44 +0900 Subject: [PATCH 06/40] removed debug print --- src/afterfact.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 82287678..e0579f4f 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -657,7 +657,6 @@ fn output_json_str(ext_field: &LinkedHashMap) -> String { let mut latest_valid_index = 0; for (idx, detail_contents) in vec_data.iter().enumerate() { let val: Vec<&str> = detail_contents.split(": ").collect(); - println!("{} | {} | {}", idx, latest_valid_index, target.len()); if val.len() != 1 { target.push(format!(" \"{}\": {:?}", val[0], val[1..].join(""))); latest_valid_index = idx; From 98e972f24b18ec83920b4f66ad46cac2f47976dd Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Mon, 15 Aug 2022 04:24:49 +0900 Subject: [PATCH 07/40] fixed output to json #854 --- src/afterfact.rs | 123 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 101 insertions(+), 22 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index e0579f4f..72d2197d 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -7,6 +7,7 @@ use crate::detections::utils::{get_writable_color, write_color_buffer}; use crate::options::profile::PROFILES; use bytesize::ByteSize; use chrono::{DateTime, Local, TimeZone, Utc}; +use core::cmp::max; use csv::{QuoteStyle, WriterBuilder}; use itertools::Itertools; use krapslog::{build_sparkline, build_time_markers}; @@ -186,7 +187,7 @@ fn emit_csv( let mut wtr = if json_output_flag { WriterBuilder::new() .delimiter(b'\n') - .double_quote(true) + .double_quote(false) .quote_style(QuoteStyle::Never) .from_writer(writer) } else { @@ -220,7 +221,13 @@ fn emit_csv( if json_output_flag { wtr.write_field("[")?; } - for time in message::MESSAGES.clone().into_read_only().keys().sorted() { + for (processed_message_cnt, time) in message::MESSAGES + .clone() + .into_read_only() + .keys() + .sorted() + .enumerate() + { let multi = message::MESSAGES.get(time).unwrap(); let (_, detect_infos) = multi.pair(); timestamps.push(_get_timestamp(time)); @@ -255,7 +262,11 @@ fn emit_csv( } else if json_output_flag { wtr.write_field(" {")?; wtr.write_field(&output_json_str(&detect_info.ext_field))?; - wtr.write_field(" },")?; + if processed_message_cnt == message::MESSAGES.len() - 1 { + wtr.write_field(" }")?; + } else { + wtr.write_field(" },")?; + } } else { // csv output format if plus_header { @@ -619,7 +630,7 @@ fn _get_json_vec(target_alias_context: &str, target_data: &String) -> Vec = target_data .to_owned() - .split(" : ") + .split(": ") .map(|x| x.to_string()) .collect(); ret @@ -639,6 +650,53 @@ fn _get_json_vec(target_alias_context: &str, target_data: &String) -> Vec String { + // 4 space is json indent. + if let Ok(i) = i64::from_str(value) { + format!(" \"{}\": {}", key, i) + } else if let Ok(b) = bool::from_str(value) { + format!(" \"{}\": {}", key, b) + } else if concat_flag { + format!(" \"{}\": {}", key, value) + } else { + format!(" \"{}\": \"{}\"", key, value) + } +} + +fn _convert_valid_json_str(input: &[&str]) -> String { + let tmp = if input.is_empty() { + input[0].to_string() + } else { + input[1..].join(": ") + }; + let char_cnt = tmp.char_indices().count(); + let con_val = tmp.as_str(); + if char_cnt == 0 { + tmp + } else if con_val.starts_with('\"') { + let (head, _) = tmp.char_indices().nth(min(1, char_cnt - 1)).unwrap(); + let (tail, _) = tmp.char_indices().nth(max(0, char_cnt - 1)).unwrap(); + let addition_quote = if !con_val.ends_with('\"') { "\"" } else { "" }; + [ + &con_val[..head], + con_val[head..tail] + .to_string() + .replace('\\', "\\\\") + .replace('\"', "\\\"") + .trim(), + &con_val[tail..], + addition_quote, + ] + .join("") + } else { + con_val + .replace('\\', "\\\\") + .replace('\"', "\\\"") + .trim() + .to_string() + } +} + fn output_json_str(ext_field: &LinkedHashMap) -> String { let mut target: Vec = vec![]; let profile = PROFILES.clone().unwrap_or_default(); @@ -646,28 +704,49 @@ fn output_json_str(ext_field: &LinkedHashMap) -> String { let output_value_fmt = profile.get(k).unwrap(); let vec_data = _get_json_vec(output_value_fmt, v); if vec_data.is_empty() { - if let Ok(i) = i64::from_str(v) { - target.push(format!(" \"{}\": {}", k, i)); - } else if let Ok(b) = bool::from_str(v) { - target.push(format!(" \"{}\": {}", k, b)); - } else { - target.push(format!(" \"{}\": \"{}\"", k, v)); - } + target.push(_create_json_output_format(k, v, v.starts_with('\"'))); } else if output_value_fmt.contains("%Details%") { - let mut latest_valid_index = 0; + let mut tmp = String::default(); for (idx, detail_contents) in vec_data.iter().enumerate() { let val: Vec<&str> = detail_contents.split(": ").collect(); - if val.len() != 1 { - target.push(format!(" \"{}\": {:?}", val[0], val[1..].join(""))); - latest_valid_index = idx; - } else { - let prev_idx = if latest_valid_index == 0 { - latest_valid_index + + // ': 'の記載が値の中にある場合があるため、追加予定の文面が存在するかを確認する + if !tmp.is_empty() { + // 次の要素が確認できるかを確認する + if idx != vec_data.len() - 1 { + let space_split: Vec<&str> = vec_data[idx + 1].split(' ').collect(); + if !space_split[0].ends_with(':') { + // 次の要素を確認してJSONのキー名がなかった場合は連結する + let end_pos = if tmp.is_empty() { 0 } else { tmp.len() - 1 }; + tmp = tmp[..end_pos].to_string(); + tmp = [&tmp, val.join(": ").trim()].join(" | "); + } else { + // 次の要素がJSONのキー名に合致する箇所が存在した場合は現在の文字列を展開してJSONの要素として追加する + // TODO: functionalize + let tmp_val: Vec<&str> = tmp.split(": ").collect(); + let fmted_json_val = _convert_valid_json_str(&tmp_val); + target.push(_create_json_output_format( + &tmp_val[0].to_string(), + &fmted_json_val, + fmted_json_val.starts_with('\"'), + )); + tmp = "".to_string(); + } } else { - target.len() - 1 - }; - let prev = &target[prev_idx]; - target[prev_idx] = format!("{} | {}", prev, val[0]); + // 配列の一番最後の要素の場合、JSONのオブジェクトとして出力する + // TODO: functionalize + let tmp_val: Vec<&str> = tmp.split(": ").collect(); + let fmted_json_val = _convert_valid_json_str(&tmp_val); + target.push(_create_json_output_format( + &tmp_val[0].to_string(), + &fmted_json_val, + fmted_json_val.starts_with('\"'), + )); + tmp = String::default(); + } + } else { + // JSONの出力町文字列がない場合は値を追加する + tmp = format!("{}: {}", val[0], val[1..].join(": ").trim()); } } } From a8498429aadaa11df17cacb7be043a5e636e34cc Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Mon, 15 Aug 2022 05:54:17 +0900 Subject: [PATCH 08/40] added comment --- src/afterfact.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 72d2197d..fa339668 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -169,7 +169,7 @@ pub fn after_fact(all_record_cnt: usize) { Box::new(BufWriter::new(io::stdout())) }; let color_map = set_output_color(); - if let Err(err) = emit_csv(&mut target, displayflag, color_map, all_record_cnt as u128) { + if let Err(err) = emit_csv(&mut target, displayflag, color_map, all_record_cnt as u128, PROFILES.clone().unwrap_or_default()) { fn_emit_csv_err(Box::new(err)); } } @@ -179,6 +179,7 @@ fn emit_csv( displayflag: bool, color_map: HashMap, all_record_cnt: u128, + profile: LinkedHashMap ) -> io::Result<()> { let disp_wtr = BufferWriter::stdout(ColorChoice::Always); let mut disp_wtr_buf = disp_wtr.buffer(); @@ -241,7 +242,7 @@ fn emit_csv( write_color_buffer( &disp_wtr, get_writable_color(None), - &_get_serialized_disp_output(PROFILES.as_ref().unwrap(), true), + &_get_serialized_disp_output(&profile, true), false, ) .ok(); @@ -261,7 +262,7 @@ fn emit_csv( .ok(); } else if json_output_flag { wtr.write_field(" {")?; - wtr.write_field(&output_json_str(&detect_info.ext_field))?; + wtr.write_field(&output_json_str(&detect_info.ext_field, &profile))?; if processed_message_cnt == message::MESSAGES.len() - 1 { wtr.write_field(" }")?; } else { @@ -650,6 +651,7 @@ fn _get_json_vec(target_alias_context: &str, target_data: &String) -> Vec String { // 4 space is json indent. if let Ok(i) = i64::from_str(value) { @@ -663,6 +665,7 @@ fn _create_json_output_format(key: &String, value: &str, concat_flag: bool) -> S } } +/// JSONの値に対して文字列の出力形式をJSON出力でエラーにならないようにするための変換を行う関数 fn _convert_valid_json_str(input: &[&str]) -> String { let tmp = if input.is_empty() { input[0].to_string() @@ -697,9 +700,10 @@ fn _convert_valid_json_str(input: &[&str]) -> String { } } -fn output_json_str(ext_field: &LinkedHashMap) -> String { + +/// JSONに出力する1検知分のオブジェクトの文字列を出力する関数 +fn output_json_str(ext_field: &LinkedHashMap, profile: &LinkedHashMap) -> String { let mut target: Vec = vec![]; - let profile = PROFILES.clone().unwrap_or_default(); for (k, v) in ext_field.iter() { let output_value_fmt = profile.get(k).unwrap(); let vec_data = _get_json_vec(output_value_fmt, v); @@ -842,7 +846,7 @@ mod tests { eventid: test_eventid.to_string(), detail: String::default(), record_information: Option::Some(test_recinfo.to_string()), - ext_field: output_profile, + ext_field: output_profile.clone(), }, expect_time, &mut profile_converter, @@ -882,7 +886,7 @@ mod tests { + test_attack + "\n"; let mut file: Box = Box::new(File::create("./test_emit_csv.csv").unwrap()); - assert!(emit_csv(&mut file, false, HashMap::new(), 1).is_ok()); + assert!(emit_csv(&mut file, false, HashMap::new(), 1, output_profile).is_ok()); match read_to_string("./test_emit_csv.csv") { Err(_) => panic!("Failed to open file."), Ok(s) => { From 28799894b5a0c7f8c1534551d7eeb4a9453d3625 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Mon, 15 Aug 2022 05:58:10 +0900 Subject: [PATCH 09/40] updated changelog #639 --- CHANGELOG-Japanese.md | 1 + CHANGELOG.md | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG-Japanese.md b/CHANGELOG-Japanese.md index a4b8ae4e..764bf951 100644 --- a/CHANGELOG-Japanese.md +++ b/CHANGELOG-Japanese.md @@ -19,6 +19,7 @@ **バグ修正:** - aggregation conditionのルール検知が原因で検知しなかったイベント数の集計に誤りがあったので修正した。 (#640) (@hitenkoku) +- 検知の際に競合状態が発生して検知回数が安定しないバグを修正した。 (#639) (@fukusuket) ## v1.4.3 [2022/08/03] diff --git a/CHANGELOG.md b/CHANGELOG.md index 683d25d4..7686c06a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ **Bug Fixes:** - Fixed miscalculation of Data Reduction due to aggregation condition rule detection. (#640) (@hitenkoku) +- Fixed race condition in detection. (#639) (@fukusuket) ## v1.4.3 [2022/08/03] From f1b1d919dcdeafbb64f8647a4a61880b3015e1f4 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Sun, 21 Aug 2022 00:59:12 +0900 Subject: [PATCH 10/40] Added output JSON format feature #654 --- src/afterfact.rs | 138 +++++++++++++++++++++----------------- src/detections/configs.rs | 4 +- 2 files changed, 79 insertions(+), 63 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index fa339668..afd57b4f 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -154,22 +154,19 @@ pub fn after_fact(all_record_cnt: usize) { process::exit(1); } } - } else if let Some(json_path) = &configs::CONFIG.read().unwrap().args.json_timeline { - // output to file - match File::create(json_path) { - Ok(file) => Box::new(BufWriter::new(file)), - Err(err) => { - AlertMessage::alert(&format!("Failed to open file. {}", err)).ok(); - process::exit(1); - } - } } else { displayflag = true; // stdoutput (termcolor crate color output is not csv writer) Box::new(BufWriter::new(io::stdout())) }; let color_map = set_output_color(); - if let Err(err) = emit_csv(&mut target, displayflag, color_map, all_record_cnt as u128, PROFILES.clone().unwrap_or_default()) { + if let Err(err) = emit_csv( + &mut target, + displayflag, + color_map, + all_record_cnt as u128, + PROFILES.clone().unwrap_or_default(), + ) { fn_emit_csv_err(Box::new(err)); } } @@ -179,11 +176,11 @@ fn emit_csv( displayflag: bool, color_map: HashMap, all_record_cnt: u128, - profile: LinkedHashMap + profile: LinkedHashMap, ) -> io::Result<()> { let disp_wtr = BufferWriter::stdout(ColorChoice::Always); let mut disp_wtr_buf = disp_wtr.buffer(); - let json_output_flag = configs::CONFIG.read().unwrap().args.json_timeline.is_some(); + let json_output_flag = configs::CONFIG.read().unwrap().args.json_timeline; let mut wtr = if json_output_flag { WriterBuilder::new() @@ -651,23 +648,28 @@ fn _get_json_vec(target_alias_context: &str, target_data: &String) -> Vec String { +/// JSONの出力フォーマットに合わせた文字列を出力する関数 +fn _create_json_output_format(key: &String, value: &str, key_quote_exclude_flag: bool, concat_flag: bool) -> String { + let head = if key_quote_exclude_flag { + key.to_string() + } else { + format!("\"{}\"", key) + }; // 4 space is json indent. if let Ok(i) = i64::from_str(value) { - format!(" \"{}\": {}", key, i) + format!(" {}: {}", head, i) } else if let Ok(b) = bool::from_str(value) { - format!(" \"{}\": {}", key, b) + format!(" {}: {}", head, b) } else if concat_flag { - format!(" \"{}\": {}", key, value) + format!(" {}: {}", head, value) } else { - format!(" \"{}\": \"{}\"", key, value) + format!(" {}: \"{}\"", head, value) } } /// JSONの値に対して文字列の出力形式をJSON出力でエラーにならないようにするための変換を行う関数 fn _convert_valid_json_str(input: &[&str]) -> String { - let tmp = if input.is_empty() { + let tmp = if input.len() == 1 { input[0].to_string() } else { input[1..].join(": ") @@ -700,57 +702,71 @@ fn _convert_valid_json_str(input: &[&str]) -> String { } } - /// JSONに出力する1検知分のオブジェクトの文字列を出力する関数 -fn output_json_str(ext_field: &LinkedHashMap, profile: &LinkedHashMap) -> String { +fn output_json_str( + ext_field: &LinkedHashMap, + profile: &LinkedHashMap, +) -> String { let mut target: Vec = vec![]; for (k, v) in ext_field.iter() { let output_value_fmt = profile.get(k).unwrap(); let vec_data = _get_json_vec(output_value_fmt, v); if vec_data.is_empty() { - target.push(_create_json_output_format(k, v, v.starts_with('\"'))); + let tmp_val: Vec<&str> = v.split(": ").collect(); + target.push(_create_json_output_format( + k, + &_convert_valid_json_str(&tmp_val), + k.starts_with('\"'), + v.starts_with('\"'), + )); } else if output_value_fmt.contains("%Details%") { - let mut tmp = String::default(); - for (idx, detail_contents) in vec_data.iter().enumerate() { - let val: Vec<&str> = detail_contents.split(": ").collect(); - - // ': 'の記載が値の中にある場合があるため、追加予定の文面が存在するかを確認する - if !tmp.is_empty() { - // 次の要素が確認できるかを確認する - if idx != vec_data.len() - 1 { - let space_split: Vec<&str> = vec_data[idx + 1].split(' ').collect(); - if !space_split[0].ends_with(':') { - // 次の要素を確認してJSONのキー名がなかった場合は連結する - let end_pos = if tmp.is_empty() { 0 } else { tmp.len() - 1 }; - tmp = tmp[..end_pos].to_string(); - tmp = [&tmp, val.join(": ").trim()].join(" | "); - } else { - // 次の要素がJSONのキー名に合致する箇所が存在した場合は現在の文字列を展開してJSONの要素として追加する - // TODO: functionalize - let tmp_val: Vec<&str> = tmp.split(": ").collect(); - let fmted_json_val = _convert_valid_json_str(&tmp_val); - target.push(_create_json_output_format( - &tmp_val[0].to_string(), - &fmted_json_val, - fmted_json_val.starts_with('\"'), - )); - tmp = "".to_string(); - } + let mut stocked_value = vec![]; + let mut key_index_stock = vec![]; + for detail_contents in vec_data.iter() { + // 分解してキーとなりえる箇所を抽出する + let space_split: Vec<&str> = detail_contents.split(' ').collect(); + let mut tmp_stock = vec![]; + for sp in space_split.iter() { + if sp.ends_with(':') && sp != &":" && sp.len() != 2 { + stocked_value.push(tmp_stock); + tmp_stock = vec![]; + key_index_stock.push(sp.replace(':', "").to_owned()); } else { - // 配列の一番最後の要素の場合、JSONのオブジェクトとして出力する - // TODO: functionalize - let tmp_val: Vec<&str> = tmp.split(": ").collect(); - let fmted_json_val = _convert_valid_json_str(&tmp_val); - target.push(_create_json_output_format( - &tmp_val[0].to_string(), - &fmted_json_val, - fmted_json_val.starts_with('\"'), - )); - tmp = String::default(); + tmp_stock.push(sp.to_owned()); } - } else { - // JSONの出力町文字列がない場合は値を追加する - tmp = format!("{}: {}", val[0], val[1..].join(": ").trim()); + } + if !tmp_stock.is_empty() { + stocked_value.push(tmp_stock); + } + } + let mut tmp = String::default(); + let mut key_idx = 0; + let mut output_value_stock = String::default(); + for (value_idx, value) in stocked_value.iter().enumerate() { + if value.is_empty() { + tmp = key_index_stock[key_idx].to_string(); + key_idx += 1; + continue; + } else if value_idx == 0 { + tmp = k.to_string(); + } + if !output_value_stock.is_empty() { + output_value_stock.push_str(" | "); + } + output_value_stock.push_str(&value.join(" ")); + if value_idx < stocked_value.len() -1 && stocked_value[value_idx + 1].is_empty() { + // 次の要素を確認して、存在しないもしくは、キーが入っているとなった場合現在ストックしている内容が出力していいことが確定するので主力処理を行う + let output_tmp = format!("{}: {}", tmp, output_value_stock); + let output:Vec<&str> = output_tmp.split(": ").collect(); + let key = _convert_valid_json_str(&[output[0]]); + let fmted_val = _convert_valid_json_str(&output); + target.push( + _create_json_output_format( + &key, &fmted_val, key.starts_with('\"'),fmted_val.starts_with('\"') + ) + ); + output_value_stock.clear(); + tmp = String::default(); } } } diff --git a/src/detections/configs.rs b/src/detections/configs.rs index 02c1dab8..43929d2e 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -225,8 +225,8 @@ pub struct Config { pub set_default_profile: Option, /// Output result in JSON format - #[clap(help_heading = Some("OTHER-ACTIONS"), short = 'j', long = "json-timeline", value_name = "FILE")] - pub json_timeline: Option, + #[clap(help_heading = Some("OTHER-ACTIONS"), short = 'j', long = "json-timeline")] + pub json_timeline: bool, } impl ConfigReader<'_> { From a2727e2e04a70ea33091b3fa2bddce72215287bb Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Sun, 21 Aug 2022 00:59:53 +0900 Subject: [PATCH 11/40] cargo fmt --- src/afterfact.rs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index afd57b4f..6cbe0d41 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -649,7 +649,12 @@ fn _get_json_vec(target_alias_context: &str, target_data: &String) -> Vec String { +fn _create_json_output_format( + key: &String, + value: &str, + key_quote_exclude_flag: bool, + concat_flag: bool, +) -> String { let head = if key_quote_exclude_flag { key.to_string() } else { @@ -743,7 +748,7 @@ fn output_json_str( let mut key_idx = 0; let mut output_value_stock = String::default(); for (value_idx, value) in stocked_value.iter().enumerate() { - if value.is_empty() { + if value.is_empty() { tmp = key_index_stock[key_idx].to_string(); key_idx += 1; continue; @@ -754,17 +759,18 @@ fn output_json_str( output_value_stock.push_str(" | "); } output_value_stock.push_str(&value.join(" ")); - if value_idx < stocked_value.len() -1 && stocked_value[value_idx + 1].is_empty() { + if value_idx < stocked_value.len() - 1 && stocked_value[value_idx + 1].is_empty() { // 次の要素を確認して、存在しないもしくは、キーが入っているとなった場合現在ストックしている内容が出力していいことが確定するので主力処理を行う let output_tmp = format!("{}: {}", tmp, output_value_stock); - let output:Vec<&str> = output_tmp.split(": ").collect(); + let output: Vec<&str> = output_tmp.split(": ").collect(); let key = _convert_valid_json_str(&[output[0]]); let fmted_val = _convert_valid_json_str(&output); - target.push( - _create_json_output_format( - &key, &fmted_val, key.starts_with('\"'),fmted_val.starts_with('\"') - ) - ); + target.push(_create_json_output_format( + &key, + &fmted_val, + key.starts_with('\"'), + fmted_val.starts_with('\"'), + )); output_value_stock.clear(); tmp = String::default(); } From ef992eeece153434ea292b3047c26331b33ce56a Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Sun, 21 Aug 2022 01:19:03 +0900 Subject: [PATCH 12/40] version up -> 1.6.0-dev --- CHANGELOG-Japanese.md | 2 +- CHANGELOG.md | 2 +- Cargo.lock | 65 +++---------------------------------------- Cargo.toml | 2 +- rules | 2 +- 5 files changed, 8 insertions(+), 65 deletions(-) diff --git a/CHANGELOG-Japanese.md b/CHANGELOG-Japanese.md index 71a7d6c1..1a14e869 100644 --- a/CHANGELOG-Japanese.md +++ b/CHANGELOG-Japanese.md @@ -1,6 +1,6 @@ # 変更点 -# v1.5.2 [2022/XX/XX] +# v1.6.0 [2022/XX/XX] **新機能:** diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fe32d73..4adc7c65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changes -## v1.5.2 [2022/08/21] +## v1.6.0 [2022/08/21] **New Features:** diff --git a/Cargo.lock b/Cargo.lock index e1facd6d..fce05778 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -263,7 +263,7 @@ dependencies = [ "encode_unicode 0.3.6", "libc", "once_cell", - "terminal_size 0.1.17", + "terminal_size", "unicode-width", "winapi", ] @@ -508,27 +508,6 @@ dependencies = [ "termcolor", ] -[[package]] -name = "errno" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" -dependencies = [ - "errno-dragonfly", - "libc", - "winapi", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "error-chain" version = "0.12.4" @@ -707,7 +686,7 @@ dependencies = [ [[package]] name = "hayabusa" -version = "1.5.1" +version = "1.6.0-dev" dependencies = [ "base64", "bytesize", @@ -743,7 +722,7 @@ dependencies = [ "serde_json", "static_vcruntime", "termcolor", - "terminal_size 0.2.1", + "terminal_size", "tokio", "yaml-rust", ] @@ -905,12 +884,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "io-lifetimes" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ea37f355c05dde75b84bba2d767906ad522e97cd9e2eef2be7a4ab7fb442c06" - [[package]] name = "is_elevated" version = "0.1.2" @@ -998,7 +971,7 @@ dependencies = [ "rayon", "regex", "tempfile", - "terminal_size 0.1.17", + "terminal_size", ] [[package]] @@ -1059,12 +1032,6 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" -[[package]] -name = "linux-raw-sys" -version = "0.0.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d" - [[package]] name = "lock_api" version = "0.4.7" @@ -1559,20 +1526,6 @@ dependencies = [ "semver 0.9.0", ] -[[package]] -name = "rustix" -version = "0.35.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72c825b8aa8010eb9ee99b75f05e10180b9278d161583034d7574c9d617aeada" -dependencies = [ - "bitflags", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys", - "windows-sys", -] - [[package]] name = "rustversion" version = "1.0.9" @@ -1852,16 +1805,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "terminal_size" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8440c860cf79def6164e4a0a983bcc2305d82419177a0e0c71930d049e3ac5a1" -dependencies = [ - "rustix", - "windows-sys", -] - [[package]] name = "textwrap" version = "0.11.0" diff --git a/Cargo.toml b/Cargo.toml index e19cda31..8e1b7674 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hayabusa" -version = "1.5.1" +version = "1.6.0-dev" authors = ["Yamato Security @SecurityYamato"] edition = "2021" diff --git a/rules b/rules index 85631637..c5110b27 160000 --- a/rules +++ b/rules @@ -1 +1 @@ -Subproject commit 856316374ca52ce01123c2078c7af294d29df546 +Subproject commit c5110b27f6f9f788f17ac29fc2251d95e676688d From 702b4a11527ebb0049bf0b877ee35e5fc1beed07 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Sun, 21 Aug 2022 11:40:05 +0900 Subject: [PATCH 13/40] updated usage in readme #654 --- README-Japanese.md | 13 +++++++------ README.md | 1 + 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/README-Japanese.md b/README-Japanese.md index 995952c4..03b4b309 100644 --- a/README-Japanese.md +++ b/README-Japanese.md @@ -406,13 +406,14 @@ FILTERING: --timeline-start 解析対象とするイベントログの開始時刻 (例: "2020-02-22 00:00:00 +09:00") OTHER-ACTIONS: - --contributors コントリビュータの一覧表示 - -L, --logon-summary 成功と失敗したログオン情報の要約を出力する + --contributors コントリビュータの一覧表示 + -j, --json-timeline タイムラインの出力をJSON形式保存する + -L, --logon-summary 成功と失敗したログオン情報の要約を出力する --level-tuning [] ルールlevelのチューニング (デフォルト: ./rules/config/level_tuning.txt) - -p, --pivot-keywords-list ピボットキーワードの一覧作成 - -s, --statistics イベントIDの統計情報を表示する - --set-default-profile デフォルトの出力コンフィグを設定する - -u, --update-rules rulesフォルダをhayabusa-rulesのgithubリポジトリの最新版に更新する + -p, --pivot-keywords-list ピボットキーワードの一覧作成 + -s, --statistics イベントIDの統計情報を表示する + --set-default-profile デフォルトの出力コンフィグを設定する + -u, --update-rules rulesフォルダをhayabusa-rulesのgithubリポジトリの最新版に更新する TIME-FORMAT: --European-time ヨーロッパ形式で日付と時刻を出力する (例: 22-02-2022 22:00:00.123 +02:00) diff --git a/README.md b/README.md index 780c3fc0..0cc9b51e 100644 --- a/README.md +++ b/README.md @@ -399,6 +399,7 @@ FILTERING: OTHER-ACTIONS: --contributors Print the list of contributors + -j, --json-timeline Output result in JSON format -L, --logon-summary Print a summary of successful and failed logons --level-tuning [] Tune alert levels (default: ./rules/config/level_tuning.txt) -p, --pivot-keywords-list Create a list of pivot keywords From 3b300d22132059b0450705012a60e155202cc9f6 Mon Sep 17 00:00:00 2001 From: Yamato Security <71482215+YamatoSecurity@users.noreply.github.com> Date: Mon, 22 Aug 2022 07:53:26 +0900 Subject: [PATCH 14/40] ignore json files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index eec35107..5c17e5e9 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ test_* .env /logs *.csv +*.json hayabusa* \ No newline at end of file From b8ddf2e499a1947fa539ebe218a839242f93c47c Mon Sep 17 00:00:00 2001 From: Yamato Security <71482215+YamatoSecurity@users.noreply.github.com> Date: Mon, 22 Aug 2022 07:55:12 +0900 Subject: [PATCH 15/40] readme and changelog updates --- CHANGELOG.md | 4 ++-- README-Japanese.md | 10 ++++++++-- README.md | 10 ++++++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 410831cb..016df85c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,11 @@ **New Features:** -- Added output to JSON option (`-j` and `--json-timeline` ) (#654) (@hitenkoku) +- You can now save the timeline to JSON files with the `-j, --json-timeline` option. (#654) (@hitenkoku) **Enhancements:** -- Added top alert rules to results summary. (#667) (@hitenkoku) +- Added top alerts to the results summary. (#667) (@hitenkoku) **Bug Fixes:** diff --git a/README-Japanese.md b/README-Japanese.md index 5fd72ce5..72eb9213 100644 --- a/README-Japanese.md +++ b/README-Japanese.md @@ -387,6 +387,7 @@ ADVANCED: --target-file-ext ... evtx以外の拡張子を解析対象に追加する。 (例１: evtx_data 例２：evtx1 evtx2) OUTPUT: + -j, --json タイムラインの出力をJSON形式で保存する（例: -j -o results.json） -o, --output タイムラインをCSV形式で保存する (例: results.csv) -P, --profile 利用する出力プロファイル名を指定する (minimal, standard, verbose, verbose-all-field-info, verbose-details-and-all-field-info) @@ -407,7 +408,6 @@ FILTERING: OTHER-ACTIONS: --contributors コントリビュータの一覧表示 - -j, --json-timeline タイムラインの出力をJSON形式保存する -L, --logon-summary 成功と失敗したログオン情報の要約を出力する --level-tuning [] ルールlevelのチューニング (デフォルト: ./rules/config/level_tuning.txt) -p, --pivot-keywords-list ピボットキーワードの一覧作成 @@ -441,7 +441,13 @@ hayabusa-1.5.1-win-x64.exe -d .\hayabusa-sample-evtx -P verbose * 全てのフィールド情報も含めて１つのCSVファイルにエクスポートして、Excel、Timeline Explorer、Elastic Stack等でさらに分析することができる(注意: `verbose-details-and-all-field-info`プロファイルを使すると、出力するファイルのサイズがとても大きくなる！): ```bash -hayabusa-1.5.1-win-x64.exe -d .\hayabusa-sample-evtx -o results.csv -P `verbose-details-and-all-field-info` +hayabusa-1.5.1-win-x64.exe -d .\hayabusa-sample-evtx -o results.csv -P verbose-details-and-all-field-info +``` + +* タイムラインをJSON形式で保存する: + +```bash +hayabusa-1.5.1-win-x64.exe -d .\hayabusa-sample-evtx -o results.json -j ``` * Hayabusaルールのみを実行する（デフォルトでは`-r .\rules`にあるすべてのルールが利用される）: diff --git a/README.md b/README.md index eb8d4a27..2d1213de 100644 --- a/README.md +++ b/README.md @@ -379,6 +379,7 @@ ADVANCED: --target-file-ext ... Specify additional target file extensions (ex: evtx_data) (ex: evtx1 evtx2) OUTPUT: + -j, --json Save the timeline in JSON format (ex: -j -o results.json) -o, --output Save the timeline in CSV format (ex: results.csv) -P, --profile Specify output profile (minimal, standard, verbose, verbose-all-field-info, verbose-details-and-all-field-info) @@ -399,7 +400,6 @@ FILTERING: OTHER-ACTIONS: --contributors Print the list of contributors - -j, --json-timeline Output result in JSON format -L, --logon-summary Print a summary of successful and failed logons --level-tuning [] Tune alert levels (default: ./rules/config/level_tuning.txt) -p, --pivot-keywords-list Create a list of pivot keywords @@ -433,7 +433,13 @@ hayabusa-1.5.1-win-x64.exe -d .\hayabusa-sample-evtx -P verbose * Export to a single CSV file for further analysis with excel, timeline explorer, elastic stack, etc... and include all field information (Warning: your file output size will become much larger with the `verbose-details-and-all-field-info` profile!): ```bash -hayabusa-1.5.1-win-x64.exe -d .\hayabusa-sample-evtx -o results.csv -F +hayabusa-1.5.1-win-x64.exe -d .\hayabusa-sample-evtx -o results.csv -P verbose-details-and-all-field-info +``` + +* Save the timline in JSON format: + +```bash +hayabusa-1.5.1-win-x64.exe -d .\hayabusa-sample-evtx -o results.json -j ``` * Only run hayabusa rules (the default is to run all the rules in `-r .\rules`): From 16191d2824c36f640bd8abe735b5673b9d0636c7 Mon Sep 17 00:00:00 2001 From: Yamato Security <71482215+YamatoSecurity@users.noreply.github.com> Date: Mon, 22 Aug 2022 07:55:36 +0900 Subject: [PATCH 16/40] rename --json and require -o --- src/detections/configs.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/detections/configs.rs b/src/detections/configs.rs index 43929d2e..b2310ef8 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -224,8 +224,8 @@ pub struct Config { #[clap(help_heading = Some("OTHER-ACTIONS"), long = "set-default-profile", value_name = "PROFILE")] pub set_default_profile: Option, - /// Output result in JSON format - #[clap(help_heading = Some("OTHER-ACTIONS"), short = 'j', long = "json-timeline")] + /// Save the timeline in JSON format (ex: -j -o results.json) + #[clap(help_heading = Some("OUTPUT"), short = 'j', long = "json", requires = "output")] pub json_timeline: bool, } From f300a07486a5c0ddc5d7712613eb2c2b0074e79d Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Wed, 24 Aug 2022 01:04:16 +0900 Subject: [PATCH 17/40] fixed comment typo --- src/afterfact.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 7720d990..75061d52 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -833,7 +833,7 @@ fn output_json_str( } output_value_stock.push_str(&value.join(" ")); if value_idx < stocked_value.len() - 1 && stocked_value[value_idx + 1].is_empty() { - // 次の要素を確認して、存在しないもしくは、キーが入っているとなった場合現在ストックしている内容が出力していいことが確定するので主力処理を行う + // 次の要素を確認して、存在しないもしくは、キーが入っているとなった場合現在ストックしている内容が出力していいことが確定するので出力処理を行う let output_tmp = format!("{}: {}", tmp, output_value_stock); let output: Vec<&str> = output_tmp.split(": ").collect(); let key = _convert_valid_json_str(&[output[0]]); From ce700f9c18949b9a67028cbe7201f90f780bc9f0 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Wed, 24 Aug 2022 01:04:40 +0900 Subject: [PATCH 18/40] added json output to Mitretactics, MitreTags, and OtherTags --- src/afterfact.rs | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/afterfact.rs b/src/afterfact.rs index 75061d52..c251fe13 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -848,6 +848,41 @@ fn output_json_str( tmp = String::default(); } } + } else if output_value_fmt.contains("%MitreTags%") || output_value_fmt.contains("%MitreTactics%") || output_value_fmt.contains("%OtherTags%") { + println!("k dbg | {} | v dbg | {:?}", k, v); + let tmp_val: Vec<&str> = v.split(": ").collect(); + println!("tmp_val dbg | {:?} | len dbg | {:?}", tmp_val, tmp_val.len()); + + let key = _convert_valid_json_str(&[k.as_str()]); + let values = &tmp_val; + let mut value:Vec = vec![]; + let mut valid_data_cnt =0; + + if tmp_val.is_empty() { + value.push("[]".to_string()); + } + for (idx, tag_val) in values.iter().enumerate() { + if idx == 0 { + value.push("[".to_string()); + } + if tag_val == &"" { + continue; + } + let insert_val = format!(" \"{}\"", tag_val.trim()); + value.push(insert_val); + if idx != values.len() - 1 { + value.push(",\n".to_string()); + } + valid_data_cnt += 1; + } + if valid_data_cnt > 0 { + value.push(" ]".to_string()); + } else { + value.push("]".to_string()); + } + + let fmted_val = value.join(""); + target.push(_create_json_output_format(&key, &fmted_val, key.starts_with('\"'), true)); } } target.join(",\n") From 6aeda740d9938c6c186d095285c3a7a9a66458b9 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Wed, 24 Aug 2022 01:05:37 +0900 Subject: [PATCH 19/40] cargo fmt --- src/afterfact.rs | 28 ++++++++++++++++++++-------- src/detections/configs.rs | 1 - 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index c251fe13..703e8fb8 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -848,19 +848,26 @@ fn output_json_str( tmp = String::default(); } } - } else if output_value_fmt.contains("%MitreTags%") || output_value_fmt.contains("%MitreTactics%") || output_value_fmt.contains("%OtherTags%") { + } else if output_value_fmt.contains("%MitreTags%") + || output_value_fmt.contains("%MitreTactics%") + || output_value_fmt.contains("%OtherTags%") + { println!("k dbg | {} | v dbg | {:?}", k, v); let tmp_val: Vec<&str> = v.split(": ").collect(); - println!("tmp_val dbg | {:?} | len dbg | {:?}", tmp_val, tmp_val.len()); + println!( + "tmp_val dbg | {:?} | len dbg | {:?}", + tmp_val, + tmp_val.len() + ); let key = _convert_valid_json_str(&[k.as_str()]); let values = &tmp_val; - let mut value:Vec = vec![]; - let mut valid_data_cnt =0; + let mut value: Vec = vec![]; + let mut valid_data_cnt = 0; if tmp_val.is_empty() { value.push("[]".to_string()); - } + } for (idx, tag_val) in values.iter().enumerate() { if idx == 0 { value.push("[".to_string()); @@ -870,19 +877,24 @@ fn output_json_str( } let insert_val = format!(" \"{}\"", tag_val.trim()); value.push(insert_val); - if idx != values.len() - 1 { + if idx != values.len() - 1 { value.push(",\n".to_string()); } valid_data_cnt += 1; } if valid_data_cnt > 0 { value.push(" ]".to_string()); - } else { + } else { value.push("]".to_string()); } let fmted_val = value.join(""); - target.push(_create_json_output_format(&key, &fmted_val, key.starts_with('\"'), true)); + target.push(_create_json_output_format( + &key, + &fmted_val, + key.starts_with('\"'), + true, + )); } } target.join(",\n") diff --git a/src/detections/configs.rs b/src/detections/configs.rs index f3d1a5e3..cbd7c412 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -224,7 +224,6 @@ pub struct Config { #[clap(help_heading = Some("OTHER-ACTIONS"), long = "set-default-profile", value_name = "PROFILE")] pub set_default_profile: Option, - /// Save the timeline in JSON format (ex: -j -o results.json) #[clap(help_heading = Some("OUTPUT"), short = 'j', long = "json", requires = "output")] pub json_timeline: bool, From cba61b19e99c1d57c1a41e13c6925f95407de5d2 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Wed, 24 Aug 2022 01:31:28 +0900 Subject: [PATCH 20/40] fixed JSON Format MiterTactics , MiterTags, and OtherTags --- src/afterfact.rs | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 703e8fb8..879faf4a 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -854,38 +854,28 @@ fn output_json_str( { println!("k dbg | {} | v dbg | {:?}", k, v); let tmp_val: Vec<&str> = v.split(": ").collect(); - println!( - "tmp_val dbg | {:?} | len dbg | {:?}", - tmp_val, - tmp_val.len() - ); let key = _convert_valid_json_str(&[k.as_str()]); - let values = &tmp_val; + let values: Vec<&&str> = tmp_val.iter().filter(|x| x.trim() != "").collect(); let mut value: Vec = vec![]; - let mut valid_data_cnt = 0; - - if tmp_val.is_empty() { - value.push("[]".to_string()); + + if values.is_empty() { + value.push("[".to_string()); } for (idx, tag_val) in values.iter().enumerate() { if idx == 0 { - value.push("[".to_string()); - } - if tag_val == &"" { - continue; + value.push("[\n".to_string()); } let insert_val = format!(" \"{}\"", tag_val.trim()); value.push(insert_val); if idx != values.len() - 1 { value.push(",\n".to_string()); } - valid_data_cnt += 1; } - if valid_data_cnt > 0 { - value.push(" ]".to_string()); - } else { + if values.is_empty() { value.push("]".to_string()); + } else { + value.push("\n ]".to_string()); } let fmted_val = value.join(""); From 59fb1b8224a8f8fbc7acc6616c96259633a80377 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Wed, 24 Aug 2022 01:32:11 +0900 Subject: [PATCH 21/40] cargo fmt --- src/afterfact.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 879faf4a..49cb4460 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -858,7 +858,7 @@ fn output_json_str( let key = _convert_valid_json_str(&[k.as_str()]); let values: Vec<&&str> = tmp_val.iter().filter(|x| x.trim() != "").collect(); let mut value: Vec = vec![]; - + if values.is_empty() { value.push("[".to_string()); } From ac2c362d48749fbaf5cc616f7ddd5008d678f663 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Thu, 25 Aug 2022 00:47:19 +0900 Subject: [PATCH 22/40] removed debug output --- src/afterfact.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 49cb4460..f19913b1 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -852,7 +852,6 @@ fn output_json_str( || output_value_fmt.contains("%MitreTactics%") || output_value_fmt.contains("%OtherTags%") { - println!("k dbg | {} | v dbg | {:?}", k, v); let tmp_val: Vec<&str> = v.split(": ").collect(); let key = _convert_valid_json_str(&[k.as_str()]); From cbf61d155b53aac15babefa9a0c7397ab542969a Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Thu, 25 Aug 2022 00:48:41 +0900 Subject: [PATCH 23/40] omitted output when tag data is none --- src/afterfact.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index f19913b1..0359aa79 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -859,7 +859,7 @@ fn output_json_str( let mut value: Vec = vec![]; if values.is_empty() { - value.push("[".to_string()); + continue; } for (idx, tag_val) in values.iter().enumerate() { if idx == 0 { @@ -871,11 +871,7 @@ fn output_json_str( value.push(",\n".to_string()); } } - if values.is_empty() { - value.push("]".to_string()); - } else { - value.push("\n ]".to_string()); - } + value.push("\n ]".to_string()); let fmted_val = value.join(""); target.push(_create_json_output_format( From adfc7812c028a01ec58c9b90514b92b47495ecda Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Sat, 27 Aug 2022 22:58:15 +0900 Subject: [PATCH 24/40] adjusted only exist key in Details #654 --- src/afterfact.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 0359aa79..00e3a65d 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -813,9 +813,7 @@ fn output_json_str( tmp_stock.push(sp.to_owned()); } } - if !tmp_stock.is_empty() { - stocked_value.push(tmp_stock); - } + stocked_value.push(tmp_stock); } let mut tmp = String::default(); let mut key_idx = 0; From 430a910320ae74b351122e2905a9e07b4c04e42c Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Thu, 8 Sep 2022 00:32:49 +0900 Subject: [PATCH 25/40] fixed output lack json when last stock value #654 --- src/afterfact.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/afterfact.rs b/src/afterfact.rs index cb4e5aec..01428368 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -922,6 +922,18 @@ fn output_json_str( )); output_value_stock.clear(); tmp = String::default(); + } + if value_idx == stocked_value.len() -1 { + let output_tmp = format!("{}: {}", tmp, output_value_stock); + let output: Vec<&str> = output_tmp.split(": ").collect(); + let key = _convert_valid_json_str(&[output[0]]); + let fmted_val = _convert_valid_json_str(&output); + target.push(_create_json_output_format( + &key, + &fmted_val, + key.starts_with('\"'), + fmted_val.starts_with('\"'), + )); } } } else if output_value_fmt.contains("%MitreTags%") From 6cecc74c03deb8d2e99d523e137f3c551c7b5c58 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Thu, 8 Sep 2022 00:36:01 +0900 Subject: [PATCH 26/40] cargo fmt --- src/afterfact.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 01428368..eac19626 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -7,10 +7,10 @@ use crate::detections::utils::{get_writable_color, write_color_buffer}; use crate::options::profile::PROFILES; use bytesize::ByteSize; use chrono::{DateTime, Local, TimeZone, Utc}; -use core::cmp::max; -use csv::{QuoteStyle, WriterBuilder}; use comfy_table::modifiers::UTF8_ROUND_CORNERS; use comfy_table::presets::UTF8_FULL; +use core::cmp::max; +use csv::{QuoteStyle, WriterBuilder}; use itertools::Itertools; use krapslog::{build_sparkline, build_time_markers}; use lazy_static::lazy_static; @@ -922,8 +922,8 @@ fn output_json_str( )); output_value_stock.clear(); tmp = String::default(); - } - if value_idx == stocked_value.len() -1 { + } + if value_idx == stocked_value.len() - 1 { let output_tmp = format!("{}: {}", tmp, output_value_stock); let output: Vec<&str> = output_tmp.split(": ").collect(); let key = _convert_valid_json_str(&[output[0]]); From 7996313f6914ec138428452cbd5e069825470be9 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Thu, 8 Sep 2022 00:51:58 +0900 Subject: [PATCH 27/40] fixed error --- src/afterfact.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/afterfact.rs b/src/afterfact.rs index eac19626..b865175d 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -923,6 +923,21 @@ fn output_json_str( output_value_stock.clear(); tmp = String::default(); } + if value_idx < stocked_value.len() - 1 && stocked_value[value_idx + 1].is_empty() { + // 次の要素を確認して、存在しないもしくは、キーが入っているとなった場合現在ストックしている内容が出力していいことが確定するので出力処理を行う + let output_tmp = format!("{}: {}", tmp, output_value_stock); + let output: Vec<&str> = output_tmp.split(": ").collect(); + let key = _convert_valid_json_str(&[output[0]]); + let fmted_val = _convert_valid_json_str(&output); + target.push(_create_json_output_format( + &key, + &fmted_val, + key.starts_with('\"'), + fmted_val.starts_with('\"'), + )); + output_value_stock.clear(); + tmp = String::default(); + } if value_idx == stocked_value.len() - 1 { let output_tmp = format!("{}: {}", tmp, output_value_stock); let output: Vec<&str> = output_tmp.split(": ").collect(); From 2c707a85a3042709e8a73731e76554392955c290 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Thu, 8 Sep 2022 01:05:41 +0900 Subject: [PATCH 28/40] removed duplicated process --- src/afterfact.rs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index b865175d..eac19626 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -923,21 +923,6 @@ fn output_json_str( output_value_stock.clear(); tmp = String::default(); } - if value_idx < stocked_value.len() - 1 && stocked_value[value_idx + 1].is_empty() { - // 次の要素を確認して、存在しないもしくは、キーが入っているとなった場合現在ストックしている内容が出力していいことが確定するので出力処理を行う - let output_tmp = format!("{}: {}", tmp, output_value_stock); - let output: Vec<&str> = output_tmp.split(": ").collect(); - let key = _convert_valid_json_str(&[output[0]]); - let fmted_val = _convert_valid_json_str(&output); - target.push(_create_json_output_format( - &key, - &fmted_val, - key.starts_with('\"'), - fmted_val.starts_with('\"'), - )); - output_value_stock.clear(); - tmp = String::default(); - } if value_idx == stocked_value.len() - 1 { let output_tmp = format!("{}: {}", tmp, output_value_stock); let output: Vec<&str> = output_tmp.split(": ").collect(); From 6e9709170b5ac746ddd42485c0f8287df33ec91b Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Thu, 8 Sep 2022 01:48:00 +0900 Subject: [PATCH 29/40] fixed process contiguous empty value in array when output json #654 --- src/afterfact.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index eac19626..40fa3353 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -897,7 +897,7 @@ fn output_json_str( let mut key_idx = 0; let mut output_value_stock = String::default(); for (value_idx, value) in stocked_value.iter().enumerate() { - if value.is_empty() { + if value.is_empty() && value_idx >= 1 && !stocked_value[value_idx - 1].is_empty() { tmp = key_index_stock[key_idx].to_string(); key_idx += 1; continue; From 2e4418dce12bad8a7ebbc01b238b05f214318432 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 9 Sep 2022 22:35:32 +0900 Subject: [PATCH 30/40] changed details separator character --- src/afterfact.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 40fa3353..f5477f06 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -786,7 +786,7 @@ fn _get_json_vec(target_alias_context: &str, target_data: &String) -> Vec = target_data .to_owned() - .split(" | ") + .split(" ¦ ") .map(|x| x.to_string()) .collect(); if target_data == &ret[0] && !target_data.contains(": ") { From 6cbf74a4cb0931078406a0d4f7d3e35ec071f99e Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Mon, 12 Sep 2022 00:13:19 +0900 Subject: [PATCH 31/40] fixed key processing in details #654 --- src/afterfact.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index cd1b8d4b..c1b0e0cc 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -926,7 +926,7 @@ fn output_json_str( let space_split: Vec<&str> = detail_contents.split(' ').collect(); let mut tmp_stock = vec![]; for sp in space_split.iter() { - if sp.ends_with(':') && sp != &":" && sp.len() != 2 { + if sp.ends_with(':') && sp != &":" { stocked_value.push(tmp_stock); tmp_stock = vec![]; key_index_stock.push(sp.replace(':', "").to_owned()); @@ -936,16 +936,15 @@ fn output_json_str( } stocked_value.push(tmp_stock); } - let mut tmp = String::default(); let mut key_idx = 0; let mut output_value_stock = String::default(); for (value_idx, value) in stocked_value.iter().enumerate() { - if value.is_empty() && value_idx >= 1 && !stocked_value[value_idx - 1].is_empty() { - tmp = key_index_stock[key_idx].to_string(); + let mut tmp = key_index_stock[key_idx].to_string(); + if value_idx == 0 && !value.is_empty() { + tmp = k.to_string(); + } else if value.is_empty() && value_idx >= 1 && !stocked_value[value_idx - 1].is_empty() { key_idx += 1; continue; - } else if value_idx == 0 { - tmp = k.to_string(); } if !output_value_stock.is_empty() { output_value_stock.push_str(" | "); From e55a1198463546e5d6a6705232e3a57878d78ebc Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Mon, 12 Sep 2022 00:14:04 +0900 Subject: [PATCH 32/40] cargo fmt --- src/afterfact.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index c1b0e0cc..319df737 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -942,7 +942,10 @@ fn output_json_str( let mut tmp = key_index_stock[key_idx].to_string(); if value_idx == 0 && !value.is_empty() { tmp = k.to_string(); - } else if value.is_empty() && value_idx >= 1 && !stocked_value[value_idx - 1].is_empty() { + } else if value.is_empty() + && value_idx >= 1 + && !stocked_value[value_idx - 1].is_empty() + { key_idx += 1; continue; } From 3c618db6618f037bfcfc37896e018777eafc1905 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Mon, 12 Sep 2022 00:30:28 +0900 Subject: [PATCH 33/40] fixed double quote json parse error in allrecordinfo field --- src/afterfact.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 319df737..d8a25f76 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -912,11 +912,12 @@ fn output_json_str( let vec_data = _get_json_vec(output_value_fmt, v); if vec_data.is_empty() { let tmp_val: Vec<&str> = v.split(": ").collect(); + let output_val = _convert_valid_json_str(&tmp_val); target.push(_create_json_output_format( k, - &_convert_valid_json_str(&tmp_val), + &output_val, k.starts_with('\"'), - v.starts_with('\"'), + output_val.starts_with('\"'), )); } else if output_value_fmt.contains("%Details%") { let mut stocked_value = vec![]; From c96ac1d0e58edd13566c95258a6d820948ea285f Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Mon, 12 Sep 2022 23:17:48 +0900 Subject: [PATCH 34/40] fixed object separate comma lack --- src/afterfact.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index d8a25f76..22efaecb 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -11,6 +11,7 @@ use comfy_table::modifiers::UTF8_ROUND_CORNERS; use comfy_table::presets::UTF8_FULL; use core::cmp::max; use csv::{QuoteStyle, WriterBuilder}; +use dashmap::Map; use itertools::Itertools; use krapslog::{build_sparkline, build_time_markers}; use lazy_static::lazy_static; @@ -263,7 +264,7 @@ fn emit_csv( let multi = message::MESSAGES.get(time).unwrap(); let (_, detect_infos) = multi.pair(); timestamps.push(_get_timestamp(time)); - for detect_info in detect_infos { + for (info_idx, detect_info) in detect_infos.iter().enumerate() { if !detect_info.detail.starts_with("[condition]") { detected_record_idset.insert(format!("{}_{}", time, detect_info.eventid)); } @@ -294,10 +295,12 @@ fn emit_csv( } else if json_output_flag { wtr.write_field(" {")?; wtr.write_field(&output_json_str(&detect_info.ext_field, &profile))?; - if processed_message_cnt == message::MESSAGES.len() - 1 { - wtr.write_field(" }")?; - } else { + if processed_message_cnt != message::MESSAGES._len() - 1 + || info_idx != detect_infos.len() - 1 + { wtr.write_field(" },")?; + } else { + wtr.write_field(" }")?; } } else { // csv output format From 0b489c5ca2456ae9574f1e4abf997a5509646ee9 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Tue, 13 Sep 2022 20:15:47 +0900 Subject: [PATCH 35/40] fixed key index crash --- src/afterfact.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 22efaecb..30ba339f 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -943,7 +943,11 @@ fn output_json_str( let mut key_idx = 0; let mut output_value_stock = String::default(); for (value_idx, value) in stocked_value.iter().enumerate() { - let mut tmp = key_index_stock[key_idx].to_string(); + let mut tmp = if key_idx >= key_index_stock.len() { + String::default() + } else { + key_index_stock[key_idx].to_string() + }; if value_idx == 0 && !value.is_empty() { tmp = k.to_string(); } else if value.is_empty() From 93070e8fcd50dc25799249efa0a233020d7ada63 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Wed, 14 Sep 2022 18:55:12 +0900 Subject: [PATCH 36/40] fixed lack of first key data in recordinformation --- src/afterfact.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 30ba339f..d72310c1 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -870,10 +870,12 @@ fn _create_json_output_format( } /// JSONの値に対して文字列の出力形式をJSON出力でエラーにならないようにするための変換を行う関数 -fn _convert_valid_json_str(input: &[&str]) -> String { +fn _convert_valid_json_str(input: &[&str], concat_flag: bool) -> String { let tmp = if input.len() == 1 { input[0].to_string() - } else { + } else if concat_flag { + input.join(": ") + } else{ input[1..].join(": ") }; let char_cnt = tmp.char_indices().count(); @@ -915,7 +917,7 @@ fn output_json_str( let vec_data = _get_json_vec(output_value_fmt, v); if vec_data.is_empty() { let tmp_val: Vec<&str> = v.split(": ").collect(); - let output_val = _convert_valid_json_str(&tmp_val); + let output_val = _convert_valid_json_str(&tmp_val, output_value_fmt.contains("%RecordInformation%")); target.push(_create_json_output_format( k, &output_val, @@ -965,8 +967,8 @@ fn output_json_str( // 次の要素を確認して、存在しないもしくは、キーが入っているとなった場合現在ストックしている内容が出力していいことが確定するので出力処理を行う let output_tmp = format!("{}: {}", tmp, output_value_stock); let output: Vec<&str> = output_tmp.split(": ").collect(); - let key = _convert_valid_json_str(&[output[0]]); - let fmted_val = _convert_valid_json_str(&output); + let key = _convert_valid_json_str(&[output[0]], false); + let fmted_val = _convert_valid_json_str(&output, false); target.push(_create_json_output_format( &key, &fmted_val, @@ -979,8 +981,8 @@ fn output_json_str( if value_idx == stocked_value.len() - 1 { let output_tmp = format!("{}: {}", tmp, output_value_stock); let output: Vec<&str> = output_tmp.split(": ").collect(); - let key = _convert_valid_json_str(&[output[0]]); - let fmted_val = _convert_valid_json_str(&output); + let key = _convert_valid_json_str(&[output[0]], false); + let fmted_val = _convert_valid_json_str(&output, false); target.push(_create_json_output_format( &key, &fmted_val, @@ -995,7 +997,7 @@ fn output_json_str( { let tmp_val: Vec<&str> = v.split(": ").collect(); - let key = _convert_valid_json_str(&[k.as_str()]); + let key = _convert_valid_json_str(&[k.as_str()], false); let values: Vec<&&str> = tmp_val.iter().filter(|x| x.trim() != "").collect(); let mut value: Vec = vec![]; From 97b6a35c88b7957ac3b9986083818b333bb0b91a Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Wed, 14 Sep 2022 18:55:49 +0900 Subject: [PATCH 37/40] cargo fmt --- src/afterfact.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index d72310c1..860abab4 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -875,7 +875,7 @@ fn _convert_valid_json_str(input: &[&str], concat_flag: bool) -> String { input[0].to_string() } else if concat_flag { input.join(": ") - } else{ + } else { input[1..].join(": ") }; let char_cnt = tmp.char_indices().count(); @@ -917,7 +917,8 @@ fn output_json_str( let vec_data = _get_json_vec(output_value_fmt, v); if vec_data.is_empty() { let tmp_val: Vec<&str> = v.split(": ").collect(); - let output_val = _convert_valid_json_str(&tmp_val, output_value_fmt.contains("%RecordInformation%")); + let output_val = + _convert_valid_json_str(&tmp_val, output_value_fmt.contains("%RecordInformation%")); target.push(_create_json_output_format( k, &output_val, From 709fc5bf15c91c9f3d66e7479330afed24ce9bf2 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Thu, 15 Sep 2022 19:44:45 +0900 Subject: [PATCH 38/40] fixed json output parse error --- src/afterfact.rs | 49 +++++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 860abab4..10abd563 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -9,7 +9,7 @@ use bytesize::ByteSize; use chrono::{DateTime, Local, TimeZone, Utc}; use comfy_table::modifiers::UTF8_ROUND_CORNERS; use comfy_table::presets::UTF8_FULL; -use core::cmp::max; + use csv::{QuoteStyle, WriterBuilder}; use dashmap::Map; use itertools::Itertools; @@ -883,17 +883,17 @@ fn _convert_valid_json_str(input: &[&str], concat_flag: bool) -> String { if char_cnt == 0 { tmp } else if con_val.starts_with('\"') { - let (head, _) = tmp.char_indices().nth(min(1, char_cnt - 1)).unwrap(); - let (tail, _) = tmp.char_indices().nth(max(0, char_cnt - 1)).unwrap(); - let addition_quote = if !con_val.ends_with('\"') { "\"" } else { "" }; + let addition_header = if !con_val.starts_with('\"') {"\""} else {""}; + let addition_quote = if !con_val.ends_with('\"') && concat_flag { "\"" } else if !con_val.ends_with('\"') { + "\\\"" + } else { "" }; [ - &con_val[..head], - con_val[head..tail] + addition_header, + con_val .to_string() .replace('\\', "\\\\") .replace('\"', "\\\"") .trim(), - &con_val[tail..], addition_quote, ] .join("") @@ -945,26 +945,39 @@ fn output_json_str( } let mut key_idx = 0; let mut output_value_stock = String::default(); + println!("kis dbg | {:?}", key_index_stock); + println!("stv dbg | {:?}", stocked_value); for (value_idx, value) in stocked_value.iter().enumerate() { let mut tmp = if key_idx >= key_index_stock.len() { String::default() + } else if value_idx == 0 && !value.is_empty(){ + k.to_string() } else { key_index_stock[key_idx].to_string() }; - if value_idx == 0 && !value.is_empty() { - tmp = k.to_string(); - } else if value.is_empty() - && value_idx >= 1 - && !stocked_value[value_idx - 1].is_empty() - { - key_idx += 1; - continue; - } + // if value.is_empty() + // && value_idx >= 1 + // && !stocked_value[value_idx - 1].is_empty() + // { + // key_idx += 1; + // continue; + // } if !output_value_stock.is_empty() { output_value_stock.push_str(" | "); } output_value_stock.push_str(&value.join(" ")); - if value_idx < stocked_value.len() - 1 && stocked_value[value_idx + 1].is_empty() { + //``1つまえのキーの段階で以降にvalueの配列で区切りとなる空の配列が存在しているかを確認する + let is_remain_split_stock = if key_idx == key_index_stock.len() - 2 && value_idx < stocked_value.len() - 1 && !output_value_stock.is_empty() { + let mut ret = true; + for remain_value in stocked_value[value_idx + 1 ..].iter() { + println!("|dbg rv {:?}|", remain_value); + if remain_value.is_empty() { ret = false;break;} + } + ret + } else { + false + }; + if (value_idx < stocked_value.len() - 1 && stocked_value[value_idx + 1].is_empty()) || is_remain_split_stock { // 次の要素を確認して、存在しないもしくは、キーが入っているとなった場合現在ストックしている内容が出力していいことが確定するので出力処理を行う let output_tmp = format!("{}: {}", tmp, output_value_stock); let output: Vec<&str> = output_tmp.split(": ").collect(); @@ -978,6 +991,7 @@ fn output_json_str( )); output_value_stock.clear(); tmp = String::default(); + key_idx += 1; } if value_idx == stocked_value.len() - 1 { let output_tmp = format!("{}: {}", tmp, output_value_stock); @@ -990,6 +1004,7 @@ fn output_json_str( key.starts_with('\"'), fmted_val.starts_with('\"'), )); + key_idx += 1; } } } else if output_value_fmt.contains("%MitreTags%") From 8e76c942cbffa30f2481c79d91be407a28439c45 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Thu, 15 Sep 2022 19:45:18 +0900 Subject: [PATCH 39/40] cargo fmt --- src/afterfact.rs | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 10abd563..57d81bf5 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -883,10 +883,14 @@ fn _convert_valid_json_str(input: &[&str], concat_flag: bool) -> String { if char_cnt == 0 { tmp } else if con_val.starts_with('\"') { - let addition_header = if !con_val.starts_with('\"') {"\""} else {""}; - let addition_quote = if !con_val.ends_with('\"') && concat_flag { "\"" } else if !con_val.ends_with('\"') { + let addition_header = if !con_val.starts_with('\"') { "\"" } else { "" }; + let addition_quote = if !con_val.ends_with('\"') && concat_flag { + "\"" + } else if !con_val.ends_with('\"') { "\\\"" - } else { "" }; + } else { + "" + }; [ addition_header, con_val @@ -950,7 +954,7 @@ fn output_json_str( for (value_idx, value) in stocked_value.iter().enumerate() { let mut tmp = if key_idx >= key_index_stock.len() { String::default() - } else if value_idx == 0 && !value.is_empty(){ + } else if value_idx == 0 && !value.is_empty() { k.to_string() } else { key_index_stock[key_idx].to_string() @@ -967,17 +971,25 @@ fn output_json_str( } output_value_stock.push_str(&value.join(" ")); //``1つまえのキーの段階で以降にvalueの配列で区切りとなる空の配列が存在しているかを確認する - let is_remain_split_stock = if key_idx == key_index_stock.len() - 2 && value_idx < stocked_value.len() - 1 && !output_value_stock.is_empty() { + let is_remain_split_stock = if key_idx == key_index_stock.len() - 2 + && value_idx < stocked_value.len() - 1 + && !output_value_stock.is_empty() + { let mut ret = true; - for remain_value in stocked_value[value_idx + 1 ..].iter() { + for remain_value in stocked_value[value_idx + 1..].iter() { println!("|dbg rv {:?}|", remain_value); - if remain_value.is_empty() { ret = false;break;} + if remain_value.is_empty() { + ret = false; + break; + } } ret } else { false }; - if (value_idx < stocked_value.len() - 1 && stocked_value[value_idx + 1].is_empty()) || is_remain_split_stock { + if (value_idx < stocked_value.len() - 1 && stocked_value[value_idx + 1].is_empty()) + || is_remain_split_stock + { // 次の要素を確認して、存在しないもしくは、キーが入っているとなった場合現在ストックしている内容が出力していいことが確定するので出力処理を行う let output_tmp = format!("{}: {}", tmp, output_value_stock); let output: Vec<&str> = output_tmp.split(": ").collect(); From f7fd837bf8496854deae9bec992f954a02cd0dae Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Thu, 15 Sep 2022 19:46:14 +0900 Subject: [PATCH 40/40] removed debug print and comment out process --- src/afterfact.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 57d81bf5..ff53bb9a 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -949,8 +949,6 @@ fn output_json_str( } let mut key_idx = 0; let mut output_value_stock = String::default(); - println!("kis dbg | {:?}", key_index_stock); - println!("stv dbg | {:?}", stocked_value); for (value_idx, value) in stocked_value.iter().enumerate() { let mut tmp = if key_idx >= key_index_stock.len() { String::default() @@ -959,13 +957,6 @@ fn output_json_str( } else { key_index_stock[key_idx].to_string() }; - // if value.is_empty() - // && value_idx >= 1 - // && !stocked_value[value_idx - 1].is_empty() - // { - // key_idx += 1; - // continue; - // } if !output_value_stock.is_empty() { output_value_stock.push_str(" | "); } @@ -977,7 +968,6 @@ fn output_json_str( { let mut ret = true; for remain_value in stocked_value[value_idx + 1..].iter() { - println!("|dbg rv {:?}|", remain_value); if remain_value.is_empty() { ret = false; break;