From c6fc18a5fbcc92f2cf95234ce4d23dbacbc719a3 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 16 Sep 2022 00:01:53 +0900 Subject: [PATCH 1/6] added JSONL output option #694 --- src/afterfact.rs | 35 +++++++++++++++++++++++++++-------- src/detections/configs.rs | 4 ++++ 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index ff53bb9a..2cda80b4 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -213,8 +213,9 @@ fn emit_csv( let disp_wtr = BufferWriter::stdout(ColorChoice::Always); let mut disp_wtr_buf = disp_wtr.buffer(); let json_output_flag = configs::CONFIG.read().unwrap().args.json_timeline; + let jsonl_output_flag = configs::CONFIG.read().unwrap().args.jsonl_timeline; - let mut wtr = if json_output_flag { + let mut wtr = if json_output_flag || jsonl_output_flag { WriterBuilder::new() .delimiter(b'\n') .double_quote(false) @@ -293,8 +294,9 @@ fn emit_csv( ) .ok(); } else if json_output_flag { + // JSON output wtr.write_field(" {")?; - wtr.write_field(&output_json_str(&detect_info.ext_field, &profile))?; + wtr.write_field(&output_json_str(&detect_info.ext_field, &profile, jsonl_output_flag))?; if processed_message_cnt != message::MESSAGES._len() - 1 || info_idx != detect_infos.len() - 1 { @@ -302,6 +304,9 @@ fn emit_csv( } else { wtr.write_field(" }")?; } + } else if jsonl_output_flag { + // JSONL output format + wtr.write_field(format!("{{ {} }}", &output_json_str(&detect_info.ext_field, &profile, jsonl_output_flag)))?; } else { // csv output format if plus_header { @@ -914,6 +919,7 @@ fn _convert_valid_json_str(input: &[&str], concat_flag: bool) -> String { fn output_json_str( ext_field: &LinkedHashMap, profile: &LinkedHashMap, + jsonl_output_flag: bool, ) -> String { let mut target: Vec = vec![]; for (k, v) in ext_field.iter() { @@ -921,8 +927,7 @@ fn output_json_str( let vec_data = _get_json_vec(output_value_fmt, v); if vec_data.is_empty() { let tmp_val: Vec<&str> = v.split(": ").collect(); - let output_val = - _convert_valid_json_str(&tmp_val, output_value_fmt.contains("%RecordInformation%")); + let output_val = _convert_valid_json_str(&tmp_val, output_value_fmt.contains("%RecordInformation%")); target.push(_create_json_output_format( k, &output_val, @@ -984,7 +989,9 @@ fn output_json_str( let output_tmp = format!("{}: {}", tmp, output_value_stock); let output: Vec<&str> = output_tmp.split(": ").collect(); let key = _convert_valid_json_str(&[output[0]], false); - let fmted_val = _convert_valid_json_str(&output, false); + let fmted_val = + _convert_valid_json_str(&output, false) + ; target.push(_create_json_output_format( &key, &fmted_val, @@ -999,7 +1006,9 @@ fn output_json_str( let output_tmp = format!("{}: {}", tmp, output_value_stock); let output: Vec<&str> = output_tmp.split(": ").collect(); let key = _convert_valid_json_str(&[output[0]], false); - let fmted_val = _convert_valid_json_str(&output, false); + let fmted_val = + _convert_valid_json_str(&output, false) + ; target.push(_create_json_output_format( &key, &fmted_val, @@ -1034,7 +1043,11 @@ fn output_json_str( } value.push("\n ]".to_string()); - let fmted_val = value.join(""); + let fmted_val = if jsonl_output_flag { + value.iter().map(|x| x.replace('\n', "")).join("") + } else { + value.join("") + }; target.push(_create_json_output_format( &key, &fmted_val, @@ -1043,7 +1056,13 @@ fn output_json_str( )); } } - target.join(",\n") + if jsonl_output_flag { + // JSONL output + target.into_iter().map(|x| x.replace(" ", "")).join(",") + } else { + // JSON format output + target.join(",\n") + } } #[cfg(test)] diff --git a/src/detections/configs.rs b/src/detections/configs.rs index 73392bfc..f1849a65 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -241,6 +241,10 @@ pub struct Config { #[clap(help_heading = Some("OUTPUT"), short = 'j', long = "json", requires = "output")] pub json_timeline: bool, + /// Save the timeline in JSONL format (ex: -J -o results.jsonl) + #[clap(help_heading = Some("OUTPUT"), short = 'J', long = "jsonl", requires = "output")] + pub jsonl_timeline: bool, + /// Do not display result summary #[clap(help_heading = Some("DISPLAY-SETTINGS"), long = "no-summary")] pub no_summary: bool, From 2f31e6bb4a6c27431ccb196ed9b21c401d0ee881 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 16 Sep 2022 00:06:41 +0900 Subject: [PATCH 2/6] updated changelog #694 --- CHANGELOG-Japanese.md | 1 + CHANGELOG.md | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG-Japanese.md b/CHANGELOG-Japanese.md index 752182ab..1c2a7144 100644 --- a/CHANGELOG-Japanese.md +++ b/CHANGELOG-Japanese.md @@ -5,6 +5,7 @@ **新機能:** - 解析結果をJSONに出力する機能を追加した (`-j` と `--json-timeline` )。 (#654) (@hitenkoku) +- 解析結果をJSONL形式で出力する機能を追加した (`-J` と `--jsonl` )。 (#694) (@hitenkoku) **改善:** diff --git a/CHANGELOG.md b/CHANGELOG.md index 27666c44..abf944ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ **New Features:** - You can now save the timeline to JSON files with the `-j, --json-timeline` option. (#654) (@hitenkoku) +- You can now save the timeline to JSONL files with the `-J, --jsonl` option. (#694) (@hitenkoku) **Enhancements:** From 0817dec06437f6549c624c0cd2e744995966c430 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 16 Sep 2022 00:07:30 +0900 Subject: [PATCH 3/6] fixed wrong json long option in changelog --- CHANGELOG-Japanese.md | 2 +- CHANGELOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG-Japanese.md b/CHANGELOG-Japanese.md index 1c2a7144..b6e96246 100644 --- a/CHANGELOG-Japanese.md +++ b/CHANGELOG-Japanese.md @@ -4,7 +4,7 @@ **新機能:** -- 解析結果をJSONに出力する機能を追加した (`-j` と `--json-timeline` )。 (#654) (@hitenkoku) +- 解析結果をJSON形式で出力する機能を追加した (`-j` と `--json` )。 (#654) (@hitenkoku) - 解析結果をJSONL形式で出力する機能を追加した (`-J` と `--jsonl` )。 (#694) (@hitenkoku) **改善:** diff --git a/CHANGELOG.md b/CHANGELOG.md index abf944ab..60364fd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ **New Features:** -- You can now save the timeline to JSON files with the `-j, --json-timeline` option. (#654) (@hitenkoku) +- You can now save the timeline to JSON files with the `-j, --json` option. (#654) (@hitenkoku) - You can now save the timeline to JSONL files with the `-J, --jsonl` option. (#694) (@hitenkoku) **Enhancements:** From 41f8527f073804844b58ff9f3d49c949205c68a0 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 16 Sep 2022 00:12:59 +0900 Subject: [PATCH 4/6] added -J, --jsonl option in Readme usage #694 --- README-Japanese.md | 1 + README.md | 1 + 2 files changed, 2 insertions(+) diff --git a/README-Japanese.md b/README-Japanese.md index 5b0f02aa..23c99035 100644 --- a/README-Japanese.md +++ b/README-Japanese.md @@ -389,6 +389,7 @@ ADVANCED: OUTPUT: -j, --json タイムラインの出力をJSON形式で保存する（例: -j -o results.json） + -J, --jsonl タイムラインの出力をJSONL形式で保存する (例: -J -o results.jsonl) -o, --output タイムラインをCSV形式で保存する (例: results.csv) -P, --profile 利用する出力プロファイル名を指定する (minimal, standard, verbose, verbose-all-field-info, verbose-details-and-all-field-info) diff --git a/README.md b/README.md index 4f4c10b9..e02e890d 100644 --- a/README.md +++ b/README.md @@ -381,6 +381,7 @@ ADVANCED: OUTPUT: -j, --json Save the timeline in JSON format (ex: -j -o results.json) + -J, --jsonl Save the timeline in JSONL format (ex: -J -o results.jsonl) -o, --output Save the timeline in CSV format (ex: results.csv) -P, --profile Specify output profile (minimal, standard, verbose, verbose-all-field-info, verbose-details-and-all-field-info) From fcaf0a15ab721f39e71e333fb41728fe24343360 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 16 Sep 2022 00:13:31 +0900 Subject: [PATCH 5/6] fixed markdown format --- README-Japanese.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README-Japanese.md b/README-Japanese.md index 23c99035..0cc655db 100644 --- a/README-Japanese.md +++ b/README-Japanese.md @@ -359,6 +359,7 @@ macOSの環境設定から「セキュリティとプライバシー」を開き これで実行できるようになります。 # 使用方法 + ## 主なコマンド * デフォルト: ファストフォレンジックタイムラインの作成。 From d9f72af2be159db13a6fb4edc14ab6df2fcbb326 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 16 Sep 2022 12:29:49 +0900 Subject: [PATCH 6/6] cargo fmt --- src/afterfact.rs | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/afterfact.rs b/src/afterfact.rs index 2cda80b4..b8bc0ecb 100644 --- a/src/afterfact.rs +++ b/src/afterfact.rs @@ -296,7 +296,11 @@ fn emit_csv( } else if json_output_flag { // JSON output wtr.write_field(" {")?; - wtr.write_field(&output_json_str(&detect_info.ext_field, &profile, jsonl_output_flag))?; + wtr.write_field(&output_json_str( + &detect_info.ext_field, + &profile, + jsonl_output_flag, + ))?; if processed_message_cnt != message::MESSAGES._len() - 1 || info_idx != detect_infos.len() - 1 { @@ -306,7 +310,10 @@ fn emit_csv( } } else if jsonl_output_flag { // JSONL output format - wtr.write_field(format!("{{ {} }}", &output_json_str(&detect_info.ext_field, &profile, jsonl_output_flag)))?; + wtr.write_field(format!( + "{{ {} }}", + &output_json_str(&detect_info.ext_field, &profile, jsonl_output_flag) + ))?; } else { // csv output format if plus_header { @@ -927,7 +934,8 @@ fn output_json_str( let vec_data = _get_json_vec(output_value_fmt, v); if vec_data.is_empty() { let tmp_val: Vec<&str> = v.split(": ").collect(); - let output_val = _convert_valid_json_str(&tmp_val, output_value_fmt.contains("%RecordInformation%")); + let output_val = + _convert_valid_json_str(&tmp_val, output_value_fmt.contains("%RecordInformation%")); target.push(_create_json_output_format( k, &output_val, @@ -989,9 +997,7 @@ fn output_json_str( let output_tmp = format!("{}: {}", tmp, output_value_stock); let output: Vec<&str> = output_tmp.split(": ").collect(); let key = _convert_valid_json_str(&[output[0]], false); - let fmted_val = - _convert_valid_json_str(&output, false) - ; + let fmted_val = _convert_valid_json_str(&output, false); target.push(_create_json_output_format( &key, &fmted_val, @@ -1006,9 +1012,7 @@ fn output_json_str( let output_tmp = format!("{}: {}", tmp, output_value_stock); let output: Vec<&str> = output_tmp.split(": ").collect(); let key = _convert_valid_json_str(&[output[0]], false); - let fmted_val = - _convert_valid_json_str(&output, false) - ; + let fmted_val = _convert_valid_json_str(&output, false); target.push(_create_json_output_format( &key, &fmted_val,