diff --git a/Cargo.lock b/Cargo.lock index a02ea5f9..32666b64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -108,6 +108,12 @@ dependencies = [ "byteorder", ] +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + [[package]] name = "bitflags" version = "1.3.2" @@ -193,7 +199,7 @@ checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" dependencies = [ "camino", "cargo-platform", - "semver 1.0.6", + "semver 1.0.7", "serde", "serde_json", ] @@ -580,9 +586,9 @@ checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" [[package]] name = "encoding_rs" -version = "0.8.30" +version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df" +checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" dependencies = [ "cfg-if 1.0.0", ] @@ -765,9 +771,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" dependencies = [ "cfg-if 1.0.0", "libc", @@ -838,7 +844,7 @@ dependencies = [ name = "hayabusa" version = "1.2.0" dependencies = [ - "base64", + "base64 0.13.0", "chrono", "clap", "colored", @@ -1004,9 +1010,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" +checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" dependencies = [ "autocfg 1.1.0", "hashbrown 0.11.2", @@ -1108,9 +1114,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.121" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f" +checksum = "ec647867e2bf0772e28c8bcde4f0d19a9216916e890543b5a03ed8ef27b8f259" [[package]] name = "libgit2-sys" @@ -1169,10 +1175,11 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" dependencies = [ + "autocfg 1.1.0", "scopeguard", ] @@ -1303,9 +1310,9 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" dependencies = [ "lazy_static", "libc", @@ -1454,8 +1461,8 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" dependencies = [ - "lock_api 0.4.6", - "parking_lot_core 0.9.1", + "lock_api 0.4.7", + "parking_lot_core 0.9.2", ] [[package]] @@ -1475,13 +1482,13 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" +checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.2.12", + "redox_syscall 0.2.13", "smallvec 1.8.0", "windows-sys", ] @@ -1518,9 +1525,9 @@ checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" [[package]] name = "pkg-config" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" [[package]] name = "proc-macro-hack" @@ -1530,9 +1537,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" +checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" dependencies = [ "unicode-xid", ] @@ -1576,9 +1583,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "632d02bff7f874a36f33ea8bb416cd484b90cc66c1194b1a1110d067a7013f58" +checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" dependencies = [ "proc-macro2", ] @@ -1731,9 +1738,9 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_syscall" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae183fc1b06c149f0c1793e1eb447c8b04bfe46d48e9e48bfb8d2d7ed64ecf0" +checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ "bitflags", ] @@ -1776,7 +1783,7 @@ version = "0.9.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f88643aea3c1343c804950d7bf983bd2067f5ab59db6d613a08e05572f2714ab" dependencies = [ - "base64", + "base64 0.10.1", "bytes 0.4.12", "cookie", "cookie_store", @@ -1904,9 +1911,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a3381e03edd24287172047536f20cabde766e2cd3e65e6b00fb3af51c4f38d" +checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4" dependencies = [ "serde", ] @@ -2012,9 +2019,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" +checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" [[package]] name = "slack-hook" @@ -2138,9 +2145,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "syn" -version = "1.0.89" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea297be220d52398dcc07ce15a209fce436d361735ac1db700cab3b6cdfb9f54" +checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d" dependencies = [ "proc-macro2", "quote", @@ -2168,7 +2175,7 @@ dependencies = [ "cfg-if 1.0.0", "fastrand", "libc", - "redox_syscall 0.2.12", + "redox_syscall 0.2.13", "remove_dir_all", "winapi 0.3.9", ] @@ -2602,9 +2609,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" +checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -2612,9 +2619,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" +checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" dependencies = [ "bumpalo", "lazy_static", @@ -2627,9 +2634,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" +checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2637,9 +2644,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" +checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" dependencies = [ "proc-macro2", "quote", @@ -2650,9 +2657,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.79" +version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" +checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" [[package]] name = "winapi" @@ -2699,9 +2706,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-sys" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" +checksum = "5acdd78cb4ba54c0045ac14f62d8f94a03d10047904ae2a40afa1e99d8f70825" dependencies = [ "windows_aarch64_msvc", "windows_i686_gnu", @@ -2712,33 +2719,33 @@ dependencies = [ [[package]] name = "windows_aarch64_msvc" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" +checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" [[package]] name = "windows_i686_gnu" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" +checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" [[package]] name = "windows_i686_msvc" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" +checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" [[package]] name = "windows_x86_64_gnu" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" +checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" [[package]] name = "windows_x86_64_msvc" -version = "0.32.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" +checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" [[package]] name = "winreg" diff --git a/README-Japanese.md b/README-Japanese.md index 34981243..1260aa7c 100644 --- a/README-Japanese.md +++ b/README-Japanese.md @@ -59,17 +59,18 @@ Hayabusaは、日本の[Yamato Security](https://yamatosecurity.connpass.com/) - [Hayabusaルール](#hayabusaルール) - [Hayabusa v.s. 変換されたSigmaルール](#hayabusa-vs-変換されたsigmaルール) - [検知ルールのチューニング](#検知ルールのチューニング) + - [検知レベルのlevelチューニング](#検知レベルのlevelチューニング) - [イベントIDフィルタリング](#イベントidフィルタリング) - [その他のWindowsイベントログ解析ツールおよび関連プロジェクト](#その他のwindowsイベントログ解析ツールおよび関連プロジェクト) - [Windowsイベントログ設定のススメ](#windowsイベントログ設定のススメ) - [Sysmon関係のプロジェクト](#sysmon関係のプロジェクト) - - [Sigmaをサポートする他の類似ツールとの比較](#sigmaをサポートする他の類似ツールとの比較) - [コミュニティによるドキュメンテーション](#コミュニティによるドキュメンテーション) - [英語](#英語) - [日本語](#日本語) - [貢献](#貢献) - [バグの報告](#バグの報告) - [ライセンス](#ライセンス) +- [Twitter](#twitter) ## 主な目的 @@ -129,7 +130,8 @@ CSVのタイムラインをExcelやTimeline Explorerで分析する方法は[こ * イベントログの統計。(どのような種類のイベントがあるのかを把握し、ログ設定のチューニングに有効です。) * 不良ルールやノイズの多いルールを除外するルールチューニング設定が可能です。 * MITRE ATT&CKとのマッピング (CSVの出力ファイルのみ)。 -* イベントログから不審なユーザやファイルを素早く特定するのに有用な、ピボットキーワードの一覧を作成することが可能です。 +* ルールレベルのチューニング。 +* イベントログから不審なユーザやファイルを素早く特定するのに有用な、ピボットキーワードの一覧作成。 # 予定されている機能 @@ -309,6 +311,7 @@ USAGE: -s --statistics 'イベント ID の統計情報を表示する。' -q --quiet 'Quietモード。起動バナーを表示しない。' -Q --quiet-errors 'Quiet errorsモード。エラーログを保存しない。' + --level-tuning 'ルールlevelのチューニング [default: ./config/level_tuning.txt]' -p --pivot-keywords-list 'ピボットキーワードの一覧作成。' --contributors 'コントリビュータの一覧表示。' ``` @@ -490,9 +493,9 @@ Hayabusaルールのディレクトリ構造は、3つのディレクトリに ルールはさらにログタイプ（例：Security、Systemなど）によってディレクトリに分けられ、次の形式で名前が付けられます。 -* アラート形式: `<イベントID>__<詳細>.yml` -* アラート例: `1102_IndicatorRemovalOnHost-ClearWindowsEventLogs_SecurityLogCleared.yml` -* イベント形式: `<イベントID>_<詳細>.yml` +* アラート形式: `<イベントID>_<イベントの説明>_<リスクの説明>.yml` +* アラート例: `1102_SecurityLogCleared_PossibleAntiForensics.yml` +* イベント形式: `<イベントID>_<イベントの説明>.yml` * イベント例: `4776_NTLM-LogonToLocalAccount.yml` 現在のルールをご確認いただき、新規作成時のテンプレートとして、また検知ロジックの確認用としてご利用ください。 @@ -509,8 +512,7 @@ Sigmaルールは、最初にHayabusaルール形式に変換する必要があ 1. [Rust正規表現クレート](https://docs.rs/regex/1.5.4/regex/)では機能しない正規表現を使用するルール。 2. [Sigmaルール仕様](https://github.com/SigmaHQ/Sigma/wiki/Specification)の`count`以外の集計式。 - -> 注意：この制限はSigmaルールの変換ツールにあり、Hayabusa自身にあるわけではありません。 +3. `|near`を使用するルール。 ## 検知ルールのチューニング @@ -520,6 +522,20 @@ Sigmaルールは、最初にHayabusaルール形式に変換する必要があ ルールIDを `rules/config/noisy_rules.txt`に追加して、デフォルトでルールを無視することもできますが、`-n`または `--enable-noisy-rules`オプションを指定してルールを使用することもできます。 +## 検知レベルのlevelチューニング + +Hayabusaルール、Sigmaルールはそれぞれの作者が検知した際のリスクレベルを決めています。 +ユーザが独自のリスクレベルに設定するには`./config/level_tuning.txt`に変換情報を書き、`hayabusa.exe --level-tuning`を実行することでルールファイルが書き換えられます。 +ルールファイルが直接書き換えられることに注意して使用してください。 + +`./config/level_tuning.txt`の例: +``` +id,new_level +00000000-0000-0000-0000-000000000000,informational # sample level tuning line +``` + +ルールディレクトリ内で`id`が`00000000-0000-0000-0000-000000000000`のルールのリスクレベルが`informational`に書き換えられます。 + ## イベントIDフィルタリング `config/target_eventids.txt`にイベントID番号を追加することで、イベントIDでフィルタリングすることができます。 @@ -569,23 +585,6 @@ Windows機での悪性な活動を検知する為には、デフォルトのロ * [Sysmon Modular](https://github.com/olafhartong/sysmon-modular) * [TrustedSec Sysmon Community Guide](https://github.com/trustedsec/SysmonCommunityGuide) -## Sigmaをサポートする他の類似ツールとの比較 - -対象となるサンプルデータ、コマンドラインオプション、ルールのチューニング等によって結果が異なるため、完全な比較はできませんが、ご了承ください。 -我々のテストでは、Hayabusaはすべてのツールの中で最も多くのSigmaルールをサポートしながらも、非常に高速な速度を維持し、大量のメモリを必要としないことが分かっています。 - -以下のベンチマークは、2021/12/23に [sample-evtx repository](https://github.com/Yamato-Security/Hayabusa-sample-evtx) から約500個のevtxファイル（130MB）を基に、Lenovo P51で計測したものです。Hayabusa 1.0.0を使いました。 - -| | 経過時間 | メモリ使用量 | 利用可能のSigmaルール数 | -| :-------: | :---------: | :--------------------------------------------: | :---------------------: | -| Chainsaw | 7.5 seconds | 70 MB | 170 | -| Hayabusa | 7.8 seconds | 340 MB | 267 | -| Zircolite | 34 seconds | 380 MB (通常、ログファイルの3倍のサイズが必要) | 237 | - -* Hayabusaルールも有効にすると、約300のユニークなアラートとイベントを検知します。 -* 合計7.5GBの多数のイベントログファイルでテストしたところ、7分以内に終了し、1GB以上のメモリを使用しませんでした。消費されるメモリ量は、ターゲットのevtxファイルのサイズではなく、結果のサイズによって増えます。 -* [Timeline Explorer](https://ericzimmerman.github.io/#!index.md)などのツールで解析するために、結果を1つのCSVタイムラインにまとめる唯一のツールです。 - # コミュニティによるドキュメンテーション ## 英語 @@ -611,3 +610,7 @@ Windows機での悪性な活動を検知する為には、デフォルトのロ # ライセンス Hayabusaは[GPLv3](https://www.gnu.org/licenses/gpl-3.0.en.html)で公開され、すべてのルールは[Detection Rule License (DRL) 1.1](https://github.com/SigmaHQ/sigma/blob/master/LICENSE.Detection.Rules.md)で公開されています。 + +# Twitter + +[https://twitter.com/SecurityYamato](@SecurityYamato)でHayabusa、ルール更新、その他の大和セキュリティツール等々について情報を提供しています。 \ No newline at end of file diff --git a/README.md b/README.md index cd6a32df..de69a540 100644 --- a/README.md +++ b/README.md @@ -59,17 +59,18 @@ Hayabusa is a **Windows event log fast forensics timeline generator** and **thre - [Hayabusa Rules](#hayabusa-rules) - [Hayabusa v.s. Converted Sigma Rules](#hayabusa-vs-converted-sigma-rules) - [Detection Rule Tuning](#detection-rule-tuning) + - [Detection Level Tuning](#detection-level-tuning) - [Event ID Filtering](#event-id-filtering) - [Other Windows Event Log Analyzers and Related Projects](#other-windows-event-log-analyzers-and-related-projects) - [Windows Logging Recommendations](#windows-logging-recommendations) - [Sysmon Related Projects](#sysmon-related-projects) - - [Comparison To Other Similar Tools](#comparison-to-other-similar-tools) - [Community Documentation](#community-documentation) - [English](#english) - [Japanese](#japanese) - [Contribution](#contribution) - [Bug Submission](#bug-submission) - [License](#license) +- [Twitter](#twitter) ## Main Goals @@ -127,6 +128,7 @@ You can learn how to analyze CSV timelines in Excel and Timeline Explorer [here] * Event log statistics. (Useful for getting a picture of what types of events there are and for tuning your log settings.) * Rule tuning configuration by excluding unneeded or noisy rules. * MITRE ATT&CK mapping of tactics (only in saved CSV files). +* Rule level tuning. * Create a list of unique pivot keywords to quickly identify abnormal users, hostnames, processes, etc... as well as correlate events. # Planned Features @@ -224,7 +226,7 @@ You may receive warning from anti-virus or EDR when trying to run hayabusa. Thes ## Windows -In Command Prompt or Windows Terminal, just run 32-bit or 64-bit Windoows binary from the hayabusa root directory. +In Command Prompt or Windows Terminal, just run the 32-bit or 64-bit Windows binary from the hayabusa root directory. Example: `hayabusa-1.2.0-windows-x64.exe` ## Linux @@ -302,6 +304,7 @@ USAGE: -s --statistics 'Prints statistics of event IDs.' -q --quiet 'Quiet mode. Do not display the launch banner.' -Q --quiet-errors 'Quiet errors mode. Do not save error logs.' + --level-tuning 'Tune the rule level [default: ./config/level_tuning.txt]' -p --pivot-keywords-list 'Create a list of pivot keywords.' --contributors 'Prints the list of contributors.' ``` @@ -481,9 +484,9 @@ The hayabusa rule directory structure is separated into 3 directories: Rules are further seperated into directories by log type (Example: Security, System, etc...) and are named in the following format: -* Alert format: `__.yml` -* Alert example: `1102_IndicatorRemovalOnHost-ClearWindowsEventLogs_SecurityLogCleared.yml` -* Event format: `_.yml` +* Alert format: `__.yml` +* Alert example: `1102_SecurityLogCleared_PossibleAntiForensics.yml` +* Event format: `_.yml` * Event example: `4776_NTLM-LogonToLocalAccount.yml` Please check out the current rules to use as a template in creating new ones or for checking the detection logic. @@ -500,8 +503,7 @@ Sigma rules need to first be converted to hayabusa rule format explained [here]( 1. Rules that use regular expressions that do not work with the [Rust regex crate](https://docs.rs/regex/1.5.4/regex/) 2. Aggregation expressions besides `count` in the [sigma rule specification](https://github.com/SigmaHQ/sigma/wiki/Specification). - -> Note: the limitation is in the sigma rule converter and not in hayabusa itself. +3. Rules that use `|near`. ## Detection Rule Tuning @@ -511,6 +513,22 @@ You can add a rule ID (Example: `4fe151c2-ecf9-4fae-95ae-b88ec9c2fca6`) to `rule You can also add a rule ID to `rules/config/noisy_rules.txt` in order to ignore the rule by default but still be able to use the rule with the `-n` or `--enable-noisy-rules` option. +## Detection Level Tuning + +Hayabusa and Sigma rule authors will determine the risk level of the alert when writing their rules. +However, the actual risk level will differ between environments. +You can tune the risk level of the rules by adding them to `./config/level_tuning.txt` and executing `hayabusa.exe --level-tuning` which will update the `level` line in the rule file. +Please note that the rule file will be updated directly. + +`./config/level_tuning.txt` sample line: + +``` +id,new_level +00000000-0000-0000-0000-000000000000,informational # sample level tuning line +``` + +In this case, the risk level of the rule with an `id` of `00000000-0000-0000-0000-000000000000` in the rules directory will have its `level` rewritten to `informational`. + ## Event ID Filtering You can filter on event IDs by placing event ID numbers in `config/target_eventids.txt`. @@ -559,23 +577,6 @@ To create the most forensic evidence and detect with the highest accuracy, you n * [Sysmon Modular](https://github.com/olafhartong/sysmon-modular) * [TrustedSec Sysmon Community Guide](https://github.com/trustedsec/SysmonCommunityGuide) -## Comparison To Other Similar Tools - -Please understand that it is not possible to do a perfect comparison as results will differ based on the target sample data, command-line options, rule tuning, etc... -In our tests, we have found hayabusa to support the largest number of sigma rules compared to other similar tools while still maintaining very fast speeds and does not require a great amount of memory. - -The following benchmarks were taken on a Lenovo P51 based on approximately 500 evtx files (130MB) from our [sample-evtx repository](https://github.com/Yamato-Security/hayabusa-sample-evtx) at 2021/12/23 with hayabusa version 1.0.0. - -| | Elapsed Time | Memory Usage | Unique Sigma Rules With Detections | -| :-------: | :----------: | :----------------------------------------------------------: | :--------------------------------: | -| Chainsaw | 7.5 seconds | 75 MB | 170 | -| Hayabusa | 7.8 seconds | 340 MB (memory usage depends on the amount of alerts) | 267 | -| Zircolite | 34 seconds | 380 MB (normally requires 3 times the size of the log files) | 237 | - -* With hayabusa rules enabled, it will detect around 300 unique alerts and events. -* When tested on many event logs files totaling 7.5 GB, it finished in under 7 minutes and used around 1 GB of memory. The amount of memory consumed is based on the size of the results, not on the size of the target evtx files. -* It is the only tool that provides a consolidated single CSV timeline to analysis in tools like [Timeline Explorer](https://ericzimmerman.github.io/#!index.md). - # Community Documentation ## English @@ -601,4 +602,8 @@ This project is currently actively maintained and we are happy to fix any bugs r # License -Hayabusa is released under [GPLv3](https://www.gnu.org/licenses/gpl-3.0.en.html) and all rules are released under the [Detection Rule License (DRL) 1.1](https://github.com/SigmaHQ/sigma/blob/master/LICENSE.Detection.Rules.md). \ No newline at end of file +Hayabusa is released under [GPLv3](https://www.gnu.org/licenses/gpl-3.0.en.html) and all rules are released under the [Detection Rule License (DRL) 1.1](https://github.com/SigmaHQ/sigma/blob/master/LICENSE.Detection.Rules.md). + +# Twitter + +You can recieve the latest news about Hayabusa, rule updates, other Yamato Security tools, etc... by following us on Twitter at [https://twitter.com/SecurityYamato](@SecurityYamato). \ No newline at end of file diff --git a/config/level_tuning.txt b/config/level_tuning.txt new file mode 100644 index 00000000..8482d822 --- /dev/null +++ b/config/level_tuning.txt @@ -0,0 +1,2 @@ +id,new_level +00000000-0000-0000-0000-000000000000,informational # sample level tuning line diff --git a/src/detections/configs.rs b/src/detections/configs.rs index 3dd2321e..f336aa1e 100644 --- a/src/detections/configs.rs +++ b/src/detections/configs.rs @@ -3,10 +3,11 @@ use crate::detections::pivot::PIVOT_KEYWORD; use crate::detections::print::AlertMessage; use crate::detections::utils; use chrono::{DateTime, Utc}; -use clap::{App, AppSettings, ArgMatches}; +use clap::{App, AppSettings, Arg, ArgMatches}; use hashbrown::HashMap; use hashbrown::HashSet; use lazy_static::lazy_static; +use regex::Regex; use std::io::BufWriter; use std::sync::RwLock; lazy_static! { @@ -24,6 +25,8 @@ lazy_static! { "{}/eventkey_alias.txt", CONFIG.read().unwrap().folder_path )); + pub static ref IDS_REGEX: Regex = + Regex::new(r"^[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12}$").unwrap(); } #[derive(Clone)] @@ -93,9 +96,14 @@ fn build_app<'a>() -> ArgMatches<'a> { --contributors 'Prints the list of contributors.'"; App::new(&program) .about("Hayabusa: Aiming to be the world's greatest Windows event log analysis tool!") - .version("1.1.0") - .author("Yamato Security (https://github.com/Yamato-Security/hayabusa)") + .version("1.2.0") + .author("Yamato Security (https://github.com/Yamato-Security/hayabusa) @SecurityYamato") .setting(AppSettings::VersionlessSubcommands) + .arg( + // TODO: When update claps to 3.x, these can write in usage texts... + Arg::from_usage("--level-tuning=[LEVEL_TUNING_FILE] 'Adjust rule level.'") + .default_value("./config/level_tuning.txt"), + ) .usage(usages) .args_from_usage(usages) .get_matches() diff --git a/src/filter.rs b/src/filter.rs index d65f6296..636436f9 100644 --- a/src/filter.rs +++ b/src/filter.rs @@ -3,17 +3,11 @@ use crate::detections::print::AlertMessage; use crate::detections::print::ERROR_LOG_STACK; use crate::detections::print::QUIET_ERRORS_FLAG; use hashbrown::HashSet; -use lazy_static::lazy_static; use regex::Regex; use std::fs::File; use std::io::BufWriter; use std::io::{BufRead, BufReader}; -lazy_static! { - static ref IDS_REGEX: Regex = - Regex::new(r"^[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12}$").unwrap(); -} - #[derive(Debug)] pub struct DataFilterRule { pub regex_rule: Regex, @@ -74,7 +68,7 @@ impl RuleExclude { let v = v.unwrap().split('#').collect::>()[0] .trim() .to_string(); - if v.is_empty() || !IDS_REGEX.is_match(&v) { + if v.is_empty() || !configs::IDS_REGEX.is_match(&v) { // 空行は無視する。IDの検証 continue; } diff --git a/src/lib.rs b/src/lib.rs index 9bd8f144..5faf0723 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,5 +3,6 @@ pub mod detections; pub mod filter; pub mod notify; pub mod omikuji; +pub mod options; pub mod timeline; pub mod yaml; diff --git a/src/main.rs b/src/main.rs index df2adb3e..dc3fdf9d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -19,6 +19,7 @@ use hayabusa::detections::print::{ use hayabusa::detections::rule::{get_detection_keys, RuleNode}; use hayabusa::filter; use hayabusa::omikuji::Omikuji; +use hayabusa::options::level_tuning::LevelTuning; use hayabusa::yaml::ParseYaml; use hayabusa::{afterfact::after_fact, detections::utils}; use hayabusa::{detections::configs, timeline::timelines::Timeline}; @@ -80,6 +81,16 @@ impl App { } let analysis_start_time: DateTime = Local::now(); + + // Show usage when no arguments. + if std::env::args().len() == 1 { + self.output_logo(); + println!(); + println!("{}", configs::CONFIG.read().unwrap().args.usage()); + println!(); + return; + } + if !configs::CONFIG.read().unwrap().args.is_present("quiet") { self.output_logo(); println!(); @@ -93,7 +104,7 @@ impl App { if !self.is_matched_architecture_and_binary() { AlertMessage::alert( &mut BufWriter::new(std::io::stderr().lock()), - "The hayabusa version you ran does not match your PC architecture.\n Please use the correct architecture. (Binary ending in -x64.exe for 64-bit and -x86.exe for 32-bit.)", + "The hayabusa version you ran does not match your PC architecture.\nPlease use the correct architecture. (Binary ending in -x64.exe for 64-bit and -x86.exe for 32-bit.)", ) .ok(); println!(); @@ -123,6 +134,7 @@ impl App { println!(); return; } + if !Path::new("./config").exists() { AlertMessage::alert( &mut BufWriter::new(std::io::stderr().lock()), @@ -131,11 +143,7 @@ impl App { .ok(); return; } - if configs::CONFIG.read().unwrap().args.args.is_empty() { - println!("{}", configs::CONFIG.read().unwrap().args.usage()); - println!(); - return; - } + if let Some(csv_path) = configs::CONFIG.read().unwrap().args.value_of("output") { for (key, _) in PIVOT_KEYWORD.read().unwrap().iter() { let keywords_file_name = csv_path.to_owned() + "-" + key + ".txt"; @@ -216,7 +224,42 @@ impl App { { self.print_contributors(); return; + } else if configs::CONFIG + .read() + .unwrap() + .args + .is_present("level-tuning") + { + let level_tuning_config_path = configs::CONFIG + .read() + .unwrap() + .args + .value_of("level-tuning") + .unwrap_or("./config/level_tuning.txt") + .to_string(); + + if Path::new(&level_tuning_config_path).exists() { + if let Err(err) = LevelTuning::run( + &level_tuning_config_path, + configs::CONFIG + .read() + .unwrap() + .args + .value_of("rules") + .unwrap_or("rules"), + ) { + AlertMessage::alert(&mut BufWriter::new(std::io::stderr().lock()), &err).ok(); + } + } else { + AlertMessage::alert( + &mut BufWriter::new(std::io::stderr().lock()), + "Need rule_levels.txt file to use --level-tuning option [default: ./config/level_tuning.txt]", + ) + .ok(); + } + return; } + let analysis_end_time: DateTime = Local::now(); let analysis_duration = analysis_end_time.signed_duration_since(analysis_start_time); println!(); diff --git a/src/options/level_tuning.rs b/src/options/level_tuning.rs new file mode 100644 index 00000000..9ca51d18 --- /dev/null +++ b/src/options/level_tuning.rs @@ -0,0 +1,161 @@ +use crate::detections::{configs, utils}; +use crate::filter; +use crate::yaml::ParseYaml; +use std::collections::HashMap; +use std::fs::{self, File}; +use std::io::Write; +pub struct LevelTuning {} + +impl LevelTuning { + pub fn run(level_tuning_config_path: &str, rules_path: &str) -> Result<(), String> { + let read_result = utils::read_csv(level_tuning_config_path); + if read_result.is_err() { + return Result::Err(read_result.as_ref().unwrap_err().to_string()); + } + + // Read Tuning files + let mut tuning_map: HashMap = HashMap::new(); + read_result.unwrap().into_iter().try_for_each(|line| -> Result<(), String> { + let id = match line.get(0) { + Some(_id) => { + if !configs::IDS_REGEX.is_match(_id) { + return Result::Err(format!("Failed to read level tuning file. {} is not correct id format, fix it.", _id)); + } + _id + } + _ => return Result::Err("Failed to read id...".to_string()) + }; + let level = match line.get(1) { + Some(_level) => { + if _level.starts_with("informational") + || _level.starts_with("low") + || _level.starts_with("medium") + || _level.starts_with("high") + || _level.starts_with("critical") { + _level.split('#').collect::>()[0] + } else { + return Result::Err("level tuning file's level must in informational, low, medium, high, critical".to_string()) + } + } + _ => return Result::Err("Failed to read level...".to_string()) + }; + tuning_map.insert(id.to_string(), level.to_string()); + Ok(()) + })?; + + // Read Rule files + let mut rulefile_loader = ParseYaml::new(); + let result_readdir = + rulefile_loader.read_dir(rules_path, "informational", &filter::exclude_ids()); + if result_readdir.is_err() { + return Result::Err(format!("{}", result_readdir.unwrap_err())); + } + + // Convert rule files + for (path, rule) in rulefile_loader.files { + if let Some(new_level) = tuning_map.get(rule["id"].as_str().unwrap()) { + println!("path: {}", path); + let mut content = match fs::read_to_string(&path) { + Ok(_content) => _content, + Err(e) => return Result::Err(e.to_string()), + }; + let past_level = "level: ".to_string() + rule["level"].as_str().unwrap(); + + if new_level.starts_with("informational") { + content = content.replace(&past_level, "level: informational"); + } + if new_level.starts_with("low") { + content = content.replace(&past_level, "level: low"); + } + if new_level.starts_with("medium") { + content = content.replace(&past_level, "level: medium"); + } + if new_level.starts_with("high") { + content = content.replace(&past_level, "level: high"); + } + if new_level.starts_with("critical") { + content = content.replace(&past_level, "level: critical"); + } + + let mut file = match File::options().write(true).truncate(true).open(&path) { + Ok(file) => file, + Err(e) => return Result::Err(e.to_string()), + }; + + file.write_all(content.as_bytes()).unwrap(); + file.flush().unwrap(); + println!( + "level: {} -> {}", + rule["level"].as_str().unwrap(), + new_level + ); + } + } + Result::Ok(()) + } +} + +#[cfg(test)] +mod tests { + + // use crate::{filter::RuleExclude, yaml}; + // use hashbrown::HashSet; + + use super::*; + + #[test] + fn rule_level_failed_to_open_file() -> Result<(), String> { + let level_tuning_config_path = "./none.txt"; + let res = LevelTuning::run(level_tuning_config_path, ""); + let expected = Result::Err("Cannot open file. [file:./none.txt]".to_string()); + assert_eq!(res, expected); + Ok(()) + } + + #[test] + fn rule_level_id_error_file() -> Result<(), String> { + let level_tuning_config_path = "./test_files/config/level_tuning_error1.txt"; + let res = LevelTuning::run(level_tuning_config_path, ""); + let expected = Result::Err("Failed to read level tuning file. 12345678-1234-1234-1234-12 is not correct id format, fix it.".to_string()); + assert_eq!(res, expected); + Ok(()) + } + + #[test] + fn rule_level_level_error_file() -> Result<(), String> { + let level_tuning_config_path = "./test_files/config/level_tuning_error2.txt"; + let res = LevelTuning::run(level_tuning_config_path, ""); + let expected = Result::Err( + "level tuning file's level must in informational, low, medium, high, critical" + .to_string(), + ); + assert_eq!(res, expected); + Ok(()) + } + + #[test] + fn test_level_tuning_update_rule_files() { + let level_tuning_config_path = "./test_files/config/level_tuning.txt"; + let rule_str = r#" + id: 12345678-1234-1234-1234-123456789012 + level: informational + "#; + + let expected_rule = r#" + id: 12345678-1234-1234-1234-123456789012 + level: high + "#; + + let path = "test_files/rules/level_tuning_test.yml"; + let mut file = File::create(path).unwrap(); + let buf = rule_str.as_bytes(); + file.write_all(buf).unwrap(); + file.flush().unwrap(); + + let res = LevelTuning::run(level_tuning_config_path, path); + assert_eq!(res, Ok(())); + + assert_eq!(fs::read_to_string(path).unwrap(), expected_rule); + fs::remove_file(path).unwrap(); + } +} diff --git a/src/options/mod.rs b/src/options/mod.rs new file mode 100644 index 00000000..1f3c32b6 --- /dev/null +++ b/src/options/mod.rs @@ -0,0 +1 @@ +pub mod level_tuning; diff --git a/test_files/config/level_tuning.txt b/test_files/config/level_tuning.txt new file mode 100644 index 00000000..58a9604a --- /dev/null +++ b/test_files/config/level_tuning.txt @@ -0,0 +1,2 @@ +id,next_level +12345678-1234-1234-1234-123456789012,high \ No newline at end of file diff --git a/test_files/config/level_tuning_error1.txt b/test_files/config/level_tuning_error1.txt new file mode 100644 index 00000000..bed1b758 --- /dev/null +++ b/test_files/config/level_tuning_error1.txt @@ -0,0 +1,2 @@ +id,new_level +12345678-1234-1234-1234-12,informational # sample level tuning line diff --git a/test_files/config/level_tuning_error2.txt b/test_files/config/level_tuning_error2.txt new file mode 100644 index 00000000..c8c1c1e4 --- /dev/null +++ b/test_files/config/level_tuning_error2.txt @@ -0,0 +1,2 @@ +id,new_level +00000000-0000-0000-0000-000000000000,no_exist_level # sample level tuning line