updates

CHANGELOG.md

@@ -8,7 +8,7 @@
 
 **Enhancements:**
 
-- Added `--deep-scan` option. Default scan target is filtered EventIDs in `config/target_event_ids.txt`. With this option, scan target is all EventIDs. (#608) (@hitenkoku)
+- Added the `-D, --deep-scan` option. Now by default, events are filtered by Event IDs that there are detection rules for defined in `rules/config/target_event_IDs.txt`. This should improve performance by 25~55% while still detecting almost everything. If you want to do a thorough scan on all events, you can disable the event ID filter with `-D, --deep-scan`. (#608) (@hitenkoku)
 
 **Bug Fixes:**
 

Cargo.lock

@@ -220,9 +220,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "3.2.6"
+version = "3.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f1fe12880bae935d142c8702d500c63a4e8634b6c3c57ad72bf978fc7b6249a"
+checksum = "5b7b16274bb247b45177db843202209b12191b631a14a9d06e41b3777d6ecf14"
 dependencies = [
  "atty",
  "bitflags",
@@ -237,9 +237,9 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "3.2.6"
+version = "3.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed6db9e867166a43a53f7199b5e4d1f522a1e5bd626654be263c999ce59df39a"
+checksum = "759bf187376e1afa7b85b959e6a664a3e7a95203415dba952ad19139e798f902"
 dependencies = [
  "heck",
  "proc-macro-error",
@@ -250,9 +250,9 @@ dependencies = [
 
 [[package]]
 name = "clap_lex"
-version = "0.2.3"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87eba3c8c7f42ef17f6c659fc7416d0f4758cd3e58861ee63c5fa4a4dde649e4"
+checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5"
 dependencies = [
  "os_str_bytes",
 ]
@@ -673,12 +673,12 @@ dependencies = [
 
 [[package]]
 name = "hayabusa"
-version = "1.4.0"
+version = "1.4.1-dev"
 dependencies = [
  "base64",
  "bytesize",
  "chrono",
- "clap 3.2.6",
+ "clap 3.2.7",
  "crossbeam-utils",
  "csv",
  "downcast-rs",
@@ -914,7 +914,7 @@ dependencies = [
  "anyhow",
  "atty",
  "chrono",
- "clap 3.2.6",
+ "clap 3.2.7",
  "file-chunker",
  "indicatif",
  "memmap2",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "hayabusa"
-version = "1.4.0"
+version = "1.4.1-dev"
 authors = ["Yamato Security @SecurityYamato"]
 edition = "2021"
 

README-Japanese.md

@@ -673,13 +673,10 @@ id,new_level
 
 ## イベントIDフィルタリング
 
-`config/target_eventids.txt`にイベントID番号を追加することで、イベントIDでフィルタリングすることができます。
+バージョン1.4.1以降では、デフォルトでパフォーマンスを上げるために、検知ルールでイベントIDが定義されていないイベントを無視しています。
-これはパフォーマンスを向上させるので、特定のIDだけを検索したい場合に推奨されます。
+デフォルトでは`rules/config/target_event_IDs.txt`で定義されたIDがスキャンされます。
-このイベントIDフィルタリングを除外したい場合は`-D`もしくは`--deepscan`オプションを利用してください。
+If you want to scan all events, please use the `-D, --deep-scan` option.
-
+すべてのイベントをスキャンしたい場合は、`-D, --deep-scan`オプションを使用してください。
-すべてのルールの`EventID`フィールドと実際のスキャン結果で見られるIDから作成したIDフィルタリストのサンプルを[`config/target_eventids_sample.txt`](https://github.com/Yamato-Security/hayabusa/blob/main/config/target_eventids_sample.txt)で提供しています。
-
-最高のパフォーマンスを得たい場合はこのリストを使用してください。ただし、検出漏れの可能性が若干あることにご注意ください。
 
 # その他のWindowsイベントログ解析ツールおよび関連リソース
 

README.md

@@ -329,7 +329,7 @@ OPTIONS:
     -c, --rules-config <RULE_CONFIG_DIRECTORY>    Specify custom rule config folder (default: ./rules/config)
         --contributors                            Print the list of contributors
     -d, --directory <DIRECTORY>                   Directory of multiple .evtx files
-    -D, --deep-scan                               Scan to all event ID
+    -D, --deep-scan                               Disable event ID filter to scan all events
         --enable-deprecated-rules                 Enable rules marked as deprecated
         --end-timeline <END_TIMELINE>             End time of the event logs to load (ex: "2022-02-22 23:59:59 +09:00")
         --exclude-status <EXCLUDE_STATUS>...      Ignore rules according to status (ex: experimental) (ex: stable test)
@@ -670,13 +670,9 @@ In this case, the risk level of the rule with an `id` of `00000000-0000-0000-000
 
 ## Event ID Filtering
 
-You can filter on event IDs by placing event ID numbers in `config/target_eventids.txt`.
+As of version 1.4.1, by default, events are filtered by ID to improve performance by ignorning events that have no detection rules.
-This will increase performance so it is recommended if you only need to search for certain IDs.
+The IDs defined in `rules/config/target_event_IDs.txt` will be scanned by default.
-You can exclude event IDs filter with `-D` or `--deep-scan` option.
+If you want to scan all events, please use the `-D, --deep-scan` option.
-
-We have provided a sample ID filter list at [`config/target_eventids_sample.txt`](https://github.com/Yamato-Security/hayabusa/blob/main/config/target_eventids_sample.txt) created from the `EventID` fields in all of the rules as well as IDs seen in actual results.
-
-Please use this list if you want the best performance but be aware that there is a slight possibility for missing events (false negatives). 
 
 # Other Windows Event Log Analyzers and Related Resources
 

config/target_eventids.txt

@@ -1,60 +1,92 @@
 1
+2
+3
+4
+5
+6
+7
+8
+9
 10
-1000
-1001
-1006
-1013
-1015
-1031
-1032
-1033
-1034
-104
-106
 11
-1102
-1116
-1116
-1117
-1121
 12
 13
 14
 15
-150
 16
 17
 18
 19
 20
-2003
 21
-2100
-2102
-213
-217
 22
 23
 24
+25
+26
+30
+35
+36
+37
+38
+50
+55
+56
+59
+98
+104
+106
+140
+141
+150
+200
+213
+217
 255
 257
-26
-3
-30
 300
 301
 302
-316
-31017
 354
-4
 400
-400
+517
-403
+524
-40300
+528
-40301
+529
-40302
+600
-4100
+675
+770
+800
+808
+823
+848
+849
+1000
+1001
+1006
+1013
+1015
+1024
+1031
+1032
+1033
+1034
+1102
+1116
+1117
+1121
+1337
+2002
+2003
+2004
+2005
+2006
+2008
+2009
+2032
+2033
+2100
+2102
 4103
 4104
 4611
@@ -81,7 +113,6 @@
 4698
 4699
 4701
-4703
 4704
 4706
 4719
@@ -96,6 +127,8 @@
 4769
 4771
 4776
+4778
+4779
 4781
 4794
 4799
@@ -105,8 +138,6 @@
 4904
 4905
 4909
-5
-50
 5001
 5007
 5010
@@ -119,36 +150,34 @@
 5142
 5145
 5156
-517
+5379
-524
+5723
-528
+5805
-529
-55
-56
 5829
+5857
 5859
+5860
 5861
-59
+6005
-6
+6006
-600
+6008
+6038
 6281
 6416
-675
+7031
-7
-70
 7036
 7040
 7045
-770
-8
-800
 8001
 8002
 8004
 8007
-808
+11724
-823
+16403
-848
+16990
-849
+16991
-9
+20001
-98
+31017
+40300
+40301
+40302

config/target_eventids_sample.txt

@@ -1,154 +0,0 @@
-1
-10
-1000
-1001
-1006
-1013
-1015
-1031
-1032
-1033
-1034
-104
-106
-11
-1102
-1116
-1116
-1117
-1121
-12
-13
-14
-15
-150
-16
-17
-18
-19
-20
-2003
-21
-2100
-2102
-213
-217
-22
-23
-24
-255
-257
-26
-3
-30
-300
-301
-302
-316
-31017
-354
-4
-400
-400
-403
-40300
-40301
-40302
-4100
-4103
-4104
-4611
-4616
-4624
-4625
-4634
-4647
-4648
-4656
-4657
-4658
-4660
-4661
-4662
-4663
-4672
-4673
-4674
-4688
-4689
-4692
-4697
-4698
-4699
-4701
-4703
-4704
-4706
-4719
-4720
-4728
-4732
-4738
-4742
-4765
-4766
-4768
-4769
-4771
-4776
-4781
-4794
-4799
-4825
-4898
-4899
-4904
-4905
-4909
-5
-50
-5001
-5007
-5010
-5012
-5013
-5038
-5101
-5136
-5140
-5142
-5145
-5156
-517
-524
-528
-529
-55
-56
-5829
-5859
-5861
-59
-6
-600
-6281
-6416
-675
-7
-70
-7036
-7040
-7045
-770
-8
-800
-8001
-8002
-8004
-8007
-808
-823
-848
-849
-9
-98

src/detections/configs.rs

@@ -118,7 +118,7 @@ pub struct Config {
     #[clap(long = "enable-deprecated-rules")]
     pub enable_deprecated_rules: bool,
 
-    /// Scan to all event ID
+    /// Disable event ID filter to scan all events
     #[clap(short = 'D', long = "deep-scan")]
     pub deep_scan: bool,