v1.2 pre-release marge (#495)

* Fix/fix clippy warn (#434) - Fixed following Clippy Warnings(previous warning count: 671 -> after: 4) - clippy::needless_return - clippy::println_empty_string - clippy::redundant_field_names - clippy::single_char_pattern - clippy::len_zero - clippy::iter_nth_zero - clippy::bool_comparison - clippy::question_mark - clippy::needless_collect - clippy::unnecessary_unwrap - clippy::ptr_arg - clippy::needless_collect - clippy::needless_borrow - clippy::new_without_default - clippy::assign_op_pattern - clippy::bool_assert_comparison - clippy::into_iter_on_ref - clippy::deref_addrof - clippy::while_let_on_iterator - clippy::match_like_matches_macro - clippy::or_fun_call - clippy::useless_conversion - clippy::let_and_return - clippy::redundant_clone - clippy::redundant_closure - clippy::cmp_owned - clippy::upper_case_acronyms - clippy::map_identity - clippy::unused_io_amount - clippy::assertions_on_constants - clippy::op_ref - clippy::useless_vec - clippy::vec_init_then_push - clippy::useless_format - clippy::bind_instead_of_map - clippy::bool_comparison - clippy::clone_on_copy - clippy::too_many_arguments - clippy::module_inception - fixed clippy::needless_lifetimes - fixed clippy::borrowed_box (Thanks for helping by hach1yon!) * Merge main and output fix#443#444 (#445) * removed tools/sigmac (#441) * removed tools/sigmac - moved tools/sigmac to hayabusa-rules repo * fixed doc link tools/sigmac * fixed submodule track * fixed submodule track from latest to v1.1.0 tag * fixed link * erased enter #444 * erased enter #444 * reverted logo enter * fixed rules submodule target commit #444 Co-authored-by: Yamato Security <71482215+YamatoSecurity@users.noreply.github.com> * readme update screenshots etc (#448) * Opensslを静的にコンパイルするためにCargo.tomlの設定変更 (#437) * cargo update - openssl static * updated cargo * macos2apple * cargo update * cargo update * aliasキーがない場合もEvent.EventDataを自動で走査する (#442) * add no event key * support not-register-alias search * added checking EventData when key do not match in alias #290 - added checking key in Event.EventData, if key is not exist in eventkey_alias.txt. * cargo fmt * fixed panic when filter files does not exists * fixed errorlog format when filter config files does not exist Co-authored-by: DustInDark <nextsasasa@gmail.com> * changed downcast library from mopa to downcast_rs #447 (#450) * Fixed Clippy Warnings (#451) * fixed clippy warn * fixed cargo clippy warnging * fixed clippy warngings in clippy ver 0.1.59 * fixed clippy warnings clippy::unnecessary_to_owned * added temporary blackhat arsenal badge * added rust report card badges #453 * added repository maintenance levels badge #453 * documentation update macOS usage etc * update * added clippy workflow #428 (#429) * added clippy workflow #428 * fixed action yaml to run clippy #428 * fixed indent * fixed workflow * fixed workflow error * fixed indent * changed no annotation #428 * adujusted annotation version * fixed clippy::needless_match * remove if let exception * removed unnecessary permission check #428 * statistics event id update (#457) * Feature/#440 refactoring #395 (#464) * updated submodule * fix degrade for pull req #464 (#468) * fix degrade for pull req #464 * add trim * Fearture/ added output update result#410 (#452) * add git2 crate #391 * added Update option #391 * updated readme #391 * fixed cargo.lock * fixed option if-statement #391 * changed utc short option and rule-update short option #391 * updated readme * updated readme * fixed -u long option & version number update #391 * added fast-forwarding rules repository #391 * updated command line option #391 * moved output logo prev update rule * fixed readme #391 * removed recursive option in readme * changed rules update from clone and pull to submodule update #391 * fixed document * changed unnecessary clone recursively to clone only * English message update. * cargo fmt * English message update. ( 4657c35e5c cherry-pick) * added create rules folder when rules folder is not exist * fixed gitmodules github-rules url from ssh to https * added output of updated file #420 * fixed error #410 * changed update rule list seq * added test * fixed output #410 * fixed output and fixed output date field when modified field is lacked #410 * fixed compile error * fixed output - added enter after Latest rule update output - added output when no exist new rule - fixed Latest rule update date format - changed output from 'Latest rule update' to 'Latest rules update' * fixed compile error * changed modified date source from rules folder to each yml rule file * formatting use chrono in main.rs * merge develop clippy ci * fixed output when no update rule #410 - removed Latest rule update - no output "Rules update successfully" when No rule changed * Change English Co-authored-by: Tanaka Zakku <71482215+YamatoSecurity@users.noreply.github.com> * Remove unnecessary code from timeline_event_info and rename files for… (#470) * Remove unnecessary code from timeline_event_info and rename files for issue462 * Remove unnecessary code #462 * add equalsfield pipe (#467) * Enhancement: add config config #456 (#471) * added config option #456 * added process of option to speicifed config folder #456 following files adjust config option. * noisy_rules.txt * exclude_rules.txt * fixed usage in readme * updated rules submodule: * fixed process when yml file exist in .git folder * ignore when yml file exist in .git folder * Add: --level-tuning option's outline * Add: read Rule files * Add: input rule_level.txt files & read rules * cargo fmt * Add: level-tuning function * Reface: split to options file * WIP: Text overwrite failed... * Fix: Text overwrite was failed * Add: Error handlings * Add: id, level validation * mv: IDS_REGEX to configs file * fix: level tuning's file name * Cargo fmt * Pivot Keyword List機能の追加 (#412) * add get_pivot_keyword() func * change function name and call it's function * [WIP] support config file * compilete output * cargo fmt * [WIP] add test * add test * support -o option in pivot * add pivot mod * fix miss * pass test in pivot.rs * add comment * pass all test * add fast return * fix output * add test config file * review * rebase * cargo fmt * test pass * fix clippy in my commit * cargo fmt * little refactor * change file input logic and config format * [WIP] change output * [wip] change deta structure * change output & change data structure * pass test * add config * cargo fmt & clippy & rebase * fix cllipy * delete /rules/ in .gitignore * clean comment * clean * clean * fix rebase miss * fix rebase miss * fix clippy * file name output on -o to stdout * add pivot_keywords.txt to ./config * updated english * Documentation update * cargo fmt and clean * updated translate japanese * readme update * readme update Co-authored-by: DustInDark <nextsasasa@gmail.com> Co-authored-by: Tanaka Zakku <71482215+YamatoSecurity@users.noreply.github.com> * Add: test * Add: README.md * Cargo fmt * Use #[cfg(test)] * Fixed output stop when control char exist in windows terminal (#485) * added control character filter in details #382 * fixed document - removed fixed windows teminal caution in readme * fixed level tuning test and added test files #390 * changed level_tuning.txt header from next_level to new_level * fixed convert miss change to low level * added run args rules path to check test easy #390 * fixed comment out processing in level_tuning.txt * fixed config to show level-tuning option * fixed level-tuning option usage from required to option * reduce output mitre attack detail tachnique No. by config file (#483) * reduced mitre attck tag output by config file #477 * prepared 1.2.0 version toml * added test files and mitre attck strategy tag file #477 * fixed cargo.toml version * updated cargo.lock * output tag english update * cargo fmt Co-authored-by: Tanaka Zakku <71482215+YamatoSecurity@users.noreply.github.com> * Fix: test file's path was incorrect * Add: add test_files/config/level_tuning.txt * Add: Flush method. * inserted debug data * reverted config usage * fixed test yaml file path * Feature/#216 output allfields csvnewcolumn (#469) * refactoring * refactoring * under constructing * underconstructing * under construction * underconstructing * fix existing testcase * finish implement * fmt * add option * change name * fix control code bug * fix disp * change format and fix testcase * fix help * Fix: show usage when hayabusa has no args * rm: debug line * Enhance/warning architecture#478 (#482) * added enhance of architecture check #478 * changed check architecture process after output logo #478 * English msg update * fixed detect method of os-bit to windows and linux * removed mac and unix architecture and binary and updated its process of windows * fix clippy * added check on Wow64 env #478 * Update contributors.txt Co-authored-by: Tanaka Zakku <71482215+YamatoSecurity@users.noreply.github.com> * added --level-tuning option to usage * Revert "added --level-tuning option to usage" This reverts commit e6a74090a3. * readme update * Update README-Japanese.md * readme, version, cargo update * typo fix * typo fix * rm: duplicated test & fix test name * Add: show logo, and some infos * small english fix * twitter link fix (#486) * added feature of tag output reducing to agg condition #477 (#488) * changed level output from informational to info #491 * updated rules submodule * v1.2 changelog update (#473) * changelog update * Update CHANGELOG.md added contributor in "Fields that are not defined in eventkey_alias.txt will automatically be searched in Event.EventData." ref #442 * Update CHANGELOG-Japanese.md Fields that are not defined in eventkey_alias.txt will automatically be searched in Event.EventData. added contributor in "Fields that are not defined in eventkey_alias.txt will automatically be searched in Event.EventData." ref #442 * Update CHANGELOG.md added bug fixes (#444) and `Performance and. accuracy` add contributor ref(#395) * Update CHANGELOG-Japanese.md * Translated v1.2 change log to Japanese v1.2の内容を日本語に修正 * fixed typo added lacked back quote. * added description added following issue and pr description to readme - #216 / #469 L8 - #390 / #459 L9 - #478 / #482 L19 - #477/ #483 L20 * added description README.md added following issue and pr description to readme - #216 / #469 L8 - #390 / #459 L9 - #478 / #482 L19 - #477/ #483 L20 * changelog update * changelog update * update Co-authored-by: DustInDark <nextsasasa@gmail.com> * updated rules #493 (#494) * Resolve conflict develop (#496) * removed tools/sigmac (#441) * removed tools/sigmac - moved tools/sigmac to hayabusa-rules repo * fixed doc link tools/sigmac * fixed submodule track * fixed submodule track from latest to v1.1.0 tag * fixed link * fixed rules submodule targe #444 * updated submodule * updated rules submodule Co-authored-by: Yamato Security <71482215+YamatoSecurity@users.noreply.github.com> Co-authored-by: Yamato Security <71482215+YamatoSecurity@users.noreply.github.com> Co-authored-by: kazuminn <warugaki.k.k@gmail.com> Co-authored-by: James / hach1yon <32596618+hach1yon@users.noreply.github.com> Co-authored-by: garigariganzy <tosada31@hotmail.co.jp> Co-authored-by: itiB <is0312vx@ed.ritsumei.ac.jp>
2022-04-15 12:13:00 +09:00
parent 2b5837dfc8
commit bcf8a33e8c
50 changed files with 3726 additions and 1982 deletions
@@ -0,0 +1,161 @@
+use crate::detections::{configs, utils};
+use crate::filter;
+use crate::yaml::ParseYaml;
+use std::collections::HashMap;
+use std::fs::{self, File};
+use std::io::Write;
+pub struct LevelTuning {}
+
+impl LevelTuning {
+    pub fn run(level_tuning_config_path: &str, rules_path: &str) -> Result<(), String> {
+        let read_result = utils::read_csv(level_tuning_config_path);
+        if read_result.is_err() {
+            return Result::Err(read_result.as_ref().unwrap_err().to_string());
+        }
+
+        // Read Tuning files
+        let mut tuning_map: HashMap<String, String> = HashMap::new();
+        read_result.unwrap().into_iter().try_for_each(|line| -> Result<(), String> {
+            let id = match line.get(0) {
+                Some(_id) => {
+                    if !configs::IDS_REGEX.is_match(_id) {
+                        return Result::Err(format!("Failed to read level tuning file. {} is not correct id format, fix it.", _id));
+                    }
+                    _id
+                }
+                _ => return Result::Err("Failed to read id...".to_string())
+            };
+            let level = match line.get(1) {
+                Some(_level) => {
+                    if _level.starts_with("informational")
+                        || _level.starts_with("low")
+                        || _level.starts_with("medium")
+                        || _level.starts_with("high")
+                        || _level.starts_with("critical") {
+                            _level.split('#').collect::<Vec<&str>>()[0]
+                        } else {
+                            return Result::Err("level tuning file's level must in informational, low, medium, high, critical".to_string())
+                        }
+                    }
+                _ => return Result::Err("Failed to read level...".to_string())
+            };
+            tuning_map.insert(id.to_string(), level.to_string());
+            Ok(())
+        })?;
+
+        // Read Rule files
+        let mut rulefile_loader = ParseYaml::new();
+        let result_readdir =
+            rulefile_loader.read_dir(rules_path, "informational", &filter::exclude_ids());
+        if result_readdir.is_err() {
+            return Result::Err(format!("{}", result_readdir.unwrap_err()));
+        }
+
+        // Convert rule files
+        for (path, rule) in rulefile_loader.files {
+            if let Some(new_level) = tuning_map.get(rule["id"].as_str().unwrap()) {
+                println!("path: {}", path);
+                let mut content = match fs::read_to_string(&path) {
+                    Ok(_content) => _content,
+                    Err(e) => return Result::Err(e.to_string()),
+                };
+                let past_level = "level: ".to_string() + rule["level"].as_str().unwrap();
+
+                if new_level.starts_with("informational") {
+                    content = content.replace(&past_level, "level: informational");
+                }
+                if new_level.starts_with("low") {
+                    content = content.replace(&past_level, "level: low");
+                }
+                if new_level.starts_with("medium") {
+                    content = content.replace(&past_level, "level: medium");
+                }
+                if new_level.starts_with("high") {
+                    content = content.replace(&past_level, "level: high");
+                }
+                if new_level.starts_with("critical") {
+                    content = content.replace(&past_level, "level: critical");
+                }
+
+                let mut file = match File::options().write(true).truncate(true).open(&path) {
+                    Ok(file) => file,
+                    Err(e) => return Result::Err(e.to_string()),
+                };
+
+                file.write_all(content.as_bytes()).unwrap();
+                file.flush().unwrap();
+                println!(
+                    "level: {} -> {}",
+                    rule["level"].as_str().unwrap(),
+                    new_level
+                );
+            }
+        }
+        Result::Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    // use crate::{filter::RuleExclude, yaml};
+    // use hashbrown::HashSet;
+
+    use super::*;
+
+    #[test]
+    fn rule_level_failed_to_open_file() -> Result<(), String> {
+        let level_tuning_config_path = "./none.txt";
+        let res = LevelTuning::run(level_tuning_config_path, "");
+        let expected = Result::Err("Cannot open file. [file:./none.txt]".to_string());
+        assert_eq!(res, expected);
+        Ok(())
+    }
+
+    #[test]
+    fn rule_level_id_error_file() -> Result<(), String> {
+        let level_tuning_config_path = "./test_files/config/level_tuning_error1.txt";
+        let res = LevelTuning::run(level_tuning_config_path, "");
+        let expected = Result::Err("Failed to read level tuning file. 12345678-1234-1234-1234-12 is not correct id format, fix it.".to_string());
+        assert_eq!(res, expected);
+        Ok(())
+    }
+
+    #[test]
+    fn rule_level_level_error_file() -> Result<(), String> {
+        let level_tuning_config_path = "./test_files/config/level_tuning_error2.txt";
+        let res = LevelTuning::run(level_tuning_config_path, "");
+        let expected = Result::Err(
+            "level tuning file's level must in informational, low, medium, high, critical"
+                .to_string(),
+        );
+        assert_eq!(res, expected);
+        Ok(())
+    }
+
+    #[test]
+    fn test_level_tuning_update_rule_files() {
+        let level_tuning_config_path = "./test_files/config/level_tuning.txt";
+        let rule_str = r#"
+        id: 12345678-1234-1234-1234-123456789012
+        level: informational
+        "#;
+
+        let expected_rule = r#"
+        id: 12345678-1234-1234-1234-123456789012
+        level: high
+        "#;
+
+        let path = "test_files/rules/level_tuning_test.yml";
+        let mut file = File::create(path).unwrap();
+        let buf = rule_str.as_bytes();
+        file.write_all(buf).unwrap();
+        file.flush().unwrap();
+
+        let res = LevelTuning::run(level_tuning_config_path, path);
+        assert_eq!(res, Ok(()));
+
+        assert_eq!(fs::read_to_string(path).unwrap(), expected_rule);
+        fs::remove_file(path).unwrap();
+    }
+}
@@ -0,0 +1 @@
+pub mod level_tuning;