CSEC_PUBLIC/hayabusa
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fearture/ added output update result#410 (#452)

Browse Source 
* add git2 crate #391

* added Update option #391

* updated readme #391

* fixed cargo.lock

* fixed option if-statement #391

* changed utc short option and rule-update short option #391

* updated readme

* updated readme

* fixed -u long option & version number update #391

* added fast-forwarding rules repository #391

* updated command line option #391

* moved output logo prev update rule

* fixed readme #391

* removed recursive option in readme

* changed rules update from clone and pull to submodule update #391

* fixed document

* changed unnecessary clone recursively to clone only

* English message update.

* cargo fmt

* English message update. ( 4657c35e5c cherry-pick)

* added create rules folder when rules folder is not exist

* fixed gitmodules github-rules url from ssh to https

* added output of updated file #420

* fixed error #410

* changed update rule list seq

* added test

* fixed output #410

* fixed output and fixed output date field  when  modified field is lacked #410

* fixed compile error

* fixed output

- added enter after Latest rule update output
- added output when no exist new rule
- fixed Latest rule update date format
- changed output from 'Latest rule update' to 'Latest rules update'

* fixed compile error

* changed modified date source from rules folder to each yml rule file

* formatting use chrono in main.rs

* merge develop clippy ci

* fixed output when no update rule #410

- removed Latest rule update

- no output "Rules update successfully" when No rule changed

* Change English

Co-authored-by: Tanaka Zakku <71482215+YamatoSecurity@users.noreply.github.com>
This commit is contained in:
DustInDark
2022-03-29 13:09:54 +09:00
committed by GitHub
parent 67cf88cddd
commit fa86a9a027
1 changed files with 139 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

166
src/main.rs
View File

@@ -5,10 +5,10 @@ extern crate serde_derive;
#[cfg(target_os = "windows")]
extern crate static_vcruntime;
use chrono::Datelike;
use chrono::{DateTime, Local};
use chrono::{DateTime, Datelike, Local, TimeZone};
use evtx::{EvtxParser, ParserSettings};
use git2::Repository;
use hashbrown::{HashMap, HashSet};
use hayabusa::detections::detection::{self, EvtxRecordInfo};
use hayabusa::detections::print::{
AlertMessage, ERROR_LOG_PATH, ERROR_LOG_STACK, QUIET_ERRORS_FLAG, STATISTICS_FLAG,
@@ -16,18 +16,20 @@ use hayabusa::detections::print::{
use hayabusa::detections::rule::{get_detection_keys, RuleNode};
use hayabusa::filter;
use hayabusa::omikuji::Omikuji;
use hayabusa::yaml::ParseYaml;
use hayabusa::{afterfact::after_fact, detections::utils};
use hayabusa::{detections::configs, timeline::timelines::Timeline};
use hhmmss::Hhmmss;
use pbr::ProgressBar;
use serde_json::Value;
use std::collections::{HashMap, HashSet};
use std::cmp::Ordering;
use std::ffi::OsStr;
use std::fmt::Display;
use std::fs::create_dir;
use std::io::BufWriter;
use std::path::Path;
use std::sync::Arc;
use std::time::SystemTime;
use std::{
fs::{self, File},
path::PathBuf,
@@ -86,7 +88,11 @@ impl App {
.is_present("update-rules")
{
match self.update_rules() {
Ok(_ok) => println!("Rules updated successfully."),
Ok(output) => {
if output != "You currently have the latest rules." {
println!("Rules updated successfully.");
}
}
Err(e) => {
AlertMessage::alert(
&mut BufWriter::new(std::io::stderr().lock()),
@@ -95,6 +101,7 @@ impl App {
.ok();
}
}
println!();
return;
}
if !Path::new("./config").exists() {
@@ -506,22 +513,29 @@ impl App {
}
/// update rules(hayabusa-rules subrepository)
fn update_rules(&self) -> Result<(), git2::Error> {
fn update_rules(&self) -> Result<String, git2::Error> {
let mut result;
let mut prev_modified_time: SystemTime = SystemTime::UNIX_EPOCH;
let mut prev_modified_rules: HashSet<String> = HashSet::default();
let hayabusa_repo = Repository::open(Path::new("."));
let hayabusa_rule_repo = Repository::open(Path::new("./rules"));
let hayabusa_rule_repo = Repository::open(Path::new("rules"));
if hayabusa_repo.is_err() && hayabusa_rule_repo.is_err() {
println!(
"Attempting to git clone the hayabusa-rules repository into the rules folder."
);
// レポジトリが開けなかった段階でhayabusa rulesのgit cloneを実施する
self.clone_rules()
result = self.clone_rules();
} else if hayabusa_rule_repo.is_ok() {
// rulesのrepositoryが確認できる場合
// origin/mainのfetchができなくなるケースはネットワークなどのケースが考えられるため、git cloneは実施しない
self.pull_repository(hayabusa_rule_repo.unwrap())
prev_modified_rules = self.get_updated_rules("rules", &prev_modified_time);
prev_modified_time = fs::metadata("rules").unwrap().modified().unwrap();
result = self.pull_repository(hayabusa_rule_repo.unwrap());
} else {
//hayabusa repositoryがあればsubmodule情報もあると思われるのでupdate
let rules_path = Path::new("./rules");
// hayabusa-rulesのrepositoryがrulesに存在しない場合
// hayabusa repositoryがあればsubmodule情報もあると思われるのでupdate
prev_modified_time = fs::metadata("rules").unwrap().modified().unwrap();
let rules_path = Path::new("rules");
if !rules_path.exists() {
create_dir(rules_path).ok();
}
@@ -533,28 +547,31 @@ impl App {
for mut submodule in submodules {
submodule.update(true, None)?;
let submodule_repo = submodule.open()?;
match self.pull_repository(submodule_repo) {
Ok(it) => it,
Err(e) => {
AlertMessage::alert(
&mut BufWriter::new(std::io::stderr().lock()),
&format!("Failed submodule update. {}", e),
)
.ok();
is_success_submodule_update = false;
}
if let Err(e) = self.pull_repository(submodule_repo) {
AlertMessage::alert(
&mut BufWriter::new(std::io::stderr().lock()),
&format!("Failed submodule update. {}", e),
)
.ok();
is_success_submodule_update = false;
}
}
if is_success_submodule_update {
Ok(())
result = Ok("Successed submodule update".to_string());
} else {
Err(git2::Error::from_str(&String::default()))
result = Err(git2::Error::from_str(&String::default()));
}
}
if result.is_ok() {
let updated_modified_rules = self.get_updated_rules("rules", &prev_modified_time);
result =
self.print_diff_modified_rule_dates(prev_modified_rules, updated_modified_rules);
}
result
}
/// Pull(fetch and fast-forward merge) repositoryto input_repo.
fn pull_repository(&self, input_repo: Repository) -> Result<(), git2::Error> {
fn pull_repository(&self, input_repo: Repository) -> Result<String, git2::Error> {
match input_repo
.find_remote("origin")?
.fetch(&["main"], None, None)
@@ -572,13 +589,13 @@ impl App {
let fetch_commit = input_repo.reference_to_annotated_commit(&fetch_head)?;
let analysis = input_repo.merge_analysis(&[&fetch_commit])?;
if analysis.0.is_up_to_date() {
Ok(())
Ok("Already up to date".to_string())
} else if analysis.0.is_fast_forward() {
let mut reference = input_repo.find_reference("refs/heads/main")?;
reference.set_target(fetch_commit.id(), "Fast-Forward")?;
input_repo.set_head("refs/heads/main")?;
input_repo.checkout_head(Some(git2::build::CheckoutBuilder::default().force()))?;
Ok(())
Ok("Finished fast forward merge.".to_string())
} else if analysis.0.is_normal() {
AlertMessage::alert(
&mut BufWriter::new(std::io::stderr().lock()),
@@ -592,14 +609,14 @@ impl App {
}
/// git clone でhauyabusa-rules レポジトリをrulesフォルダにgit cloneする関数
fn clone_rules(&self) -> Result<(), git2::Error> {
fn clone_rules(&self) -> Result<String, git2::Error> {
match Repository::clone(
"https://github.com/Yamato-Security/hayabusa-rules.git",
"rules",
) {
Ok(_repo) => {
println!("Finished cloning the hayabusa-rules repository.");
Ok(())
Ok("Finished clone".to_string())
}
Err(e) => {
AlertMessage::alert(
@@ -614,11 +631,90 @@ impl App {
}
}
}
/// Create rules folder files Hashset. Format is "[rule title in yaml]|[filepath]|[filemodified date]|[rule type in yaml]"
fn get_updated_rules(
&self,
rule_folder_path: &str,
target_date: &SystemTime,
) -> HashSet<String> {
let mut rulefile_loader = ParseYaml::new();
// level in read_dir is hard code to check all rules.
rulefile_loader
.read_dir(
rule_folder_path,
"INFORMATIONAL",
&filter::RuleExclude {
no_use_rule: HashSet::new(),
},
)
.ok();
let hash_set_keys: HashSet<String> = rulefile_loader
.files
.into_iter()
.filter_map(|(filepath, yaml)| {
let file_modified_date = fs::metadata(&filepath).unwrap().modified().unwrap();
if file_modified_date.cmp(target_date).is_gt() {
let yaml_date = yaml["date"].as_str().unwrap_or("-");
return Option::Some(format!(
"{}|{}|{}|{}",
yaml["title"].as_str().unwrap_or(&String::default()),
yaml["modified"].as_str().unwrap_or(yaml_date),
&filepath,
yaml["ruletype"].as_str().unwrap_or("Other")
));
}
Option::None
})
.collect();
hash_set_keys
}
/// print updated rule files.
fn print_diff_modified_rule_dates(
&self,
prev_sets: HashSet<String>,
updated_sets: HashSet<String>,
) -> Result<String, git2::Error> {
let diff = updated_sets.difference(&prev_sets);
let mut update_count_by_rule_type: HashMap<String, u128> = HashMap::new();
let mut latest_update_date = Local.timestamp(0, 0);
for diff_key in diff {
let tmp: Vec<&str> = diff_key.split('|').collect();
let file_modified_date = fs::metadata(&tmp[2]).unwrap().modified().unwrap();
let dt_local: DateTime<Local> = file_modified_date.into();
if latest_update_date.cmp(&dt_local) == Ordering::Less {
latest_update_date = dt_local;
}
*update_count_by_rule_type
.entry(tmp[3].to_string())
.or_insert(0b0) += 1;
println!(
"[Updated] {} (Modified: {} | Path: {})",
tmp[0], tmp[1], tmp[2]
);
}
println!();
for (key, value) in &update_count_by_rule_type {
println!("Updated {} rules: {}", key, value);
}
if !&update_count_by_rule_type.is_empty() {
Ok("Rule updated".to_string())
} else {
println!("You currently have the latest rules.");
Ok("You currently have the latest rules.".to_string())
}
}
}
#[cfg(test)]
mod tests {
use crate::App;
use std::time::SystemTime;
#[test]
fn test_collect_evtxfiles() {
@@ -635,4 +731,20 @@ mod tests {
assert_eq!(is_contains, &true);
})
}
#[test]
fn test_get_updated_rules() {
let app = App::new();
let prev_modified_time: SystemTime = SystemTime::UNIX_EPOCH;
let prev_modified_rules =
app.get_updated_rules("test_files/rules/level_yaml", &prev_modified_time);
assert_eq!(prev_modified_rules.len(), 5);
let target_time: SystemTime = SystemTime::now();
let prev_modified_rules2 =
app.get_updated_rules("test_files/rules/level_yaml", &target_time);
assert_eq!(prev_modified_rules2.len(), 0);
}
}