Feature/event stats#105 (#137)

Event集計機能実装

Co-authored-by: HajimeTakai <takai.wa.hajime@gmail.com>

config/timeline_event_info.txt

@@ -0,0 +1,73 @@
+eventid,event_title,detect_flg,comment
+1100,Event logging service was shut down,,Good for finding signs of anti-forensics but most likely false positives when the system shuts down.
+1101,Audit Events Have Been Dropped By The Transport,,
+1102,Event log was cleared,Yes,Should not happen normally so this is a good event to look out for.
+1107,Event processing error,,
+4608,Windows started up,,
+4610,An authentication package has been loaded by the Local Security Authority,,
+4611,A trusted logon process has been registered with the Local Security Authority,,
+4614,A notification package has been loaded by the Security Account Manager,,
+4616,System time was changed,,
+4622,A security package has been loaded by the Local Security Authority,,
+4624,Account logon,Yes,
+4625,Failed logon,Yes,
+4634,Logoff,Yes,
+4647,Logoff,Yes,
+4648,Explicit logon,Yes,
+4672,Admin logon,Yes,
+4688,New process started,,
+4696,Primary token assigned to process,,
+4692,Backup of data protection master key was attempted,,
+4697,Service installed,,
+4717,System security access was granted to an account,,
+4719,System audit policy was changed,,
+4720,User account created,Yes,
+4722,User account enabled,,
+4724,Password reset,,
+4725,User account disabled,,
+4726,User account deleted,,
+4728,User added to security global group,,
+4729,User removed from security global group,,
+4732,User added to security local group,,
+4733,User removed from security local group,,
+4735,Security local group was changed,,
+4727,Security global group was changed,,
+4738,User accounts properties changed,,
+4739,Domain policy changed,,
+4776,NTLM logon to local user,,
+4778,RDP session reconnected or user switched back through Fast User Switching,,
+4779,RDP session disconnected or user switched away through Fast User Switching,,
+4797,Attempt to query the account for a blank password,,
+4798,Users local group membership was enumerated,,
+4799,Local group membership was enumerated,,
+4781,User name was changed,,
+4800,Workstation was locked,,
+4801,Workstation was unlocked,,
+4826,Boot configuration data loaded,,
+4902,Per-user audit policy table was created,,
+4904,Attempt to register a security event source,,
+4905,Attempt to unregister a security event source,,
+4907,Auditing settings on object was changed,,
+4944,Policy active when firewall started,,
+4945,Rule listed when the firewall started,,Too much noise when firewall starts
+4946,Rule added to firewall exception list,,
+4947,Rule modified in firewall exception list,,
+4948,Rule deleted in firewall exception list,,
+4954,New setting applied to firewall group policy,,
+4956,Firewall active profile changed,,
+5024,Firewall started,,
+5033,Firewall driver started,,
+5038,Code integrity determined that the image hash of a file is not valid,,
+5058,Key file operation,,
+5059,Key migration operation,,
+5061,Cryptographic operation,,
+5140,Network share object was accessed,,
+5142,A network share object was added,,
+5144,A network share object was deleted,,
+5379,Credential Manager credentials were read,,
+5381,Vault credentials were read,,
+5382,Vault credentials were read,,
+5478,IPsec Services started,,
+5889,An object was deleted to the COM+ Catalog,,
+5890,An object was added to the COM+ Catalog,,
+unregistered_event_id,Unknown,,

src/detections/configs.rs

@@ -11,6 +11,7 @@ lazy_static! {
 pub struct ConfigReader {
     pub args: ArgMatches<'static>,
     pub event_key_alias_config: EventKeyAliasConfig,
+    pub event_timeline_config: EventInfoConfig,
 }
 
 impl ConfigReader {
@@ -18,6 +19,7 @@ impl ConfigReader {
         ConfigReader {
             args: build_app(),
             event_key_alias_config: load_eventkey_alias("config/eventkey_alias.txt"),
+            event_timeline_config: load_eventcode_info("config/timeline_event_info.txt"),
         }
     }
 }
@@ -107,7 +109,76 @@ fn load_eventkey_alias(path: &str) -> EventKeyAliasConfig {
             .key_to_eventkey
             .insert(alias.to_owned(), event_key.to_owned());
     });
+    return config;
+}
 
+#[derive(Debug, Clone)]
+pub struct EventInfo {
+    pub evttitle: String,
+    pub detectflg: String,
+    pub comment: String,
+}
+
+impl EventInfo {
+    pub fn new() -> EventInfo {
+        let evttitle = "Unknown".to_string();
+        let detectflg = "".to_string();
+        let comment = "".to_string();
+        return EventInfo {
+            evttitle,
+            detectflg,
+            comment,
+        };
+    }
+}
+#[derive(Debug, Clone)]
+pub struct EventInfoConfig {
+    eventinfo: HashMap<String, EventInfo>,
+}
+
+impl EventInfoConfig {
+    pub fn new() -> EventInfoConfig {
+        return EventInfoConfig {
+            eventinfo: HashMap::new(),
+        };
+    }
+    pub fn get_event_id(&self, eventid: &String) -> Option<&EventInfo> {
+        return self.eventinfo.get(eventid);
+    }
+
+    pub fn get_event_info(&self) -> Vec<(&String, &EventInfo)> {
+        return self.eventinfo.iter().map(|e| e).collect();
+    }
+
+    //    pub fn get_event_key_values(&self) -> Vec<(&String, &String)> {
+    //        return self.timeline_eventcode_info.iter().map(|e| e).collect();
+    //    }
+}
+
+fn load_eventcode_info(path: &str) -> EventInfoConfig {
+    let mut infodata = EventInfo::new();
+    let mut config = EventInfoConfig::new();
+    let read_result = utils::read_csv(path);
+    // timeline_event_infoが読み込めなかったらエラーで終了とする。
+    read_result.unwrap().into_iter().for_each(|line| {
+        if line.len() != 4 {
+            return;
+        }
+
+        let empty = &"".to_string();
+        let eventcode = line.get(0).unwrap_or(empty);
+        let event_title = line.get(1).unwrap_or(empty);
+        let detect_flg = line.get(2).unwrap_or(empty);
+        let comment = line.get(3).unwrap_or(empty);
+        infodata = EventInfo {
+            evttitle: event_title.to_string(),
+            detectflg: detect_flg.to_string(),
+            comment: comment.to_string(),
+        };
+        config
+            .eventinfo
+            .insert(eventcode.to_owned(), infodata.to_owned());
+    });
     return config;
 }
 

src/detections/detection.rs

@@ -4,13 +4,13 @@ use crate::detections::rule::AggResult;
 use serde_json::Value;
 use tokio::{runtime::Runtime, spawn, task::JoinHandle};
 
+use crate::detections::print::AlertMessage;
 use crate::detections::print::MESSAGES;
 use crate::detections::rule;
 use crate::detections::rule::RuleNode;
-use crate::detections::{print::AlertMessage, utils};
 use crate::yaml::ParseYaml;
 
-use std::{sync::Arc, usize};
+use std::sync::Arc;
 
 const DIRPATH_RULES: &str = "rules";
 

src/detections/rule/matchers.rs

@@ -532,7 +532,7 @@ mod tests {
         creation_date: 2020/11/8
         updated_date: 2020/11/8
         "#;
-        let mut rule_node = parse_rule_from_str(rule_str);
+        let rule_node = parse_rule_from_str(rule_str);
         let selection_node = &rule_node.detection.unwrap().name_to_selection["selection"];
 
         // Root

src/detections/rule/mod.rs

@@ -1,7 +1,7 @@
 extern crate regex;
 use crate::detections::print::Message;
 
-use chrono::{DateTime, TimeZone, Utc};
+use chrono::{DateTime, Utc};
 
 use std::{collections::HashMap, fmt::Debug, sync::Arc, vec};
 
@@ -30,7 +30,7 @@ pub struct RuleNode {
 }
 
 impl Debug for RuleNode {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         return Result::Ok(());
     }
 }
@@ -906,7 +906,7 @@ mod tests {
     }
 
     /// countで対象の数値確認を行うためのテスト用関数
-    fn check_count(rule_str: &str, record_str: &str, key: &str, expect_count: i32) {
+    fn _check_count(rule_str: &str, record_str: &str, key: &str, expect_count: i32) {
         let mut rule_yaml = YamlLoader::load_from_str(rule_str).unwrap().into_iter();
         let test = rule_yaml.next().unwrap();
         let mut rule_node = create_rule(test);

src/main.rs

@@ -5,7 +5,6 @@ use evtx::{EvtxParser, ParserSettings};
 use std::{
     fs::{self, File},
     path::PathBuf,
-    time::Instant,
     vec,
 };
 use yamato_event_analyzer::detections::detection;
@@ -74,13 +73,11 @@ fn print_credits() {
 }
 
 fn analysis_files(evtx_files: Vec<PathBuf>) {
-    let mut tl = Timeline::new();
     let mut detection = detection::Detection::new(detection::Detection::parse_rule_files());
 
     for evtx_file in evtx_files {
-        let ret = analysis_file(evtx_file, tl, detection);
+        let ret = analysis_file(evtx_file, detection);
-        tl = ret.0;
+        detection = ret;
-        detection = ret.1;
     }
 
     after_fact();
@@ -89,18 +86,19 @@ fn analysis_files(evtx_files: Vec<PathBuf>) {
 // Windowsイベントログファイルを1ファイル分解析する。
 fn analysis_file(
     evtx_filepath: PathBuf,
-    mut tl: Timeline,
     mut detection: detection::Detection,
-) -> (Timeline, detection::Detection) {
+) -> detection::Detection {
     let filepath_disp = evtx_filepath.display();
     let parser = evtx_to_jsons(evtx_filepath.clone());
     if parser.is_none() {
-        return (tl, detection);
+        return detection;
     }
 
+    let mut tl = Timeline::new();
     let mut parser = parser.unwrap();
     let mut records = parser.records_json_value();
     let tokio_rt = utils::create_tokio_runtime();
+
     loop {
         let mut records_per_detect = vec![];
         while records_per_detect.len() < MAX_DETECT_RECORDS {
@@ -139,8 +137,9 @@ fn analysis_file(
 
     tokio_rt.shutdown_background();
     detection.add_aggcondtion_msg();
+    tl.tm_stats_dsp_msg();
 
-    return (tl, detection);
+    return detection;
 }
 
 fn evtx_to_jsons(evtx_filepath: PathBuf) -> Option<EvtxParser<File>> {

src/timeline/statistics.rs

@@ -1,24 +1,32 @@
-use crate::detections::{configs, detection::EvtxRecordInfo};
+use crate::detections::{configs, detection::EvtxRecordInfo, utils};
+use std::collections::HashMap;
 
 #[derive(Debug)]
-pub struct EventStatistics {}
+pub struct EventStatistics {
+    pub total: usize,
+    pub start_time: String,
+    pub end_time: String,
+    pub stats_list: HashMap<String, usize>,
+}
 /**
 * Windows Event Logの統計情報を出力する
 */
 impl EventStatistics {
-    pub fn new() -> EventStatistics {
+    pub fn new(
-        return EventStatistics {};
+        total: usize,
+        start_time: String,
+        end_time: String,
+        stats_list: HashMap<String, usize>,
+    ) -> EventStatistics {
+        return EventStatistics {
+            total,
+            start_time,
+            end_time,
+            stats_list,
+        };
     }
 
-    // この関数の戻り値として、コンソールに出力する内容をStringの可変配列(Vec)として返却してください。
+    pub fn start(&mut self, records: &Vec<EvtxRecordInfo>) {
-    // 可変配列にしているのは改行を表すためで、可変配列にコンソールに出力する内容を1行ずつ追加してください。
-    // 引数の_recordsが読み込んだWindowsイベントログのを表す、EvtxRecordInfo構造体の配列になっています。
-    // EvtxRecordInfo構造体の pub record: Value というメンバーがいて、それがWindowsイベントログの1レコード分を表していますので、
-    // EvtxRecordInfo構造体のrecordから、EventIDとか統計情報を取得するようにしてください。
-    // recordからEventIDを取得するには、detection::utils::get_event_value()という関数があるので、それを使うと便利かもしれません。
-
-    // 現状では、この関数の戻り値として返すVec<String>を表示するコードは実装していません。
-    pub fn start(&mut self, _records: &Vec<EvtxRecordInfo>) -> Vec<String> {
         // 引数でstatisticsオプションが指定されている時だけ、統計情報を出力する。
         if !configs::CONFIG
             .read()
@@ -26,11 +34,62 @@ impl EventStatistics {
             .args
             .is_present("statistics")
         {
-            return vec![];
+            return;
         }
 
-        // TODO ここから下を書いて欲しいです。
+        //let mut filesize = 0;
+        // _recordsから、EventIDを取り出す。
+        self.stats_time_cnt(records);
 
-        return vec![];
+        // EventIDで集計
+        //let evtstat_map = HashMap::new();
+        self.stats_eventid(records);
+    }
+
+    fn stats_time_cnt(&mut self, records: &Vec<EvtxRecordInfo>) {
+        if records.len() == 0 {
+            return;
+        }
+
+        // sortしなくてもイベントログのTimeframeを取得できるように修正しました。
+        // sortしないことにより計算量が改善されています。
+        // もうちょっと感じに書けるといえば書けます。
+        for record in records.iter() {
+            let evttime = utils::get_event_value(
+                &"Event.System.TimeCreated_attributes.SystemTime".to_string(),
+                &record.record,
+            )
+            .and_then(|evt_value| {
+                return Option::Some(evt_value.to_string());
+            });
+            if evttime.is_none() {
+                continue;
+            }
+
+            let evttime = evttime.unwrap();
+            if self.start_time.len() == 0 || evttime < self.start_time {
+                self.start_time = evttime.to_string();
+            }
+            if self.end_time.len() == 0 || evttime > self.end_time {
+                self.end_time = evttime;
+            }
+        }
+        self.total += records.len();
+    }
+
+    // EventIDで集計
+    fn stats_eventid(&mut self, records: &Vec<EvtxRecordInfo>) {
+        //        let mut evtstat_map = HashMap::new();
+        for record in records.iter() {
+            let evtid = utils::get_event_value(&"EventID".to_string(), &record.record);
+            if evtid.is_none() {
+                continue;
+            }
+
+            let idnum = evtid.unwrap();
+            let count: &mut usize = self.stats_list.entry(idnum.to_string()).or_insert(0);
+            *count += 1;
+        }
+        //        return evtstat_map;
     }
 }

src/timeline/timeline.rs

@@ -1,17 +1,98 @@
-use crate::detections::detection::EvtxRecordInfo;
+use crate::detections::{configs, detection::EvtxRecordInfo};
 
 use super::statistics::EventStatistics;
+use std::collections::HashMap;
 
 #[derive(Debug)]
-pub struct Timeline {}
+pub struct Timeline {
+    pub stats: EventStatistics,
+}
 
 impl Timeline {
     pub fn new() -> Timeline {
-        return Timeline {};
+        let totalcnt = 0;
+        let starttm = "".to_string();
+        let endtm = "".to_string();
+        let statslst = HashMap::new();
+
+        let statistic = EventStatistics::new(totalcnt, starttm, endtm, statslst);
+        return Timeline { stats: statistic };
     }
 
     pub fn start(&mut self, records: &Vec<EvtxRecordInfo>) {
-        let mut statistic = EventStatistics::new();
+        self.stats.start(records);
-        statistic.start(records);
+    }
+
+    pub fn tm_stats_dsp_msg(&mut self) {
+        if !configs::CONFIG
+            .read()
+            .unwrap()
+            .args
+            .is_present("statistics")
+        {
+            return;
+        }
+        // 出力メッセージ作成
+        //println!("map -> {:#?}", evtstat_map);
+        let mut sammsges: Vec<String> = Vec::new();
+        sammsges.push("---------------------------------------".to_string());
+        sammsges.push(format!("Total_counts : {}\n", self.stats.total));
+        sammsges.push(format!("firstevent_time: {}", self.stats.start_time));
+        sammsges.push(format!("lastevent_time: {}\n", self.stats.end_time));
+        sammsges.push("count(rate)\tID\tevent\t\ttimeline".to_string());
+        sammsges.push("--------------- ------- --------------- -------".to_string());
+
+        // 集計件数でソート
+        let mut mapsorted: Vec<_> = self.stats.stats_list.iter().collect();
+        mapsorted.sort_by(|x, y| y.1.cmp(&x.1));
+
+        // イベントID毎の出力メッセージ生成
+        let stats_msges: Vec<String> = self.tm_stats_set_msg(mapsorted);
+
+        for msgprint in sammsges.iter() {
+            println!("{}", msgprint);
+        }
+        for msgprint in stats_msges.iter() {
+            println!("{}", msgprint);
+        }
+    }
+    // イベントID毎の出力メッセージ生成
+    fn tm_stats_set_msg(&self, mapsorted: Vec<(&std::string::String, &usize)>) -> Vec<String> {
+        let mut msges: Vec<String> = Vec::new();
+
+        for (event_id, event_cnt) in mapsorted.iter() {
+            // 件数の割合を算出
+            let rate: f32 = **event_cnt as f32 / self.stats.total as f32;
+
+            // イベント情報取得(eventtitleなど)
+            let conf = configs::CONFIG.read().unwrap();
+            // timeline_event_info.txtに登録あるものは情報設定
+            match conf.event_timeline_config.get_event_id(*event_id) {
+                Some(e) => {
+                    // 出力メッセージ1行作成
+                    msges.push(format!(
+                        "{0} ({1:.1}%)\t{2}\t{3}\t{4}",
+                        event_cnt,
+                        (rate * 1000.0).round() / 10.0,
+                        event_id,
+                        e.evttitle,
+                        e.detectflg
+                    ));
+                }
+                None => {
+                    // 出力メッセージ1行作成
+                    msges.push(format!(
+                        "{0} ({1:.1}%)\t{2}\t{3}\t{4}",
+                        event_cnt,
+                        (rate * 1000.0).round() / 10.0,
+                        event_id,
+                        "Unknown".to_string(),
+                        "".to_string()
+                    ));
+                }
+            }
+        }
+        msges.push("---------------------------------------".to_string());
+        return msges;
     }
 }