Merge branch 'master' into feature/#11

Cargo.lock

@@ -1071,6 +1071,15 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "toml"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffc92d160b1eef40665be3a05630d003936a3bc7da7421277846c2613e92c71a"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "unicode-width"
 version = "0.1.8"
@@ -1185,5 +1194,7 @@ dependencies = [
  "quick-xml 0.17.2",
  "regex",
  "serde",
+ "serde_derive",
  "serde_json",
+ "toml",
 ]

Cargo.toml

@@ -11,8 +11,10 @@ evtx = { git = "https://github.com/omerbenamram/evtx.git" }
 quick-xml = {version = "0.17", features = ["serialize"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0"}
+serde_derive = "1.0"
 clap = "*"
 regex = "1"
 csv = "1.1"
 base64 = "*"
 flate2 = "1.0"
+toml = "0.5"

rules/test.toml

@@ -0,0 +1,2 @@
+[rule]
+severity = "high"

src/detections/configs.rs

@@ -0,0 +1,50 @@
+use std::fs::File;
+use std::io::prelude::*;
+use std::sync::Once;
+
+#[derive(Clone)]
+pub struct SingletonReader {
+    pub regex: Vec<Vec<String>>,
+    pub whitelist: Vec<Vec<String>>,
+}
+
+pub fn singleton() -> Box<SingletonReader> {
+    static mut SINGLETON: Option<Box<SingletonReader>> = Option::None;
+    static ONCE: Once = Once::new();
+
+    unsafe {
+        ONCE.call_once(|| {
+            let singleton = SingletonReader {
+                regex: read_csv("regexes.txt"),
+                whitelist: read_csv("whitelist.txt"),
+            };
+
+            SINGLETON = Some(Box::new(singleton));
+        });
+
+        return SINGLETON.clone().unwrap();
+    }
+}
+
+fn read_csv(filename: &str) -> Vec<Vec<String>> {
+    let mut f = File::open(filename).expect("file not found!!!");
+    let mut contents: String = String::new();
+    let mut ret = vec![];
+    if f.read_to_string(&mut contents).is_err() {
+        return ret;
+    }
+
+    let mut rdr = csv::Reader::from_reader(contents.as_bytes());
+    rdr.records().for_each(|r| {
+        if r.is_err() {
+            return;
+        }
+
+        let line = r.unwrap();
+        let mut v = vec![];
+        line.iter().for_each(|s| v.push(s.to_string()));
+        ret.push(v);
+    });
+
+    return ret;
+}

src/detections/detection.rs

@@ -11,8 +11,6 @@ use crate::models::event;
 use evtx::EvtxParser;
 use quick_xml::de::DeError;
 use std::collections::BTreeMap;
-use std::fs::File;
-use std::io::prelude::*;
 
 #[derive(Debug)]
 pub struct Detection {
@@ -34,12 +32,6 @@ impl Detection {
         let mut sysmon = sysmon::Sysmon::new();
         let mut powershell = powershell::PowerShell::new();
 
-        let mut f = File::open("whitelist.txt").expect("file not found");
-        let mut contents = String::new();
-        let _ = f.read_to_string(&mut contents);
-
-        let mut rdr = csv::Reader::from_reader(contents.as_bytes());
-
         for record in parser.records() {
             match record {
                 Ok(r) => {
@@ -57,7 +49,7 @@ impl Detection {
                     } else if channel == "Application" {
                         &application.detection(event_id, &event.system, event_data);
                     } else if channel == "Microsoft-Windows-PowerShell/Operational" {
-                        &powershell.detection(event_id, &event.system, event_data, &mut rdr);
+                        &powershell.detection(event_id, &event.system, event_data);
                     } else if channel == "Microsoft-Windows-Sysmon/Operational" {
                         &sysmon.detection(event_id, &event.system, event_data);
                     } else {

src/detections/mod.rs

@@ -1,5 +1,6 @@
 mod application;
 mod common;
+mod configs;
 pub mod detection;
 mod powershell;
 mod security;

src/detections/powershell.rs

@@ -16,20 +16,15 @@ impl PowerShell {
         event_id: String,
         _system: &event::System,
         event_data: HashMap<String, String>,
-        rdr: &mut csv::Reader<&[u8]>,
     ) {
         if event_id == "4103" {
-            &self.execute_pipeline(&event_data, rdr);
+            &self.execute_pipeline(&event_data);
         } else if event_id == "4104" {
-            &self.execute_remote_command(&event_data, rdr);
+            &self.execute_remote_command(&event_data);
         }
     }
 
-    fn execute_pipeline(
-        &mut self,
-        event_data: &HashMap<String, String>,
-        rdr: &mut csv::Reader<&[u8]>,
-    ) {
+    fn execute_pipeline(&mut self, event_data: &HashMap<String, String>) {
         // パイプライン実行をしています
         let default = String::from("");
         let commandline = event_data.get("ContextInfo").unwrap_or(&default);
@@ -45,16 +40,12 @@ impl PowerShell {
             let command = rm_after.replace_all(&temp_command_with_extra, "");
 
             if command != "" {
-                utils::check_command(4103, &command, 1000, 0, &default, &default, rdr);
+                utils::check_command(4103, &command, 1000, 0, &default, &default);
             }
         }
     }
 
-    fn execute_remote_command(
-        &mut self,
-        event_data: &HashMap<String, String>,
-        rdr: &mut csv::Reader<&[u8]>,
-    ) {
+    fn execute_remote_command(&mut self, event_data: &HashMap<String, String>) {
         // リモートコマンドを実行します
         let default = String::from("");
         let path = event_data.get("Path").unwrap().to_string();

src/detections/utils.rs

@@ -2,9 +2,9 @@ extern crate base64;
 extern crate csv;
 extern crate regex;
 
+use crate::detections::configs;
 use flate2::read::GzDecoder;
 use regex::Regex;
-use std::fs::File;
 use std::io::prelude::*;
 use std::str;
 use std::string::String;
@@ -16,20 +16,23 @@ pub fn check_command(
     servicecmd: usize,
     servicename: &str,
     creator: &str,
-    rdr: &mut csv::Reader<&[u8]>,
 ) {
     let mut text = "".to_string();
     let mut base64 = "".to_string();
 
-    for entry in rdr.records() {
-        if let Ok(_data) = entry {
-            if let Ok(_re) = Regex::new(&_data[0]) {
-                if _re.is_match(commandline) {
+    let empty = "".to_string();
+    for line in configs::singleton().whitelist {
+        let r_str = line.get(0).unwrap_or(&empty);
+        if r_str.is_empty() {
+            continue;
+        }
+
+        let r = Regex::new(r_str);
+        if r.is_ok() && r.unwrap().is_match(commandline) {
             return;
         }
     }
-        }
-    }
+
     if commandline.len() > minlength {
         text.push_str("Long Command Line: greater than ");
         text.push_str(&minlength.to_string());
@@ -124,33 +127,33 @@ fn check_obfu(string: &str) -> std::string::String {
 }
 
 fn check_regex(string: &str, r#type: usize) -> std::string::String {
-    let mut f = File::open("regexes.txt").expect("file not found");
-    let mut contents = String::new();
-    let ret = f.read_to_string(&mut contents);
-    if let Err(_) = ret {
-        return "".to_string();
+    let empty = "".to_string();
+    let mut regextext = "".to_string();
+    for line in configs::singleton().regex {
+        let type_str = line.get(0).unwrap_or(&empty);
+        if type_str != &r#type.to_string() {
+            continue;
         }
 
-    let mut rdr = csv::Reader::from_reader(contents.as_bytes());
+        let regex_str = line.get(1).unwrap_or(&empty);
+        if regex_str.is_empty() {
+            continue;
+        }
 
-    let mut regextext = "".to_string();
-    for regex in rdr.records() {
-        if let Ok(_data) = regex {
-            /*
-            data[0] is type in csv.
-            data[1] is regex in csv.
-            data[2] is string in csv.
-            */
-            if &_data[0] == r#type.to_string() {
-                if let Ok(_re) = Regex::new(&_data[1]) {
-                    if _re.is_match(string) {
-                        regextext.push_str(&_data[2]);
+        let re = Regex::new(regex_str);
+        if re.is_err() || re.unwrap().is_match(string) == false {
+            continue;
+        }
+
+        let text = line.get(2).unwrap_or(&empty);
+        if text.is_empty() {
+            continue;
+        }
+
+        regextext.push_str(text);
         regextext.push_str("\n");
     }
-                }
-            }
-        }
-    }
+
     return regextext;
 }
 
@@ -175,8 +178,6 @@ fn check_creator(command: &str, creator: &str) -> std::string::String {
 #[cfg(test)]
 mod tests {
     use crate::detections::utils;
-    use std::fs::File;
-    use std::io::Read;
     #[test]
     fn test_check_regex() {
         let regextext = utils::check_regex("\\cvtres.exe", 0);
@@ -199,12 +200,7 @@ mod tests {
 
     #[test]
     fn test_check_command() {
-        let mut f = File::open("whitelist.txt").expect("file not found");
-        let mut contents = String::new();
-        f.read_to_string(&mut contents);
-
-        let mut rdr = csv::Reader::from_reader(contents.as_bytes());
-        utils::check_command(1, "dir", 100, 100, "dir", "dir", &mut rdr);
+        utils::check_command(1, "dir", 100, 100, "dir", "dir");
 
         //test return with whitelist.
         utils::check_command(
@@ -214,7 +210,6 @@ mod tests {
             100,
             "dir",
             "dir",
-            &mut rdr,
         );
     }
 }

src/lib.rs

@@ -1,2 +1,3 @@
 pub mod detections;
 pub mod models;
+pub mod toml;

src/main.rs

@@ -6,6 +6,7 @@ use evtx::EvtxParser;
 use quick_xml::de::DeError;
 use std::{path::PathBuf, process};
 use yamato_event_analyzer::detections::detection;
+use yamato_event_analyzer::toml;
 
 fn build_app() -> clap::App<'static, 'static> {
     let program = std::env::args()

src/models/mod.rs

@@ -1 +1,2 @@
 pub mod event;
+pub mod rule;

src/models/rule.rs

@@ -0,0 +1,13 @@
+extern crate serde;
+use serde::Deserialize;
+
+#[derive(Debug, Deserialize)]
+pub struct Rule {
+    pub severity: Option<String>,
+    pub name: Option<String>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct Toml {
+    pub rule: Rule,
+}

src/toml.rs

@@ -0,0 +1,69 @@
+extern crate serde_derive;
+extern crate toml;
+
+use crate::models::rule;
+use std::fs;
+use std::io;
+use std::io::{BufReader, Read};
+use std::path::{Path, PathBuf};
+
+pub struct ParseToml {
+    pub rules: Vec<Result<rule::Toml, toml::de::Error>>,
+}
+
+impl ParseToml {
+    pub fn new() -> ParseToml {
+        ParseToml { rules: Vec::new() }
+    }
+
+    fn read_file(&self, path: PathBuf) -> Result<String, String> {
+        let mut file_content = String::new();
+
+        let mut fr = fs::File::open(path)
+            .map(|f| BufReader::new(f))
+            .map_err(|e| e.to_string())?;
+
+        fr.read_to_string(&mut file_content)
+            .map_err(|e| e.to_string())?;
+
+        Ok(file_content)
+    }
+
+    fn read_dir<P: AsRef<Path>>(&mut self, path: P) -> io::Result<String> {
+        Ok(fs::read_dir(path)?
+            .filter_map(|entry| {
+                let entry = entry.ok()?;
+                if entry.file_type().ok()?.is_file() {
+                    match self.read_file(entry.path()) {
+                        Ok(s) => &self.rules.push(toml::from_str(&s)),
+                        Err(e) => panic!("fail to read file: {}", e),
+                    };
+                }
+                Some("")
+            })
+            .collect())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use crate::toml;
+
+    #[test]
+    fn test_read_toml() {
+        let mut toml = toml::ParseToml::new();
+        &toml.read_dir("test_files/rules".to_string());
+
+        for rule in toml.rules {
+            match rule {
+                Ok(_rule) => {
+                    if let Some(severity) = _rule.rule.severity {
+                        assert_eq!("high", severity);
+                    }
+                }
+                Err(_) => (),
+            }
+        }
+    }
+}

test_files/rules/test.toml

@@ -0,0 +1,2 @@
+[rule]
+severity = "high"

test_files/rules/test2.toml

@@ -0,0 +1,3 @@
+[rule]
+severity = "high"
+name = "test2"