Merge pull request #967 from Security-Onion-Solutions/feature/low-level-alerts

Feature - low level alerts

salt/elastalert/files/modules/so/playbook-es.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-from datetime import date   
+from time import gmtime, strftime
 import requests,json
 from elastalert.alerts import Alerter
 
@@ -13,10 +13,11 @@ class PlaybookESAlerter(Alerter):
 
     def alert(self, matches):
        for match in matches:
+            today = strftime("%Y.%m.%d", gmtime())
+            timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S", gmtime())
             headers = {"Content-Type": "application/json"}
-            payload = {"play_title": self.rule['play_title'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"data": match}
-            today = str(date.today())
-            url = f"http://{self.rule['elasticsearch_host']}/playbook-alerts-{today}/_doc/"
+            payload = {"rule.name": self.rule['play_title'],"event.severity": self.rule['event.severity'],"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"event.module": self.rule['event.module'],"event.dataset": self.rule['event.dataset'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"rule.category": self.rule['rule.category'],"data": match, "@timestamp": timestamp}
+            url = f"http://{self.rule['elasticsearch_host']}/so-playbook-alerts-{today}/_doc/"
             requests.post(url, data=json.dumps(payload), headers=headers, verify=False)
                             
     def get_info(self):

salt/elasticsearch/files/elasticsearch.yml

@@ -26,3 +26,4 @@ cluster.routing.allocation.disk.threshold_enabled: true
 cluster.routing.allocation.disk.watermark.low: 95%
 cluster.routing.allocation.disk.watermark.high: 98%
 cluster.routing.allocation.disk.watermark.flood_stage: 98%
+script.max_compilations_rate: 1000/1m

salt/elasticsearch/files/ingest/beats.common

@@ -2,10 +2,17 @@
   "description" : "beats.common",
   "processors" : [
     {"community_id":   {"if": "ctx.winlog.event_data?.Protocol != null", "field":["winlog.event_data.SourceIp","winlog.event_data.SourcePort","winlog.event_data.DestinationIp","winlog.event_data.DestinationPort","winlog.event_data.Protocol"],"target_field":"network.community_id"}},    
-    { "set":           { "if": "ctx.winlog?.channel != null",   "field": "dataset", "value": "wel-{{winlog.channel}}", "override": true }  },
+    { "set":           { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'",   "field": "event.module", "value": "sysmon", "override": true }  },
+    { "set":           { "if": "ctx.winlog?.channel!= null",   "field": "event.module", "value": "win_eventlog", "override": true, "ignore_failure": true }  },
+    { "set":           { "if": "ctx.winlog?.channel != null",   "field": "dataset", "value": "{{winlog.channel}}", "override": true }  },
     { "set":           { "if": "ctx.agent?.type != null",   "field": "module", "value": "{{agent.type}}", "override": true }  },  
     { "set":           { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 3",   "field": "event.category", "value": "host,process,network", "override": true }  },
     { "set":           { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 1",   "field": "event.category", "value": "host,process", "override": true }  },
+    { "set":          { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 1",   "field": "event.dataset", "value": "process_creation", "override": true }  },
+    { "set":          { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 2",   "field": "event.dataset", "value": "process_changed_file", "override": true }  },
+    { "set":          { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 3",   "field": "event.dataset", "value": "network_connection", "override": true }  },
+    { "set":          { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 5",   "field": "event.dataset", "value": "driver_loaded", "override": true }  },
+    { "set":          { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 6",   "field": "event.dataset", "value": "image_loaded", "override": true }  },
     { "rename": 	   { "field": "agent.hostname", 			            "target_field": "agent.name",		"ignore_missing": true 	} },
     { "rename": 	   { "field": "winlog.event_data.SubjectUserName", 			  "target_field": "user.name",		"ignore_missing": true 	} },
     { "rename": 	   { "field": "winlog.event_data.DestinationHostname", 	  "target_field": "destination.hostname",	"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/common

@@ -37,13 +37,18 @@
         	"index_name_format": "yyyy.MM.dd"
         }
     },  
-    { "rename":         { "field": "module",              "target_field": "event.module",                  "ignore_missing": true  } },
-    { "rename":         { "field": "dataset",              "target_field": "event.dataset",                  "ignore_missing": true  } },
+    { "set":         { "if": "ctx.event?.severity == 3",   "field": "event.severity_label", "value": "low", "override": true }  },
+    { "set":         { "if": "ctx.event?.severity == 5",   "field": "event.severity_label", "value": "medium", "override": true }  },
+    { "set":         { "if": "ctx.event?.severity == 7",   "field": "event.severity_label", "value": "high", "override": true }  },
+    { "set":         { "if": "ctx.event?.severity == 10",   "field": "event.severity_label", "value": "critical", "override": true }  },
+    { "rename":         { "field": "module",              "target_field": "event.module",                  "ignore_failure": true, "ignore_missing": true  } },
+    { "rename":         { "field": "dataset",              "target_field": "event.dataset",                  "ignore_failure": true, "ignore_missing": true  } },
     { "rename":         { "field": "category",              "target_field": "event.category",                  "ignore_missing": true  } },
     { "rename":         { "field": "message2.community_id", "target_field": "network.community_id",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "destination.port", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "source.port", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "log.id.uid", "type": "string",  "ignore_failure": true,  "ignore_missing": true  } },
+    { "convert":         { "field": "event.severity", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
     { 
         "remove": {
                 "field": [ "index_name_prefix", "message2", "type" ],

salt/elasticsearch/files/ingest/ossec.alert

@@ -49,6 +49,9 @@
     { "set":            { "if": "ctx.rule.level == 13",                 "field": "rule.category", "value": "Unusal error (high importance)"     } },
     { "set":            { "if": "ctx.rule.level == 14",                 "field": "rule.category", "value": "High importance security event"     } },
     { "set":            { "if": "ctx.rule.level == 15",                 "field": "rule.category", "value": "Severe attack"                      } },
+    { "set":            { "if": "ctx.rule.level <= 7",                  "field": "event.severity", "value": 1, "override": true }  },
+    { "set":            { "if": "ctx.rule.level >= 8 && ctx.rule.level <= 11",     "field": "event.severity", "value": 2, "override": true }  },
+    { "set":            { "if": "ctx.rule.level >= 12",          "field": "event.severity", "value": 3, "override": true }  },
     { "append":         { "if": "ctx.rule.level != null",               "field": "tags", "value": ["alert"]                                     } },
     { "remove": { "field": [ "predecoder", "decoder" ], "ignore_missing": true, "ignore_failure": false } },
     { "pipeline":       { "name": "common" } }

salt/elasticsearch/files/ingest/suricata.alert

@@ -7,6 +7,9 @@
     { "rename":{ "field": "rule.signature_id",     "target_field": "rule.uuid", "ignore_failure": true  } },
     { "rename":{ "field": "rule.signature_id",     "target_field": "rule.signature", "ignore_failure": true  } },
     { "rename":{ "field": "message2.payload_printable",     "target_field": "network.data.decoded", "ignore_failure": true  } },
+    { "set":         { "if": "ctx.rule.severity == 3",   "field": "event.severity", "value": 1, "override": true }  },
+    { "set":         { "if": "ctx.rule.severity == 2",   "field": "event.severity", "value": 2, "override": true }  },
+    { "set":         { "if": "ctx.rule.severity == 1",   "field": "event.severity", "value": 3, "override": true }  },
     { "pipeline": { "name": "common" } }
   ]
 }

salt/soctopus/files/SOCtopus.conf

@@ -61,6 +61,7 @@ slack_webhook = YOURSLACKWEBHOOK
 
 [playbook]
 playbook_url = http://{{MASTER}}:3200/playbook
+playbook_ext_url = https://{{MASTER}}/playbook
 playbook_key = de6639318502476f2fa5aa06f43f51fb389a3d7f
 playbook_verifycert = no
 playbook_unit_test_index = playbook-testing

salt/soctopus/files/templates/es-generic.template

@@ -1,7 +0,0 @@
-{% set ES = salt['pillar.get']('static:masterip', '') %}
-
-alert: modules.so.playbook-es.PlaybookESAlerter
-elasticsearch_host: "{{ ES }}:9200"
-play_title: ""
-play_url: "https://{{ ES }}/playbook/issues/6000"
-sigma_level: ""

salt/soctopus/files/templates/generic.template

@@ -1,6 +1,6 @@
-{% set es = salt['pillar.get']('static:masterip', '') %}
-{% set hivehost = salt['pillar.get']('static:masterip', '') %}
-{% set hivekey = salt['pillar.get']('static:hivekey', '') %}
+{%- set es = salt['pillar.get']('static:masterip', '') %}
+{%- set hivehost = salt['pillar.get']('static:masterip', '') %}
+{%- set hivekey = salt['pillar.get']('static:hivekey', '') %}
 alert: hivealerter
 
 hive_connection:
@@ -23,3 +23,15 @@ hive_alert_config:
   status: 'New'
   follow: True
   caseTemplate: '5000'
+
+alert: modules.so.playbook-es.PlaybookESAlerter
+elasticsearch_host: "{{ es }}:9200"
+play_title: ""
+event.module: "playbook"
+event.dataset: "alert"
+event.severity:
+rule.category: 
+play_url: "https://{{ es }}/playbook/issues/6000"
+kibana_pivot: "https://{{es}}/kibana/app/kibana#/discover?_g=()&_a=(columns:!(_source),interval:auto,query:(language:lucene,query:'_id:{[_id]}'),sort:!('@timestamp',desc))"
+soc_pivot: "https://{{es}}/#/hunt"
+sigma_level: ""

salt/soctopus/files/templates/osquery.template

@@ -31,3 +31,14 @@ hive_alert_config:
   caseTemplate: '5000'
   
  
+alert: modules.so.playbook-es.PlaybookESAlerter
+elasticsearch_host: "{{ es }}:9200"
+play_title: ""
+event.module: "playbook"
+event.dataset: "alert"
+event.severity:
+rule.category: 
+play_url: "https://{{ es }}/playbook/issues/6000"
+kibana_pivot: "https://{{es}}/kibana/app/kibana#/discover?_g=()&_a=(columns:!(_source),interval:auto,query:(language:lucene,query:'_id:{[_id]}'),sort:!('@timestamp',desc))"
+soc_pivot: "https://{{es}}/#/hunt"
+sigma_level: ""

setup/so-functions

@@ -838,18 +838,6 @@ firewall_generate_templates() {
 
 }
 
-fleet_pillar() {
-
-	local pillar_file="$temp_install_dir"/pillar/minions/"$MINION_ID".sls
-
-	# Create the fleet pillar
-	printf '%s\n'\
-		"fleet:"\
-		"  mainip: $MAINIP"\
-		"  master: $MSRV"\
-		"" > "$pillar_file"
-}
-
 generate_passwords(){
   # Generate Random Passwords for Things
   MYSQLPASS=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 20 | head -n 1)