Merge branch 'dev' into feature/espillarz

salt/common/tools/sbin/so-import-pcap

@@ -206,13 +206,16 @@ if [ "$INVALID_PCAPS" = "yes" ]; then
   echo "Please note!  One or more pcaps was invalid!  You can scroll up to see which ones were invalid."
 fi
 
+START_OLDEST_SLASH=$(echo $START_OLDEST | sed -e 's/-/%2F/g')
+END_NEWEST_SLASH=$(echo $END_NEWEST | sed -e 's/-/%2F/g')
+
 if [ "$VALID_PCAPS" = "yes" ]; then
 cat << EOF
 
 Import complete!
 
 You can use the following hyperlink to view data in the time range of your import.  You can triple-click to quickly highlight the entire hyperlink and you can then copy it into your browser:
-https://{{ MASTERIP }}/kibana/app/kibana#/dashboard/a8411b30-6d03-11ea-b301-3d6c35840645?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:'${START_OLDEST}T00:00:00.000Z',mode:absolute,to:'${END_NEWEST}T00:00:00.000Z'))
+https://{{ MASTERIP }}/#/hunt?q=%2a%20%7C%20groupby%20event.module%20event.dataset&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20PM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20PM
 
 or you can manually set your Time Range to be:
 From: $START_OLDEST    To: $END_NEWEST

salt/elastalert/files/elastalert_config.yaml

@@ -86,3 +86,25 @@ alert_time_limit:
 index_settings:
   shards: 1
   replicas: 0
+
+logging:
+  version: 1
+  incremental: false
+  disable_existing_loggers: false
+  formatters:
+    logline:
+      format: '%(asctime)s %(levelname)+8s %(name)+20s %(message)s'
+
+  handlers:
+    file:
+      class : logging.FileHandler
+      formatter: logline
+      level: INFO
+      filename: /var/log/elastalert/elastalert.log
+
+  loggers:
+    '':
+      level: INFO
+      handlers:
+        - file
+      propagate: false

salt/elastalert/files/modules/so/playbook-es.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-from datetime import date   
+from time import gmtime, strftime
 import requests,json
 from elastalert.alerts import Alerter
 
@@ -13,10 +13,11 @@ class PlaybookESAlerter(Alerter):
 
     def alert(self, matches):
        for match in matches:
+            today = strftime("%Y.%m.%d", gmtime())
+            timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S", gmtime())
             headers = {"Content-Type": "application/json"}
-            payload = {"play_title": self.rule['play_title'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"data": match}
+            payload = {"rule.name": self.rule['play_title'],"event.severity": self.rule['event.severity'],"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"event.module": self.rule['event.module'],"event.dataset": self.rule['event.dataset'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"rule.category": self.rule['rule.category'],"data": match, "@timestamp": timestamp}
-            today = str(date.today())
+            url = f"http://{self.rule['elasticsearch_host']}/so-playbook-alerts-{today}/_doc/"
-            url = f"http://{self.rule['elasticsearch_host']}/playbook-alerts-{today}/_doc/"
             requests.post(url, data=json.dumps(payload), headers=headers, verify=False)
                             
     def get_info(self):

salt/elasticsearch/files/elasticsearch.yml

@@ -24,3 +24,4 @@ cluster.routing.allocation.disk.watermark.high: 98%
 cluster.routing.allocation.disk.watermark.flood_stage: 98%
 node.attr.box_type: {{ NODE_ROUTE_TYPE }}
 node.name: {{ ESCLUSTERNAME }}
+script.max_compilations_rate: 1000/1m

salt/elasticsearch/files/ingest/beats.common

@@ -2,10 +2,17 @@
   "description" : "beats.common",
   "processors" : [
     {"community_id":   {"if": "ctx.winlog.event_data?.Protocol != null", "field":["winlog.event_data.SourceIp","winlog.event_data.SourcePort","winlog.event_data.DestinationIp","winlog.event_data.DestinationPort","winlog.event_data.Protocol"],"target_field":"network.community_id"}},    
-    { "set":           { "if": "ctx.winlog?.channel != null",   "field": "dataset", "value": "wel-{{winlog.channel}}", "override": true }  },
+    { "set":           { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'",   "field": "event.module", "value": "sysmon", "override": true }  },
+    { "set":           { "if": "ctx.winlog?.channel!= null",   "field": "event.module", "value": "win_eventlog", "override": true, "ignore_failure": true }  },
+    { "set":           { "if": "ctx.winlog?.channel != null",   "field": "dataset", "value": "{{winlog.channel}}", "override": true }  },
     { "set":           { "if": "ctx.agent?.type != null",   "field": "module", "value": "{{agent.type}}", "override": true }  },  
     { "set":           { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 3",   "field": "event.category", "value": "host,process,network", "override": true }  },
     { "set":           { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 1",   "field": "event.category", "value": "host,process", "override": true }  },
+    { "set":          { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 1",   "field": "event.dataset", "value": "process_creation", "override": true }  },
+    { "set":          { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 2",   "field": "event.dataset", "value": "process_changed_file", "override": true }  },
+    { "set":          { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 3",   "field": "event.dataset", "value": "network_connection", "override": true }  },
+    { "set":          { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 5",   "field": "event.dataset", "value": "driver_loaded", "override": true }  },
+    { "set":          { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 6",   "field": "event.dataset", "value": "image_loaded", "override": true }  },
     { "rename": 	   { "field": "agent.hostname", 			            "target_field": "agent.name",		"ignore_missing": true 	} },
     { "rename": 	   { "field": "winlog.event_data.SubjectUserName", 			  "target_field": "user.name",		"ignore_missing": true 	} },
     { "rename": 	   { "field": "winlog.event_data.DestinationHostname", 	  "target_field": "destination.hostname",	"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/common

@@ -37,13 +37,18 @@
         	"index_name_format": "yyyy.MM.dd"
         }
     },  
-    { "rename":         { "field": "module",              "target_field": "event.module",                  "ignore_missing": true  } },
+    { "set":         { "if": "ctx.event?.severity == 3",   "field": "event.severity_label", "value": "low", "override": true }  },
-    { "rename":         { "field": "dataset",              "target_field": "event.dataset",                  "ignore_missing": true  } },
+    { "set":         { "if": "ctx.event?.severity == 5",   "field": "event.severity_label", "value": "medium", "override": true }  },
+    { "set":         { "if": "ctx.event?.severity == 7",   "field": "event.severity_label", "value": "high", "override": true }  },
+    { "set":         { "if": "ctx.event?.severity == 10",   "field": "event.severity_label", "value": "critical", "override": true }  },
+    { "rename":         { "field": "module",              "target_field": "event.module",                  "ignore_failure": true, "ignore_missing": true  } },
+    { "rename":         { "field": "dataset",              "target_field": "event.dataset",                  "ignore_failure": true, "ignore_missing": true  } },
     { "rename":         { "field": "category",              "target_field": "event.category",                  "ignore_missing": true  } },
     { "rename":         { "field": "message2.community_id", "target_field": "network.community_id",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "destination.port", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "source.port", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "log.id.uid", "type": "string",  "ignore_failure": true,  "ignore_missing": true  } },
+    { "convert":         { "field": "event.severity", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
     { 
         "remove": {
                 "field": [ "index_name_prefix", "message2", "type" ],

salt/elasticsearch/files/ingest/ossec.alert

@@ -49,6 +49,9 @@
     { "set":            { "if": "ctx.rule.level == 13",                 "field": "rule.category", "value": "Unusal error (high importance)"     } },
     { "set":            { "if": "ctx.rule.level == 14",                 "field": "rule.category", "value": "High importance security event"     } },
     { "set":            { "if": "ctx.rule.level == 15",                 "field": "rule.category", "value": "Severe attack"                      } },
+    { "set":            { "if": "ctx.rule.level <= 7",                  "field": "event.severity", "value": 1, "override": true }  },
+    { "set":            { "if": "ctx.rule.level >= 8 && ctx.rule.level <= 11",     "field": "event.severity", "value": 2, "override": true }  },
+    { "set":            { "if": "ctx.rule.level >= 12",          "field": "event.severity", "value": 3, "override": true }  },
     { "append":         { "if": "ctx.rule.level != null",               "field": "tags", "value": ["alert"]                                     } },
     { "remove": { "field": [ "predecoder", "decoder" ], "ignore_missing": true, "ignore_failure": false } },
     { "pipeline":       { "name": "common" } }

salt/elasticsearch/files/ingest/suricata.alert

@@ -7,6 +7,9 @@
     { "rename":{ "field": "rule.signature_id",     "target_field": "rule.uuid", "ignore_failure": true  } },
     { "rename":{ "field": "rule.signature_id",     "target_field": "rule.signature", "ignore_failure": true  } },
     { "rename":{ "field": "message2.payload_printable",     "target_field": "network.data.decoded", "ignore_failure": true  } },
+    { "set":         { "if": "ctx.rule.severity == 3",   "field": "event.severity", "value": 1, "override": true }  },
+    { "set":         { "if": "ctx.rule.severity == 2",   "field": "event.severity", "value": 2, "override": true }  },
+    { "set":         { "if": "ctx.rule.severity == 1",   "field": "event.severity", "value": 3, "override": true }  },
     { "pipeline": { "name": "common" } }
   ]
 }

salt/pcap/files/sensoroni.json

@@ -8,6 +8,7 @@
     "serverUrl": "https://{{ MASTER }}/sensoroniagents",
     "verifyCert": false,
     "modules": {
+      "importer": {},
       "statickeyauth": {
         "apiKey": "{{ SENSORONIKEY }}"
       },

salt/soc/files/soc/soc.json

@@ -83,7 +83,7 @@
         },
         "queries": [
           { "name": "Default Query", "description": "Show all events grouped by the origin host", "query": "* | groupby observer.name"},
-          { "name": "Log Type", "description": "Show all events grouped by type", "query": "* | groupby event.module"},
+          { "name": "Log Type", "description": "Show all events grouped by module and dataset", "query": "* | groupby event.module event.dataset"},
           { "name": "Elastalerts", "description": "", "query": "_type:elastalert | groupby rule.name"},
           { "name": "Alerts", "description": "Show all alerts grouped by alert source", "query": "event.dataset: alert | groupby event.module"},
           { "name": "NIDS Alerts", "description": "Show all NIDS alerts grouped by alert name", "query": "event.category: network AND event.dataset: alert | groupby rule.name"},
@@ -107,6 +107,7 @@
           { "name": "DNS", "description": "DNS queries grouped by response code", "query": "event.dataset:dns | groupby dns.response.code_name destination.port"},
           { "name": "DNS", "description": "DNS highest registered domain", "query": "event.dataset:dns | groupby dns.highest_registered_domain.keyword destination.port"},
           { "name": "DNS", "description": "DNS grouped by parent domain", "query": "event.dataset:dns | groupby dns.parent_domain.keyword destination.port"},
+          { "name": "DPD", "description": "Dynamic Protocol Detection errors", "query": "event.dataset:dpd | groupby error.reason"},
           { "name": "Files", "description": "Files grouped by mimetype", "query": "event.dataset:file | groupby file.mime_type source.ip"},
           { "name": "Files", "description": "Files grouped by source", "query": "event.dataset:file | groupby file.source source.ip"},
           { "name": "FTP", "description": "FTP grouped by argument", "query": "event.dataset:ftp | groupby ftp.argument"},

salt/soctopus/files/SOCtopus.conf

@@ -61,6 +61,7 @@ slack_webhook = YOURSLACKWEBHOOK
 
 [playbook]
 playbook_url = http://{{MASTER}}:3200/playbook
+playbook_ext_url = https://{{MASTER}}/playbook
 playbook_key = de6639318502476f2fa5aa06f43f51fb389a3d7f
 playbook_verifycert = no
 playbook_unit_test_index = playbook-testing

salt/soctopus/files/templates/es-generic.template

@@ -1,7 +0,0 @@
-{% set ES = salt['pillar.get']('static:masterip', '') %}
-
-alert: modules.so.playbook-es.PlaybookESAlerter
-elasticsearch_host: "{{ ES }}:9200"
-play_title: ""
-play_url: "https://{{ ES }}/playbook/issues/6000"
-sigma_level: ""

salt/soctopus/files/templates/generic.template

@@ -1,6 +1,6 @@
-{% set es = salt['pillar.get']('static:masterip', '') %}
+{%- set es = salt['pillar.get']('static:masterip', '') %}
-{% set hivehost = salt['pillar.get']('static:masterip', '') %}
+{%- set hivehost = salt['pillar.get']('static:masterip', '') %}
-{% set hivekey = salt['pillar.get']('static:hivekey', '') %}
+{%- set hivekey = salt['pillar.get']('static:hivekey', '') %}
 alert: hivealerter
 
 hive_connection:
@@ -23,3 +23,15 @@ hive_alert_config:
   status: 'New'
   follow: True
   caseTemplate: '5000'
+
+alert: modules.so.playbook-es.PlaybookESAlerter
+elasticsearch_host: "{{ es }}:9200"
+play_title: ""
+event.module: "playbook"
+event.dataset: "alert"
+event.severity:
+rule.category: 
+play_url: "https://{{ es }}/playbook/issues/6000"
+kibana_pivot: "https://{{es}}/kibana/app/kibana#/discover?_g=()&_a=(columns:!(_source),interval:auto,query:(language:lucene,query:'_id:{[_id]}'),sort:!('@timestamp',desc))"
+soc_pivot: "https://{{es}}/#/hunt"
+sigma_level: ""

salt/soctopus/files/templates/osquery.template

@@ -31,3 +31,14 @@ hive_alert_config:
   caseTemplate: '5000'
   
  
+alert: modules.so.playbook-es.PlaybookESAlerter
+elasticsearch_host: "{{ es }}:9200"
+play_title: ""
+event.module: "playbook"
+event.dataset: "alert"
+event.severity:
+rule.category: 
+play_url: "https://{{ es }}/playbook/issues/6000"
+kibana_pivot: "https://{{es}}/kibana/app/kibana#/discover?_g=()&_a=(columns:!(_source),interval:auto,query:(language:lucene,query:'_id:{[_id]}'),sort:!('@timestamp',desc))"
+soc_pivot: "https://{{es}}/#/hunt"
+sigma_level: ""

setup/so-functions

@@ -838,18 +838,6 @@ firewall_generate_templates() {
 
 }
 
-fleet_pillar() {
-
-	local pillar_file="$temp_install_dir"/pillar/minions/"$MINION_ID".sls
-
-	# Create the fleet pillar
-	printf '%s\n'\
-		"fleet:"\
-		"  mainip: $MAINIP"\
-		"  master: $MSRV"\
-		"" > "$pillar_file"
-}
-
 generate_passwords(){
   # Generate Random Passwords for Things
   MYSQLPASS=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 20 | head -n 1)

setup/so-setup

@@ -367,14 +367,14 @@ fi
 
 set_main_ip >> $setup_log 2>&1
 
+host_pillar >> $setup_log 2>&1
+
 if [[ $is_minion ]]; then
 	set_updates >> $setup_log 2>&1
 	copy_ssh_key >> $setup_log 2>&1
 fi
 
-if [[ "$OSQUERY" = 1 ]]; then
+
-	host_pillar >> $setup_log 2>&1
-fi
 
 # Begin install
 {