From 52f7111e1d3919150106ff2a2a87e4b47bba4221 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Thu, 9 Jul 2020 13:53:55 -0400 Subject: [PATCH] Feature - low level alerts --- .../elastalert/files/modules/so/playbook-es.py | 11 ++++++----- salt/elasticsearch/files/elasticsearch.yml | 1 + salt/elasticsearch/files/ingest/beats.common | 11 +++++++++-- salt/elasticsearch/files/ingest/common | 11 ++++++++--- salt/elasticsearch/files/ingest/ossec.alert | 3 +++ salt/elasticsearch/files/ingest/suricata.alert | 3 +++ salt/soctopus/files/SOCtopus.conf | 1 + .../files/templates/es-generic.template | 7 ------- salt/soctopus/files/templates/generic.template | 18 +++++++++++++++--- salt/soctopus/files/templates/osquery.template | 11 +++++++++++ setup/so-functions | 12 ------------ 11 files changed, 57 insertions(+), 32 deletions(-) delete mode 100644 salt/soctopus/files/templates/es-generic.template diff --git a/salt/elastalert/files/modules/so/playbook-es.py b/salt/elastalert/files/modules/so/playbook-es.py index 0ba3f3601..c794bdf12 100644 --- a/salt/elastalert/files/modules/so/playbook-es.py +++ b/salt/elastalert/files/modules/so/playbook-es.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from datetime import date +from time import gmtime, strftime import requests,json from elastalert.alerts import Alerter @@ -13,11 +13,12 @@ class PlaybookESAlerter(Alerter): def alert(self, matches): for match in matches: + today = strftime("%Y.%m.%d", gmtime()) + timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S", gmtime()) headers = {"Content-Type": "application/json"} - payload = {"play_title": self.rule['play_title'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"data": match} - today = str(date.today()) - url = f"http://{self.rule['elasticsearch_host']}/playbook-alerts-{today}/_doc/" + payload = {"rule.name": self.rule['play_title'],"event.severity": self.rule['event.severity'],"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"event.module": self.rule['event.module'],"event.dataset": self.rule['event.dataset'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"rule.category": self.rule['rule.category'],"data": match, "@timestamp": timestamp} + url = f"http://{self.rule['elasticsearch_host']}/so-playbook-alerts-{today}/_doc/" requests.post(url, data=json.dumps(payload), headers=headers, verify=False) def get_info(self): - return {'type': 'PlaybookESAlerter'} + return {'type': 'PlaybookESAlerter'} \ No newline at end of file diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index 271ef40cf..afb06057a 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -26,3 +26,4 @@ cluster.routing.allocation.disk.threshold_enabled: true cluster.routing.allocation.disk.watermark.low: 95% cluster.routing.allocation.disk.watermark.high: 98% cluster.routing.allocation.disk.watermark.flood_stage: 98% +script.max_compilations_rate: 1000/1m \ No newline at end of file diff --git a/salt/elasticsearch/files/ingest/beats.common b/salt/elasticsearch/files/ingest/beats.common index 0e93abb03..9ea586156 100644 --- a/salt/elasticsearch/files/ingest/beats.common +++ b/salt/elasticsearch/files/ingest/beats.common @@ -2,10 +2,17 @@ "description" : "beats.common", "processors" : [ {"community_id": {"if": "ctx.winlog.event_data?.Protocol != null", "field":["winlog.event_data.SourceIp","winlog.event_data.SourcePort","winlog.event_data.DestinationIp","winlog.event_data.DestinationPort","winlog.event_data.Protocol"],"target_field":"network.community_id"}}, - { "set": { "if": "ctx.winlog?.channel != null", "field": "dataset", "value": "wel-{{winlog.channel}}", "override": true } }, - { "set": { "if": "ctx.agent?.type != null", "field": "module", "value": "{{agent.type}}", "override": true } }, + { "set": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "field": "event.module", "value": "sysmon", "override": true } }, + { "set": { "if": "ctx.winlog?.channel!= null", "field": "event.module", "value": "win_eventlog", "override": true, "ignore_failure": true } }, + { "set": { "if": "ctx.winlog?.channel != null", "field": "dataset", "value": "{{winlog.channel}}", "override": true } }, + { "set": { "if": "ctx.agent?.type != null", "field": "module", "value": "{{agent.type}}", "override": true } }, { "set": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 3", "field": "event.category", "value": "host,process,network", "override": true } }, { "set": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 1", "field": "event.category", "value": "host,process", "override": true } }, + { "set": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 1", "field": "event.dataset", "value": "process_creation", "override": true } }, + { "set": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 2", "field": "event.dataset", "value": "process_changed_file", "override": true } }, + { "set": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 3", "field": "event.dataset", "value": "network_connection", "override": true } }, + { "set": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 5", "field": "event.dataset", "value": "driver_loaded", "override": true } }, + { "set": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 6", "field": "event.dataset", "value": "image_loaded", "override": true } }, { "rename": { "field": "agent.hostname", "target_field": "agent.name", "ignore_missing": true } }, { "rename": { "field": "winlog.event_data.SubjectUserName", "target_field": "user.name", "ignore_missing": true } }, { "rename": { "field": "winlog.event_data.DestinationHostname", "target_field": "destination.hostname", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/common b/salt/elasticsearch/files/ingest/common index d8de06f31..0878c0463 100644 --- a/salt/elasticsearch/files/ingest/common +++ b/salt/elasticsearch/files/ingest/common @@ -34,14 +34,19 @@ "ignore_failure": true, "index_name_format": "yyyy.MM.dd" } - }, - { "rename": { "field": "module", "target_field": "event.module", "ignore_missing": true } }, - { "rename": { "field": "dataset", "target_field": "event.dataset", "ignore_missing": true } }, + }, + { "set": { "if": "ctx.event?.severity == 3", "field": "event.severity_label", "value": "low", "override": true } }, + { "set": { "if": "ctx.event?.severity == 5", "field": "event.severity_label", "value": "medium", "override": true } }, + { "set": { "if": "ctx.event?.severity == 7", "field": "event.severity_label", "value": "high", "override": true } }, + { "set": { "if": "ctx.event?.severity == 10", "field": "event.severity_label", "value": "critical", "override": true } }, + { "rename": { "field": "module", "target_field": "event.module", "ignore_failure": true, "ignore_missing": true } }, + { "rename": { "field": "dataset", "target_field": "event.dataset", "ignore_failure": true, "ignore_missing": true } }, { "rename": { "field": "category", "target_field": "event.category", "ignore_missing": true } }, { "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "destination.port", "type": "integer", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "source.port", "type": "integer", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "log.id.uid", "type": "string", "ignore_failure": true, "ignore_missing": true } }, + { "convert": { "field": "event.severity", "type": "integer", "ignore_failure": true, "ignore_missing": true } }, { "remove": { "field": [ "index_name_prefix", "message2", "type" ], diff --git a/salt/elasticsearch/files/ingest/ossec.alert b/salt/elasticsearch/files/ingest/ossec.alert index 2aab7f630..7654bf255 100644 --- a/salt/elasticsearch/files/ingest/ossec.alert +++ b/salt/elasticsearch/files/ingest/ossec.alert @@ -49,6 +49,9 @@ { "set": { "if": "ctx.rule.level == 13", "field": "rule.category", "value": "Unusal error (high importance)" } }, { "set": { "if": "ctx.rule.level == 14", "field": "rule.category", "value": "High importance security event" } }, { "set": { "if": "ctx.rule.level == 15", "field": "rule.category", "value": "Severe attack" } }, + { "set": { "if": "ctx.rule.level <= 7", "field": "event.severity", "value": 1, "override": true } }, + { "set": { "if": "ctx.rule.level >= 8 && ctx.rule.level <= 11", "field": "event.severity", "value": 2, "override": true } }, + { "set": { "if": "ctx.rule.level >= 12", "field": "event.severity", "value": 3, "override": true } }, { "append": { "if": "ctx.rule.level != null", "field": "tags", "value": ["alert"] } }, { "remove": { "field": [ "predecoder", "decoder" ], "ignore_missing": true, "ignore_failure": false } }, { "pipeline": { "name": "common" } } diff --git a/salt/elasticsearch/files/ingest/suricata.alert b/salt/elasticsearch/files/ingest/suricata.alert index 82486adc6..5a0cfc4df 100644 --- a/salt/elasticsearch/files/ingest/suricata.alert +++ b/salt/elasticsearch/files/ingest/suricata.alert @@ -7,6 +7,9 @@ { "rename":{ "field": "rule.signature_id", "target_field": "rule.uuid", "ignore_failure": true } }, { "rename":{ "field": "rule.signature_id", "target_field": "rule.signature", "ignore_failure": true } }, { "rename":{ "field": "message2.payload_printable", "target_field": "network.data.decoded", "ignore_failure": true } }, + { "set": { "if": "ctx.rule.severity == 3", "field": "event.severity", "value": 1, "override": true } }, + { "set": { "if": "ctx.rule.severity == 2", "field": "event.severity", "value": 2, "override": true } }, + { "set": { "if": "ctx.rule.severity == 1", "field": "event.severity", "value": 3, "override": true } }, { "pipeline": { "name": "common" } } ] } diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf index d2c3eea2d..e32aaf8da 100644 --- a/salt/soctopus/files/SOCtopus.conf +++ b/salt/soctopus/files/SOCtopus.conf @@ -61,6 +61,7 @@ slack_webhook = YOURSLACKWEBHOOK [playbook] playbook_url = http://{{MASTER}}:3200/playbook +playbook_ext_url = https://{{MASTER}}/playbook playbook_key = de6639318502476f2fa5aa06f43f51fb389a3d7f playbook_verifycert = no playbook_unit_test_index = playbook-testing diff --git a/salt/soctopus/files/templates/es-generic.template b/salt/soctopus/files/templates/es-generic.template deleted file mode 100644 index cdda8a19b..000000000 --- a/salt/soctopus/files/templates/es-generic.template +++ /dev/null @@ -1,7 +0,0 @@ -{% set ES = salt['pillar.get']('static:masterip', '') %} - -alert: modules.so.playbook-es.PlaybookESAlerter -elasticsearch_host: "{{ ES }}:9200" -play_title: "" -play_url: "https://{{ ES }}/playbook/issues/6000" -sigma_level: "" diff --git a/salt/soctopus/files/templates/generic.template b/salt/soctopus/files/templates/generic.template index 68dc040fc..0bdb6a2ba 100644 --- a/salt/soctopus/files/templates/generic.template +++ b/salt/soctopus/files/templates/generic.template @@ -1,6 +1,6 @@ -{% set es = salt['pillar.get']('static:masterip', '') %} -{% set hivehost = salt['pillar.get']('static:masterip', '') %} -{% set hivekey = salt['pillar.get']('static:hivekey', '') %} +{%- set es = salt['pillar.get']('static:masterip', '') %} +{%- set hivehost = salt['pillar.get']('static:masterip', '') %} +{%- set hivekey = salt['pillar.get']('static:hivekey', '') %} alert: hivealerter hive_connection: @@ -23,3 +23,15 @@ hive_alert_config: status: 'New' follow: True caseTemplate: '5000' + +alert: modules.so.playbook-es.PlaybookESAlerter +elasticsearch_host: "{{ es }}:9200" +play_title: "" +event.module: "playbook" +event.dataset: "alert" +event.severity: +rule.category: +play_url: "https://{{ es }}/playbook/issues/6000" +kibana_pivot: "https://{{es}}/kibana/app/kibana#/discover?_g=()&_a=(columns:!(_source),interval:auto,query:(language:lucene,query:'_id:{[_id]}'),sort:!('@timestamp',desc))" +soc_pivot: "https://{{es}}/#/hunt" +sigma_level: "" \ No newline at end of file diff --git a/salt/soctopus/files/templates/osquery.template b/salt/soctopus/files/templates/osquery.template index 28ea29ee9..90bc0743e 100644 --- a/salt/soctopus/files/templates/osquery.template +++ b/salt/soctopus/files/templates/osquery.template @@ -31,3 +31,14 @@ hive_alert_config: caseTemplate: '5000' +alert: modules.so.playbook-es.PlaybookESAlerter +elasticsearch_host: "{{ es }}:9200" +play_title: "" +event.module: "playbook" +event.dataset: "alert" +event.severity: +rule.category: +play_url: "https://{{ es }}/playbook/issues/6000" +kibana_pivot: "https://{{es}}/kibana/app/kibana#/discover?_g=()&_a=(columns:!(_source),interval:auto,query:(language:lucene,query:'_id:{[_id]}'),sort:!('@timestamp',desc))" +soc_pivot: "https://{{es}}/#/hunt" +sigma_level: "" \ No newline at end of file diff --git a/setup/so-functions b/setup/so-functions index fdfcf37eb..4d8951fb9 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -783,18 +783,6 @@ firewall_generate_templates() { } -fleet_pillar() { - - local pillar_file="$temp_install_dir"/pillar/minions/"$MINION_ID".sls - - # Create the fleet pillar - printf '%s\n'\ - "fleet:"\ - " mainip: $MAINIP"\ - " master: $MSRV"\ - "" > "$pillar_file" -} - generate_passwords(){ # Generate Random Passwords for Things MYSQLPASS=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 20 | head -n 1)