diff --git a/salt/common/tools/sbin/so-import-pcap b/salt/common/tools/sbin/so-import-pcap index 9e54820e0..0c99ede64 100755 --- a/salt/common/tools/sbin/so-import-pcap +++ b/salt/common/tools/sbin/so-import-pcap @@ -206,13 +206,16 @@ if [ "$INVALID_PCAPS" = "yes" ]; then echo "Please note! One or more pcaps was invalid! You can scroll up to see which ones were invalid." fi +START_OLDEST_SLASH=$(echo $START_OLDEST | sed -e 's/-/%2F/g') +END_NEWEST_SLASH=$(echo $END_NEWEST | sed -e 's/-/%2F/g') + if [ "$VALID_PCAPS" = "yes" ]; then cat << EOF Import complete! You can use the following hyperlink to view data in the time range of your import. You can triple-click to quickly highlight the entire hyperlink and you can then copy it into your browser: -https://{{ MASTERIP }}/kibana/app/kibana#/dashboard/a8411b30-6d03-11ea-b301-3d6c35840645?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:'${START_OLDEST}T00:00:00.000Z',mode:absolute,to:'${END_NEWEST}T00:00:00.000Z')) +https://{{ MASTERIP }}/#/hunt?q=%2a%20%7C%20groupby%20event.module%20event.dataset&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20PM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20PM or you can manually set your Time Range to be: From: $START_OLDEST To: $END_NEWEST diff --git a/salt/elastalert/files/elastalert_config.yaml b/salt/elastalert/files/elastalert_config.yaml index 2ecf08ffa..c7b8b52bb 100644 --- a/salt/elastalert/files/elastalert_config.yaml +++ b/salt/elastalert/files/elastalert_config.yaml @@ -86,3 +86,25 @@ alert_time_limit: index_settings: shards: 1 replicas: 0 + +logging: + version: 1 + incremental: false + disable_existing_loggers: false + formatters: + logline: + format: '%(asctime)s %(levelname)+8s %(name)+20s %(message)s' + + handlers: + file: + class : logging.FileHandler + formatter: logline + level: INFO + filename: /var/log/elastalert/elastalert.log + + loggers: + '': + level: INFO + handlers: + - file + propagate: false diff --git a/salt/elastalert/files/modules/so/playbook-es.py b/salt/elastalert/files/modules/so/playbook-es.py index 0ba3f3601..c794bdf12 100644 --- a/salt/elastalert/files/modules/so/playbook-es.py +++ b/salt/elastalert/files/modules/so/playbook-es.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from datetime import date +from time import gmtime, strftime import requests,json from elastalert.alerts import Alerter @@ -13,11 +13,12 @@ class PlaybookESAlerter(Alerter): def alert(self, matches): for match in matches: + today = strftime("%Y.%m.%d", gmtime()) + timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S", gmtime()) headers = {"Content-Type": "application/json"} - payload = {"play_title": self.rule['play_title'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"data": match} - today = str(date.today()) - url = f"http://{self.rule['elasticsearch_host']}/playbook-alerts-{today}/_doc/" + payload = {"rule.name": self.rule['play_title'],"event.severity": self.rule['event.severity'],"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"event.module": self.rule['event.module'],"event.dataset": self.rule['event.dataset'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"rule.category": self.rule['rule.category'],"data": match, "@timestamp": timestamp} + url = f"http://{self.rule['elasticsearch_host']}/so-playbook-alerts-{today}/_doc/" requests.post(url, data=json.dumps(payload), headers=headers, verify=False) def get_info(self): - return {'type': 'PlaybookESAlerter'} + return {'type': 'PlaybookESAlerter'} \ No newline at end of file diff --git a/salt/elasticsearch/files/elasticsearch.yml b/salt/elasticsearch/files/elasticsearch.yml index a38c3db87..4d5d5b2e4 100644 --- a/salt/elasticsearch/files/elasticsearch.yml +++ b/salt/elasticsearch/files/elasticsearch.yml @@ -24,3 +24,4 @@ cluster.routing.allocation.disk.watermark.high: 98% cluster.routing.allocation.disk.watermark.flood_stage: 98% node.attr.box_type: {{ NODE_ROUTE_TYPE }} node.name: {{ ESCLUSTERNAME }} +script.max_compilations_rate: 1000/1m diff --git a/salt/elasticsearch/files/ingest/beats.common b/salt/elasticsearch/files/ingest/beats.common index 0e93abb03..9ea586156 100644 --- a/salt/elasticsearch/files/ingest/beats.common +++ b/salt/elasticsearch/files/ingest/beats.common @@ -2,10 +2,17 @@ "description" : "beats.common", "processors" : [ {"community_id": {"if": "ctx.winlog.event_data?.Protocol != null", "field":["winlog.event_data.SourceIp","winlog.event_data.SourcePort","winlog.event_data.DestinationIp","winlog.event_data.DestinationPort","winlog.event_data.Protocol"],"target_field":"network.community_id"}}, - { "set": { "if": "ctx.winlog?.channel != null", "field": "dataset", "value": "wel-{{winlog.channel}}", "override": true } }, - { "set": { "if": "ctx.agent?.type != null", "field": "module", "value": "{{agent.type}}", "override": true } }, + { "set": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "field": "event.module", "value": "sysmon", "override": true } }, + { "set": { "if": "ctx.winlog?.channel!= null", "field": "event.module", "value": "win_eventlog", "override": true, "ignore_failure": true } }, + { "set": { "if": "ctx.winlog?.channel != null", "field": "dataset", "value": "{{winlog.channel}}", "override": true } }, + { "set": { "if": "ctx.agent?.type != null", "field": "module", "value": "{{agent.type}}", "override": true } }, { "set": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 3", "field": "event.category", "value": "host,process,network", "override": true } }, { "set": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 1", "field": "event.category", "value": "host,process", "override": true } }, + { "set": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 1", "field": "event.dataset", "value": "process_creation", "override": true } }, + { "set": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 2", "field": "event.dataset", "value": "process_changed_file", "override": true } }, + { "set": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 3", "field": "event.dataset", "value": "network_connection", "override": true } }, + { "set": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 5", "field": "event.dataset", "value": "driver_loaded", "override": true } }, + { "set": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational' && ctx.event?.code == 6", "field": "event.dataset", "value": "image_loaded", "override": true } }, { "rename": { "field": "agent.hostname", "target_field": "agent.name", "ignore_missing": true } }, { "rename": { "field": "winlog.event_data.SubjectUserName", "target_field": "user.name", "ignore_missing": true } }, { "rename": { "field": "winlog.event_data.DestinationHostname", "target_field": "destination.hostname", "ignore_missing": true } }, diff --git a/salt/elasticsearch/files/ingest/common b/salt/elasticsearch/files/ingest/common index 2f67a1ddc..3bb19535e 100644 --- a/salt/elasticsearch/files/ingest/common +++ b/salt/elasticsearch/files/ingest/common @@ -36,14 +36,19 @@ "ignore_failure": true, "index_name_format": "yyyy.MM.dd" } - }, - { "rename": { "field": "module", "target_field": "event.module", "ignore_missing": true } }, - { "rename": { "field": "dataset", "target_field": "event.dataset", "ignore_missing": true } }, + }, + { "set": { "if": "ctx.event?.severity == 3", "field": "event.severity_label", "value": "low", "override": true } }, + { "set": { "if": "ctx.event?.severity == 5", "field": "event.severity_label", "value": "medium", "override": true } }, + { "set": { "if": "ctx.event?.severity == 7", "field": "event.severity_label", "value": "high", "override": true } }, + { "set": { "if": "ctx.event?.severity == 10", "field": "event.severity_label", "value": "critical", "override": true } }, + { "rename": { "field": "module", "target_field": "event.module", "ignore_failure": true, "ignore_missing": true } }, + { "rename": { "field": "dataset", "target_field": "event.dataset", "ignore_failure": true, "ignore_missing": true } }, { "rename": { "field": "category", "target_field": "event.category", "ignore_missing": true } }, { "rename": { "field": "message2.community_id", "target_field": "network.community_id", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "destination.port", "type": "integer", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "source.port", "type": "integer", "ignore_failure": true, "ignore_missing": true } }, { "convert": { "field": "log.id.uid", "type": "string", "ignore_failure": true, "ignore_missing": true } }, + { "convert": { "field": "event.severity", "type": "integer", "ignore_failure": true, "ignore_missing": true } }, { "remove": { "field": [ "index_name_prefix", "message2", "type" ], diff --git a/salt/elasticsearch/files/ingest/ossec.alert b/salt/elasticsearch/files/ingest/ossec.alert index 2aab7f630..7654bf255 100644 --- a/salt/elasticsearch/files/ingest/ossec.alert +++ b/salt/elasticsearch/files/ingest/ossec.alert @@ -49,6 +49,9 @@ { "set": { "if": "ctx.rule.level == 13", "field": "rule.category", "value": "Unusal error (high importance)" } }, { "set": { "if": "ctx.rule.level == 14", "field": "rule.category", "value": "High importance security event" } }, { "set": { "if": "ctx.rule.level == 15", "field": "rule.category", "value": "Severe attack" } }, + { "set": { "if": "ctx.rule.level <= 7", "field": "event.severity", "value": 1, "override": true } }, + { "set": { "if": "ctx.rule.level >= 8 && ctx.rule.level <= 11", "field": "event.severity", "value": 2, "override": true } }, + { "set": { "if": "ctx.rule.level >= 12", "field": "event.severity", "value": 3, "override": true } }, { "append": { "if": "ctx.rule.level != null", "field": "tags", "value": ["alert"] } }, { "remove": { "field": [ "predecoder", "decoder" ], "ignore_missing": true, "ignore_failure": false } }, { "pipeline": { "name": "common" } } diff --git a/salt/elasticsearch/files/ingest/suricata.alert b/salt/elasticsearch/files/ingest/suricata.alert index 82486adc6..5a0cfc4df 100644 --- a/salt/elasticsearch/files/ingest/suricata.alert +++ b/salt/elasticsearch/files/ingest/suricata.alert @@ -7,6 +7,9 @@ { "rename":{ "field": "rule.signature_id", "target_field": "rule.uuid", "ignore_failure": true } }, { "rename":{ "field": "rule.signature_id", "target_field": "rule.signature", "ignore_failure": true } }, { "rename":{ "field": "message2.payload_printable", "target_field": "network.data.decoded", "ignore_failure": true } }, + { "set": { "if": "ctx.rule.severity == 3", "field": "event.severity", "value": 1, "override": true } }, + { "set": { "if": "ctx.rule.severity == 2", "field": "event.severity", "value": 2, "override": true } }, + { "set": { "if": "ctx.rule.severity == 1", "field": "event.severity", "value": 3, "override": true } }, { "pipeline": { "name": "common" } } ] } diff --git a/salt/pcap/files/sensoroni.json b/salt/pcap/files/sensoroni.json index ed673d969..654148f2a 100644 --- a/salt/pcap/files/sensoroni.json +++ b/salt/pcap/files/sensoroni.json @@ -8,6 +8,7 @@ "serverUrl": "https://{{ MASTER }}/sensoroniagents", "verifyCert": false, "modules": { + "importer": {}, "statickeyauth": { "apiKey": "{{ SENSORONIKEY }}" }, diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json index 693c44aeb..aeb017e68 100644 --- a/salt/soc/files/soc/soc.json +++ b/salt/soc/files/soc/soc.json @@ -83,7 +83,7 @@ }, "queries": [ { "name": "Default Query", "description": "Show all events grouped by the origin host", "query": "* | groupby observer.name"}, - { "name": "Log Type", "description": "Show all events grouped by type", "query": "* | groupby event.module"}, + { "name": "Log Type", "description": "Show all events grouped by module and dataset", "query": "* | groupby event.module event.dataset"}, { "name": "Elastalerts", "description": "", "query": "_type:elastalert | groupby rule.name"}, { "name": "Alerts", "description": "Show all alerts grouped by alert source", "query": "event.dataset: alert | groupby event.module"}, { "name": "NIDS Alerts", "description": "Show all NIDS alerts grouped by alert name", "query": "event.category: network AND event.dataset: alert | groupby rule.name"}, @@ -107,6 +107,7 @@ { "name": "DNS", "description": "DNS queries grouped by response code", "query": "event.dataset:dns | groupby dns.response.code_name destination.port"}, { "name": "DNS", "description": "DNS highest registered domain", "query": "event.dataset:dns | groupby dns.highest_registered_domain.keyword destination.port"}, { "name": "DNS", "description": "DNS grouped by parent domain", "query": "event.dataset:dns | groupby dns.parent_domain.keyword destination.port"}, + { "name": "DPD", "description": "Dynamic Protocol Detection errors", "query": "event.dataset:dpd | groupby error.reason"}, { "name": "Files", "description": "Files grouped by mimetype", "query": "event.dataset:file | groupby file.mime_type source.ip"}, { "name": "Files", "description": "Files grouped by source", "query": "event.dataset:file | groupby file.source source.ip"}, { "name": "FTP", "description": "FTP grouped by argument", "query": "event.dataset:ftp | groupby ftp.argument"}, diff --git a/salt/soctopus/files/SOCtopus.conf b/salt/soctopus/files/SOCtopus.conf index d2c3eea2d..e32aaf8da 100644 --- a/salt/soctopus/files/SOCtopus.conf +++ b/salt/soctopus/files/SOCtopus.conf @@ -61,6 +61,7 @@ slack_webhook = YOURSLACKWEBHOOK [playbook] playbook_url = http://{{MASTER}}:3200/playbook +playbook_ext_url = https://{{MASTER}}/playbook playbook_key = de6639318502476f2fa5aa06f43f51fb389a3d7f playbook_verifycert = no playbook_unit_test_index = playbook-testing diff --git a/salt/soctopus/files/templates/es-generic.template b/salt/soctopus/files/templates/es-generic.template deleted file mode 100644 index cdda8a19b..000000000 --- a/salt/soctopus/files/templates/es-generic.template +++ /dev/null @@ -1,7 +0,0 @@ -{% set ES = salt['pillar.get']('static:masterip', '') %} - -alert: modules.so.playbook-es.PlaybookESAlerter -elasticsearch_host: "{{ ES }}:9200" -play_title: "" -play_url: "https://{{ ES }}/playbook/issues/6000" -sigma_level: "" diff --git a/salt/soctopus/files/templates/generic.template b/salt/soctopus/files/templates/generic.template index 68dc040fc..0bdb6a2ba 100644 --- a/salt/soctopus/files/templates/generic.template +++ b/salt/soctopus/files/templates/generic.template @@ -1,6 +1,6 @@ -{% set es = salt['pillar.get']('static:masterip', '') %} -{% set hivehost = salt['pillar.get']('static:masterip', '') %} -{% set hivekey = salt['pillar.get']('static:hivekey', '') %} +{%- set es = salt['pillar.get']('static:masterip', '') %} +{%- set hivehost = salt['pillar.get']('static:masterip', '') %} +{%- set hivekey = salt['pillar.get']('static:hivekey', '') %} alert: hivealerter hive_connection: @@ -23,3 +23,15 @@ hive_alert_config: status: 'New' follow: True caseTemplate: '5000' + +alert: modules.so.playbook-es.PlaybookESAlerter +elasticsearch_host: "{{ es }}:9200" +play_title: "" +event.module: "playbook" +event.dataset: "alert" +event.severity: +rule.category: +play_url: "https://{{ es }}/playbook/issues/6000" +kibana_pivot: "https://{{es}}/kibana/app/kibana#/discover?_g=()&_a=(columns:!(_source),interval:auto,query:(language:lucene,query:'_id:{[_id]}'),sort:!('@timestamp',desc))" +soc_pivot: "https://{{es}}/#/hunt" +sigma_level: "" \ No newline at end of file diff --git a/salt/soctopus/files/templates/osquery.template b/salt/soctopus/files/templates/osquery.template index 28ea29ee9..90bc0743e 100644 --- a/salt/soctopus/files/templates/osquery.template +++ b/salt/soctopus/files/templates/osquery.template @@ -31,3 +31,14 @@ hive_alert_config: caseTemplate: '5000' +alert: modules.so.playbook-es.PlaybookESAlerter +elasticsearch_host: "{{ es }}:9200" +play_title: "" +event.module: "playbook" +event.dataset: "alert" +event.severity: +rule.category: +play_url: "https://{{ es }}/playbook/issues/6000" +kibana_pivot: "https://{{es}}/kibana/app/kibana#/discover?_g=()&_a=(columns:!(_source),interval:auto,query:(language:lucene,query:'_id:{[_id]}'),sort:!('@timestamp',desc))" +soc_pivot: "https://{{es}}/#/hunt" +sigma_level: "" \ No newline at end of file diff --git a/setup/so-functions b/setup/so-functions index 51d6d43e6..2eca9874d 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -838,18 +838,6 @@ firewall_generate_templates() { } -fleet_pillar() { - - local pillar_file="$temp_install_dir"/pillar/minions/"$MINION_ID".sls - - # Create the fleet pillar - printf '%s\n'\ - "fleet:"\ - " mainip: $MAINIP"\ - " master: $MSRV"\ - "" > "$pillar_file" -} - generate_passwords(){ # Generate Random Passwords for Things MYSQLPASS=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 20 | head -n 1) diff --git a/setup/so-setup b/setup/so-setup index 3f6d42380..6ac24f33d 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -367,14 +367,14 @@ fi set_main_ip >> $setup_log 2>&1 +host_pillar >> $setup_log 2>&1 + if [[ $is_minion ]]; then set_updates >> $setup_log 2>&1 copy_ssh_key >> $setup_log 2>&1 fi -if [[ "$OSQUERY" = 1 ]]; then - host_pillar >> $setup_log 2>&1 -fi + # Begin install {