From a13e6257c368b20c09acdd676da298b93d928677 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 8 May 2023 14:38:55 -0400 Subject: [PATCH 01/12] Don't read from 'known_hosts.log', 'known_services.log', or 'ntp.log' --- salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json b/salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json index a4e0c94ee..ebe5173eb 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json @@ -20,7 +20,7 @@ "data_stream.dataset": "zeek", "tags": [], "processors": "- dissect:\n tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n field: \"log.file.path\"\n trim_chars: \".log\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\");\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n- add_tags:\n tags: \"ics\"\n when:\n regexp:\n pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"", - "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|loaded_scripts|packet_filter|stats|stderr|stdout.log$\"]\n" + "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|known_hosts|known_services|loaded_scripts|ntp|packet_filter|stats|stderr|stdout.log$\"]\n" } } } From 82c3d78672422daace57ec43e4e3ed5e35273246 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 18 May 2023 16:52:27 +0000 Subject: [PATCH 02/12] Change Elasticsearch host syntax --- salt/elastalert/map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elastalert/map.jinja b/salt/elastalert/map.jinja index cc395d8ee..7cec262d0 100644 --- a/salt/elastalert/map.jinja +++ b/salt/elastalert/map.jinja @@ -8,7 +8,7 @@ {% set elastalert_pillar = salt['pillar.get']('elastalert:config', {}) %} -{% do ELASTALERTDEFAULTS.elastalert.config.update({'es_host': GLOBALS.manager}) %} +{% do ELASTALERTDEFAULTS.elastalert.config.update({'es_hosts': 'https://' + GLOBALS.manager + ':' + ELASTALERTDEFAULTS.elastalert.config.es_port|string}) %} {% do ELASTALERTDEFAULTS.elastalert.config.update({'es_username': pillar.elasticsearch.auth.users.so_elastic_user.user}) %} {% do ELASTALERTDEFAULTS.elastalert.config.update({'es_password': pillar.elasticsearch.auth.users.so_elastic_user.pass}) %} From d3c7ea4805374b4c3b1c22095163eb71c6bc4a51 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 18 May 2023 16:55:26 +0000 Subject: [PATCH 03/12] Add EQL option --- salt/elastalert/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elastalert/defaults.yaml b/salt/elastalert/defaults.yaml index 1083fa8fd..0b31a6b99 100644 --- a/salt/elastalert/defaults.yaml +++ b/salt/elastalert/defaults.yaml @@ -13,6 +13,7 @@ elastalert: es_port: 9200 es_conn_timeout: 55 max_query_size: 5000 + eql: true use_ssl: true verify_certs: false writeback_index: elastalert_status From 4f8fcd3369e2d68e554a645d7f97d6e5140ff007 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 19 May 2023 12:19:44 -0400 Subject: [PATCH 04/12] Update config_saved_objects.ndjson --- salt/kibana/files/config_saved_objects.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index e2eced11e..9b69eb781 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.7.0","id": "8.7.0","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.7.1","id": "8.7.1","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} From 87c42ece007aa048869afa18a5503ef59a4e268d Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 19 May 2023 12:21:09 -0400 Subject: [PATCH 05/12] Update so-kibana-config-load --- salt/kibana/tools/sbin_jinja/so-kibana-config-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/tools/sbin_jinja/so-kibana-config-load b/salt/kibana/tools/sbin_jinja/so-kibana-config-load index 12466cae1..e65955178 100644 --- a/salt/kibana/tools/sbin_jinja/so-kibana-config-load +++ b/salt/kibana/tools/sbin_jinja/so-kibana-config-load @@ -63,7 +63,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.7.0" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.7.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done From 1ddf45bbbefe367b496ca4800a68fac473d77c04 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 19 May 2023 12:39:27 -0400 Subject: [PATCH 06/12] Change Elastalert writeback index name from 'elastalert_status' to 'elastalert' --- salt/elastalert/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elastalert/defaults.yaml b/salt/elastalert/defaults.yaml index 0b31a6b99..c073e4ee6 100644 --- a/salt/elastalert/defaults.yaml +++ b/salt/elastalert/defaults.yaml @@ -16,7 +16,7 @@ elastalert: eql: true use_ssl: true verify_certs: false - writeback_index: elastalert_status + writeback_index: elastalert alert_time_limit: days: 2 index_settings: From 03f97b309aae5b4d22b3b634f7aeaefc45631ee1 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 19 May 2023 14:18:19 -0400 Subject: [PATCH 07/12] fix lib dependency issue with whoisit --- salt/sensoroni/files/analyzers/whoislookup/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/sensoroni/files/analyzers/whoislookup/requirements.txt b/salt/sensoroni/files/analyzers/whoislookup/requirements.txt index e19053e6c..3e7cb1d86 100755 --- a/salt/sensoroni/files/analyzers/whoislookup/requirements.txt +++ b/salt/sensoroni/files/analyzers/whoislookup/requirements.txt @@ -1 +1,2 @@ +requests==2.29.0 whoisit>=2.5.3 From 167051af2810b7f519dfdb8a255f1424a8763755 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 19 May 2023 15:44:09 -0400 Subject: [PATCH 08/12] Exclude Zeek's reporter.log from being picked up by Elastic Agent --- salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json b/salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json index f0127f2ae..03543b124 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json @@ -20,7 +20,7 @@ "data_stream.dataset": "zeek", "tags": [], "processors": "- dissect:\n tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n field: \"log.file.path\"\n trim_chars: \".log\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\");\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n- add_tags:\n tags: \"ics\"\n when:\n regexp:\n pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"", - "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|known_hosts|known_services|loaded_scripts|ntp|packet_filter|stats|stderr|stdout.log$\"]\n" + "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|known_hosts|known_services|loaded_scripts|ntp|packet_filter|reporter|stats|stderr|stdout.log$\"]\n" } } } From 2c10ad7eec813444154a15fe6bfb5d00c5de7686 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 19 May 2023 15:50:33 -0400 Subject: [PATCH 09/12] Check if 'dns.query' is null --- salt/elasticsearch/files/ingest/zeek.dns | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/zeek.dns b/salt/elasticsearch/files/ingest/zeek.dns index 5f39b8017..57a9347a5 100644 --- a/salt/elasticsearch/files/ingest/zeek.dns +++ b/salt/elasticsearch/files/ingest/zeek.dns @@ -25,7 +25,7 @@ { "rename": { "field": "message2.rejected", "target_field": "dns.query.rejected", "ignore_missing": true } }, { "script": { "lang": "painless", "source": "ctx.dns.query.length = ctx.dns.query.name.length()", "ignore_failure": true } }, { "set": { "if": "ctx._index == 'so-zeek'", "field": "_index", "value": "so-zeek_dns", "override": true } }, - { "pipeline": { "if": "ctx.dns.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld" } }, + { "pipeline": { "if": "ctx.dns?.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld" } }, { "pipeline": { "name": "zeek.common" } } ] } From 00bd93c026a6da5675417e61d4952c478adcc784 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 19 May 2023 17:14:13 -0400 Subject: [PATCH 10/12] Update 'url' to use 'es_hosts' --- salt/elastalert/files/modules/so/playbook-es.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elastalert/files/modules/so/playbook-es.py b/salt/elastalert/files/modules/so/playbook-es.py index 62afab41e..680c81d53 100644 --- a/salt/elastalert/files/modules/so/playbook-es.py +++ b/salt/elastalert/files/modules/so/playbook-es.py @@ -31,8 +31,8 @@ class PlaybookESAlerter(Alerter): creds = (self.rule['es_username'], self.rule['es_password']) payload = {"rule": { "name": self.rule['play_title'],"case_template": self.rule['play_id'],"uuid": self.rule['play_id'],"category": self.rule['rule.category']},"event":{ "severity": self.rule['event.severity'],"module": self.rule['event.module'],"dataset": self.rule['event.dataset'],"severity_label": self.rule['sigma_level']},"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"event_data": match, "@timestamp": timestamp} - url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/so-playbook-alerts-{today}/_doc/" + url = f"{self.rule['es_hosts']}/so-playbook-alerts-{today}/_doc/" requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds) def get_info(self): - return {'type': 'PlaybookESAlerter'} \ No newline at end of file + return {'type': 'PlaybookESAlerter'} From a881cab469b334a9769993b6573d87f69458d2f9 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 19 May 2023 23:54:30 -0400 Subject: [PATCH 11/12] use the same requests version that's already packaged with the analyzer --- salt/sensoroni/files/analyzers/whoislookup/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/sensoroni/files/analyzers/whoislookup/requirements.txt b/salt/sensoroni/files/analyzers/whoislookup/requirements.txt index 3e7cb1d86..a3901f38c 100755 --- a/salt/sensoroni/files/analyzers/whoislookup/requirements.txt +++ b/salt/sensoroni/files/analyzers/whoislookup/requirements.txt @@ -1,2 +1,2 @@ -requests==2.29.0 +requests==2.27.1 whoisit>=2.5.3 From ba0ec18a33a823aa24dea4e1b435dd1d77a71ea2 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 22 May 2023 14:52:07 -0400 Subject: [PATCH 12/12] Ignore Synchronize button clicks when an active salt job is running and another is already in queue --- salt/soc/files/bin/salt-relay.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/files/bin/salt-relay.sh b/salt/soc/files/bin/salt-relay.sh index 8a81fc715..a98a587a6 100755 --- a/salt/soc/files/bin/salt-relay.sh +++ b/salt/soc/files/bin/salt-relay.sh @@ -143,12 +143,12 @@ function manage_salt() { state) log "Performing '$op' for '$state' on minion '$minion'" state=$(echo "$request" | jq -r .state) - response=$($CMD_PREFIX salt --async "$minion" state.apply "$state" queue=True) + response=$($CMD_PREFIX salt --async "$minion" state.apply "$state" queue=2) exit_code=$? ;; highstate) log "Performing '$op' on minion $minion" - response=$($CMD_PREFIX salt --async "$minion" state.highstate queue=True) + response=$($CMD_PREFIX salt --async "$minion" state.highstate queue=2) exit_code=$? ;; activejobs)