From a13e6257c368b20c09acdd676da298b93d928677 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 8 May 2023 14:38:55 -0400 Subject: [PATCH 01/18] Don't read from 'known_hosts.log', 'known_services.log', or 'ntp.log' --- salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json b/salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json index a4e0c94ee..ebe5173eb 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json @@ -20,7 +20,7 @@ "data_stream.dataset": "zeek", "tags": [], "processors": "- dissect:\n tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n field: \"log.file.path\"\n trim_chars: \".log\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\");\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n- add_tags:\n tags: \"ics\"\n when:\n regexp:\n pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"", - "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|loaded_scripts|packet_filter|stats|stderr|stdout.log$\"]\n" + "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|known_hosts|known_services|loaded_scripts|ntp|packet_filter|stats|stderr|stdout.log$\"]\n" } } } From 4930ae4ba6548dd05c0a9724dc21d013d1bd8f3a Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Wed, 17 May 2023 18:14:21 -0400 Subject: [PATCH 02/18] add missing var for local dev --- salt/soc/files/bin/salt-relay.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/soc/files/bin/salt-relay.sh b/salt/soc/files/bin/salt-relay.sh index 1b21ac225..8a81fc715 100755 --- a/salt/soc/files/bin/salt-relay.sh +++ b/salt/soc/files/bin/salt-relay.sh @@ -70,7 +70,7 @@ function manage_user() { lastName=$(echo "$request" | jq -r .lastName) note=$(echo "$request" | jq -r .note) log "Performing user '$op' for user '$email' with firstname '$firstName', lastname '$lastName', note '$note' and role '$role'" - response=$(echo "$password" | so-user "$op" --email "$email" --firstName "$firstName" --lastName "$lastName" --note "$note" --role "$role" --skip-sync) + response=$(echo "$password" | $CMD_PREFIX so-user "$op" --email "$email" --firstName "$firstName" --lastName "$lastName" --note "$note" --role "$role" --skip-sync) exit_code=$? ;; add|enable|disable|delete) From f4b8d385ee6cf9c540aaa18f2b0f2e81e7b2ce4f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 18 May 2023 08:36:24 -0400 Subject: [PATCH 03/18] remove conditional on cacertz and capemz --- salt/elasticsearch/ca.sls | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/elasticsearch/ca.sls b/salt/elasticsearch/ca.sls index 49eb44a94..5485bb676 100644 --- a/salt/elasticsearch/ca.sls +++ b/salt/elasticsearch/ca.sls @@ -26,7 +26,6 @@ catrustscript: GLOBALS: {{ GLOBALS }} {% endif %} -{% if grains.role in ['so-manager', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-searchnode'] %} cacertz: file.managed: - name: /opt/so/conf/ca/cacerts @@ -40,7 +39,6 @@ capemz: - source: salt://elasticsearch/tls-ca-bundle.pem - user: 939 - group: 939 -{% endif %} {% else %} From 82c3d78672422daace57ec43e4e3ed5e35273246 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 18 May 2023 16:52:27 +0000 Subject: [PATCH 04/18] Change Elasticsearch host syntax --- salt/elastalert/map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elastalert/map.jinja b/salt/elastalert/map.jinja index cc395d8ee..7cec262d0 100644 --- a/salt/elastalert/map.jinja +++ b/salt/elastalert/map.jinja @@ -8,7 +8,7 @@ {% set elastalert_pillar = salt['pillar.get']('elastalert:config', {}) %} -{% do ELASTALERTDEFAULTS.elastalert.config.update({'es_host': GLOBALS.manager}) %} +{% do ELASTALERTDEFAULTS.elastalert.config.update({'es_hosts': 'https://' + GLOBALS.manager + ':' + ELASTALERTDEFAULTS.elastalert.config.es_port|string}) %} {% do ELASTALERTDEFAULTS.elastalert.config.update({'es_username': pillar.elasticsearch.auth.users.so_elastic_user.user}) %} {% do ELASTALERTDEFAULTS.elastalert.config.update({'es_password': pillar.elasticsearch.auth.users.so_elastic_user.pass}) %} From d3c7ea4805374b4c3b1c22095163eb71c6bc4a51 Mon Sep 17 00:00:00 2001 From: Wes Date: Thu, 18 May 2023 16:55:26 +0000 Subject: [PATCH 05/18] Add EQL option --- salt/elastalert/defaults.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/elastalert/defaults.yaml b/salt/elastalert/defaults.yaml index 1083fa8fd..0b31a6b99 100644 --- a/salt/elastalert/defaults.yaml +++ b/salt/elastalert/defaults.yaml @@ -13,6 +13,7 @@ elastalert: es_port: 9200 es_conn_timeout: 55 max_query_size: 5000 + eql: true use_ssl: true verify_certs: false writeback_index: elastalert_status From 25b0934cda538008783947c02c2d1b4e1e9399f9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 18 May 2023 13:06:20 -0400 Subject: [PATCH 06/18] confirm manager ip when found in setup --- setup/so-functions | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 9dd1b7713..e5ec16e4a 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -49,8 +49,9 @@ add_admin_user() { } add_mngr_ip_to_hosts() { - info "Adding $MSRV to /etc/hosts with IP: $MSRVIP" - echo "$MSRVIP $MSRV" >> /etc/hosts + info "Adding $MSRV to /etc/hosts with IP: $MSRVIP" + whiptail_manager_ip $MSRVIP + echo "$MSRVIP $MSRV" >> /etc/hosts } add_socore_user_manager() { From 02920b5ac9f26eb612f454ea17ef3ae467c9aa30 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 18 May 2023 13:25:12 -0400 Subject: [PATCH 07/18] confirm manager ip when found in setup --- setup/so-functions | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index e5ec16e4a..d9322c171 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -49,8 +49,7 @@ add_admin_user() { } add_mngr_ip_to_hosts() { - info "Adding $MSRV to /etc/hosts with IP: $MSRVIP" - whiptail_manager_ip $MSRVIP + info "Adding $MSRV to /etc/hosts with IP: $MSRVIP" echo "$MSRVIP $MSRV" >> /etc/hosts } @@ -400,6 +399,11 @@ collect_mngr_hostname() { done else MSRVIP=$(getent hosts "$MSRV" | awk 'NR==1{print $1}') + whiptail_manager_ip "$MSRVIP" + while ! valid_ip4 "$MSRVIP" || [[ $MSRVIP == "$MAINIP" || $MSRVIP == "127.0.0.1" ]]; do + whiptail_invalid_input + whiptail_manager_ip "$MSRVIP" + done fi } From bab2f7282c70acf5a1186657e0798cba73076a99 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 18 May 2023 13:27:48 -0400 Subject: [PATCH 08/18] fix spaciong --- setup/so-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index d9322c171..09e219cfd 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -49,8 +49,8 @@ add_admin_user() { } add_mngr_ip_to_hosts() { - info "Adding $MSRV to /etc/hosts with IP: $MSRVIP" - echo "$MSRVIP $MSRV" >> /etc/hosts + info "Adding $MSRV to /etc/hosts with IP: $MSRVIP" + echo "$MSRVIP $MSRV" >> /etc/hosts } add_socore_user_manager() { From 4ef77f90505dde657b2ff15b67662c762ba442db Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 18 May 2023 17:34:59 -0400 Subject: [PATCH 09/18] enable playbook on eval in pillar during setup --- salt/manager/tools/sbin/so-minion | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index 7d0703653..0ba62e4b7 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -266,7 +266,7 @@ function add_redis_to_minion() { " " >> $PILLARFILE } -function add_strelka_strelka_to_minion() { +function add_strelka_to_minion() { printf '%s\n'\ "strelka:"\ " backend:"\ @@ -407,7 +407,8 @@ function apply_ES_state() { function createEVAL() { add_elasticsearch_to_minion add_sensor_to_minion - add_strelka_strelka_to_minion + add_strelka_to_minion + add_playbook_to_minion add_elastalert_to_minion add_kibana_to_minion add_curator_to_minion @@ -427,7 +428,7 @@ function createSTANDALONE() { add_elasticsearch_to_minion add_logstash_to_minion add_sensor_to_minion - add_strelka_strelka_to_minion + add_strelka_to_minion add_playbook_to_minion add_elastalert_to_minion add_kibana_to_minion @@ -518,7 +519,7 @@ function createHEAVYNODE() { add_elasticsearch_to_minion add_logstash_to_minion add_sensor_to_minion - add_strelka_strelka_to_minion + add_strelka_to_minion add_redis_to_minion add_curator_to_minion add_telegraf_to_minion @@ -526,7 +527,7 @@ function createHEAVYNODE() { function createSENSOR() { add_sensor_to_minion - add_strelka_strelka_to_minion + add_strelka_to_minion add_telegraf_to_minion } From 4f8fcd3369e2d68e554a645d7f97d6e5140ff007 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 19 May 2023 12:19:44 -0400 Subject: [PATCH 10/18] Update config_saved_objects.ndjson --- salt/kibana/files/config_saved_objects.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index e2eced11e..9b69eb781 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.7.0","id": "8.7.0","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.7.1","id": "8.7.1","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} From 87c42ece007aa048869afa18a5503ef59a4e268d Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Fri, 19 May 2023 12:21:09 -0400 Subject: [PATCH 11/18] Update so-kibana-config-load --- salt/kibana/tools/sbin_jinja/so-kibana-config-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/tools/sbin_jinja/so-kibana-config-load b/salt/kibana/tools/sbin_jinja/so-kibana-config-load index 12466cae1..e65955178 100644 --- a/salt/kibana/tools/sbin_jinja/so-kibana-config-load +++ b/salt/kibana/tools/sbin_jinja/so-kibana-config-load @@ -63,7 +63,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.7.0" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.7.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done From 1ddf45bbbefe367b496ca4800a68fac473d77c04 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 19 May 2023 12:39:27 -0400 Subject: [PATCH 12/18] Change Elastalert writeback index name from 'elastalert_status' to 'elastalert' --- salt/elastalert/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elastalert/defaults.yaml b/salt/elastalert/defaults.yaml index 0b31a6b99..c073e4ee6 100644 --- a/salt/elastalert/defaults.yaml +++ b/salt/elastalert/defaults.yaml @@ -16,7 +16,7 @@ elastalert: eql: true use_ssl: true verify_certs: false - writeback_index: elastalert_status + writeback_index: elastalert alert_time_limit: days: 2 index_settings: From 03f97b309aae5b4d22b3b634f7aeaefc45631ee1 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 19 May 2023 14:18:19 -0400 Subject: [PATCH 13/18] fix lib dependency issue with whoisit --- salt/sensoroni/files/analyzers/whoislookup/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/sensoroni/files/analyzers/whoislookup/requirements.txt b/salt/sensoroni/files/analyzers/whoislookup/requirements.txt index e19053e6c..3e7cb1d86 100755 --- a/salt/sensoroni/files/analyzers/whoislookup/requirements.txt +++ b/salt/sensoroni/files/analyzers/whoislookup/requirements.txt @@ -1 +1,2 @@ +requests==2.29.0 whoisit>=2.5.3 From 167051af2810b7f519dfdb8a255f1424a8763755 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 19 May 2023 15:44:09 -0400 Subject: [PATCH 14/18] Exclude Zeek's reporter.log from being picked up by Elastic Agent --- salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json b/salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json index f0127f2ae..03543b124 100644 --- a/salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json +++ b/salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json @@ -20,7 +20,7 @@ "data_stream.dataset": "zeek", "tags": [], "processors": "- dissect:\n tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n field: \"log.file.path\"\n trim_chars: \".log\"\n target_prefix: \"\"\n- script:\n lang: javascript\n source: >\n function process(event) {\n var pl = event.Get(\"pipeline\");\n event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n }\n- add_fields:\n target: event\n fields:\n category: network\n module: zeek\n- add_tags:\n tags: \"ics\"\n when:\n regexp:\n pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"", - "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|known_hosts|known_services|loaded_scripts|ntp|packet_filter|stats|stderr|stdout.log$\"]\n" + "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|known_hosts|known_services|loaded_scripts|ntp|packet_filter|reporter|stats|stderr|stdout.log$\"]\n" } } } From 2c10ad7eec813444154a15fe6bfb5d00c5de7686 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 19 May 2023 15:50:33 -0400 Subject: [PATCH 15/18] Check if 'dns.query' is null --- salt/elasticsearch/files/ingest/zeek.dns | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/zeek.dns b/salt/elasticsearch/files/ingest/zeek.dns index 5f39b8017..57a9347a5 100644 --- a/salt/elasticsearch/files/ingest/zeek.dns +++ b/salt/elasticsearch/files/ingest/zeek.dns @@ -25,7 +25,7 @@ { "rename": { "field": "message2.rejected", "target_field": "dns.query.rejected", "ignore_missing": true } }, { "script": { "lang": "painless", "source": "ctx.dns.query.length = ctx.dns.query.name.length()", "ignore_failure": true } }, { "set": { "if": "ctx._index == 'so-zeek'", "field": "_index", "value": "so-zeek_dns", "override": true } }, - { "pipeline": { "if": "ctx.dns.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld" } }, + { "pipeline": { "if": "ctx.dns?.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld" } }, { "pipeline": { "name": "zeek.common" } } ] } From 00bd93c026a6da5675417e61d4952c478adcc784 Mon Sep 17 00:00:00 2001 From: weslambert Date: Fri, 19 May 2023 17:14:13 -0400 Subject: [PATCH 16/18] Update 'url' to use 'es_hosts' --- salt/elastalert/files/modules/so/playbook-es.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elastalert/files/modules/so/playbook-es.py b/salt/elastalert/files/modules/so/playbook-es.py index 62afab41e..680c81d53 100644 --- a/salt/elastalert/files/modules/so/playbook-es.py +++ b/salt/elastalert/files/modules/so/playbook-es.py @@ -31,8 +31,8 @@ class PlaybookESAlerter(Alerter): creds = (self.rule['es_username'], self.rule['es_password']) payload = {"rule": { "name": self.rule['play_title'],"case_template": self.rule['play_id'],"uuid": self.rule['play_id'],"category": self.rule['rule.category']},"event":{ "severity": self.rule['event.severity'],"module": self.rule['event.module'],"dataset": self.rule['event.dataset'],"severity_label": self.rule['sigma_level']},"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"event_data": match, "@timestamp": timestamp} - url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/so-playbook-alerts-{today}/_doc/" + url = f"{self.rule['es_hosts']}/so-playbook-alerts-{today}/_doc/" requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds) def get_info(self): - return {'type': 'PlaybookESAlerter'} \ No newline at end of file + return {'type': 'PlaybookESAlerter'} From a881cab469b334a9769993b6573d87f69458d2f9 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 19 May 2023 23:54:30 -0400 Subject: [PATCH 17/18] use the same requests version that's already packaged with the analyzer --- salt/sensoroni/files/analyzers/whoislookup/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/sensoroni/files/analyzers/whoislookup/requirements.txt b/salt/sensoroni/files/analyzers/whoislookup/requirements.txt index 3e7cb1d86..a3901f38c 100755 --- a/salt/sensoroni/files/analyzers/whoislookup/requirements.txt +++ b/salt/sensoroni/files/analyzers/whoislookup/requirements.txt @@ -1,2 +1,2 @@ -requests==2.29.0 +requests==2.27.1 whoisit>=2.5.3 From ba0ec18a33a823aa24dea4e1b435dd1d77a71ea2 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 22 May 2023 14:52:07 -0400 Subject: [PATCH 18/18] Ignore Synchronize button clicks when an active salt job is running and another is already in queue --- salt/soc/files/bin/salt-relay.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/soc/files/bin/salt-relay.sh b/salt/soc/files/bin/salt-relay.sh index 8a81fc715..a98a587a6 100755 --- a/salt/soc/files/bin/salt-relay.sh +++ b/salt/soc/files/bin/salt-relay.sh @@ -143,12 +143,12 @@ function manage_salt() { state) log "Performing '$op' for '$state' on minion '$minion'" state=$(echo "$request" | jq -r .state) - response=$($CMD_PREFIX salt --async "$minion" state.apply "$state" queue=True) + response=$($CMD_PREFIX salt --async "$minion" state.apply "$state" queue=2) exit_code=$? ;; highstate) log "Performing '$op' on minion $minion" - response=$($CMD_PREFIX salt --async "$minion" state.highstate queue=True) + response=$($CMD_PREFIX salt --async "$minion" state.highstate queue=2) exit_code=$? ;; activejobs)