Merge remote-tracking branch 'remotes/origin/2.4/dev' into 2.4/smallfixes

salt/elastalert/defaults.yaml

@@ -13,9 +13,10 @@ elastalert:
     es_port: 9200
     es_conn_timeout: 55
     max_query_size: 5000
+    eql: true
     use_ssl: true
     verify_certs: false
-    writeback_index: elastalert_status
+    writeback_index: elastalert
     alert_time_limit:
       days: 2
     index_settings:

salt/elastalert/files/modules/so/playbook-es.py

@@ -31,8 +31,8 @@ class PlaybookESAlerter(Alerter):
                 creds = (self.rule['es_username'], self.rule['es_password'])
 
             payload = {"rule": { "name": self.rule['play_title'],"case_template": self.rule['play_id'],"uuid": self.rule['play_id'],"category": self.rule['rule.category']},"event":{ "severity": self.rule['event.severity'],"module": self.rule['event.module'],"dataset": self.rule['event.dataset'],"severity_label": self.rule['sigma_level']},"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"event_data": match, "@timestamp": timestamp}
-            url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/so-playbook-alerts-{today}/_doc/"
+            url = f"{self.rule['es_hosts']}/so-playbook-alerts-{today}/_doc/"
             requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds)
                             
     def get_info(self):
-        return {'type': 'PlaybookESAlerter'} 
+        return {'type': 'PlaybookESAlerter'} 

salt/elastalert/map.jinja

@@ -8,7 +8,7 @@
 {% set elastalert_pillar = salt['pillar.get']('elastalert:config', {}) %}
 
 
-{% do ELASTALERTDEFAULTS.elastalert.config.update({'es_host': GLOBALS.manager}) %}
+{% do ELASTALERTDEFAULTS.elastalert.config.update({'es_hosts': 'https://' + GLOBALS.manager + ':' + ELASTALERTDEFAULTS.elastalert.config.es_port|string}) %}
 {% do ELASTALERTDEFAULTS.elastalert.config.update({'es_username': pillar.elasticsearch.auth.users.so_elastic_user.user}) %}
 {% do ELASTALERTDEFAULTS.elastalert.config.update({'es_password': pillar.elasticsearch.auth.users.so_elastic_user.pass}) %}
 

salt/elasticfleet/files/integrations/grid-nodes/zeek-logs.json

@@ -20,7 +20,7 @@
             "data_stream.dataset": "zeek",
             "tags": [],
             "processors": "- dissect:\n    tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n    field: \"log.file.path\"\n    trim_chars: \".log\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"pipeline\");\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"",
-            "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|loaded_scripts|packet_filter|stats|stderr|stdout.log$\"]\n"
+            "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|known_hosts|known_services|loaded_scripts|ntp|packet_filter|reporter|stats|stderr|stdout.log$\"]\n"
           }
         }
       }

salt/elasticsearch/ca.sls

@@ -26,7 +26,6 @@ catrustscript:
         GLOBALS: {{ GLOBALS }}
 {%   endif %}
 
-{%   if grains.role in ['so-manager', 'so-helix', 'so-managersearch', 'so-standalone', 'so-import', 'so-searchnode'] %}
 cacertz:
   file.managed:
     - name: /opt/so/conf/ca/cacerts
@@ -40,7 +39,6 @@ capemz:
     - source: salt://elasticsearch/tls-ca-bundle.pem
     - user: 939
     - group: 939
-{%   endif %}
 
 {% else %}
 

salt/elasticsearch/files/ingest/zeek.dns

@@ -25,7 +25,7 @@
     { "rename": 	{ "field": "message2.rejected", 	"target_field": "dns.query.rejected",		"ignore_missing": true 	} },
     { "script": 	{ "lang": "painless", "source": "ctx.dns.query.length = ctx.dns.query.name.length()",	"ignore_failure": true  } },
     { "set":      { "if": "ctx._index == 'so-zeek'", "field": "_index",      "value": "so-zeek_dns", "override": true   } },
-    { "pipeline": { "if": "ctx.dns.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld" } },
+    { "pipeline": { "if": "ctx.dns?.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld" } },
     { "pipeline":       { "name": "zeek.common"											} }
   ]
 }

salt/kibana/files/config_saved_objects.ndjson

@@ -1 +1 @@
-{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion": "8.7.0","id": "8.7.0","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="}
+{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n  \"from\": \"now-24h\",\n  \"to\": \"now\"\n}"},"coreMigrationVersion": "8.7.1","id": "8.7.1","migrationVersion": {"config": "7.13.0"},"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="}

salt/kibana/tools/sbin_jinja/so-kibana-config-load

@@ -63,7 +63,7 @@ update() {
 
     IFS=$'\r\n' GLOBIGNORE='*' command eval  'LINES=($(cat $1))'
     for i in "${LINES[@]}"; do
-      RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.7.0" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ")
+      RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.7.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ")
       echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi
     done
  

salt/manager/tools/sbin/so-minion

@@ -266,7 +266,7 @@ function add_redis_to_minion() {
 	"  " >> $PILLARFILE
 }
 
-function add_strelka_strelka_to_minion() {
+function add_strelka_to_minion() {
     printf '%s\n'\
     "strelka:"\
 	"  backend:"\
@@ -407,7 +407,8 @@ function apply_ES_state() {
 function createEVAL() {
 	add_elasticsearch_to_minion
 	add_sensor_to_minion
-	add_strelka_strelka_to_minion
+	add_strelka_to_minion
+	add_playbook_to_minion
 	add_elastalert_to_minion
 	add_kibana_to_minion
 	add_curator_to_minion
@@ -427,7 +428,7 @@ function createSTANDALONE() {
 	add_elasticsearch_to_minion
 	add_logstash_to_minion
 	add_sensor_to_minion
-	add_strelka_strelka_to_minion
+	add_strelka_to_minion
 	add_playbook_to_minion
 	add_elastalert_to_minion
 	add_kibana_to_minion
@@ -518,7 +519,7 @@ function createHEAVYNODE() {
 	add_elasticsearch_to_minion
 	add_logstash_to_minion
 	add_sensor_to_minion
-	add_strelka_strelka_to_minion
+	add_strelka_to_minion
 	add_redis_to_minion
 	add_curator_to_minion
 	add_telegraf_to_minion
@@ -526,7 +527,7 @@ function createHEAVYNODE() {
 
 function createSENSOR() {
 	add_sensor_to_minion
-	add_strelka_strelka_to_minion
+	add_strelka_to_minion
 	add_telegraf_to_minion
 }
 

salt/sensoroni/files/analyzers/whoislookup/requirements.txt

@@ -1 +1,2 @@
+requests==2.27.1
 whoisit>=2.5.3

salt/soc/files/bin/salt-relay.sh

@@ -70,7 +70,7 @@ function manage_user() {
         lastName=$(echo "$request" | jq -r .lastName)
         note=$(echo "$request" | jq -r .note)
         log "Performing user '$op' for user '$email' with firstname '$firstName', lastname '$lastName', note '$note' and role '$role'"
-        response=$(echo "$password" | so-user "$op" --email "$email" --firstName "$firstName" --lastName "$lastName" --note "$note" --role "$role" --skip-sync)
+        response=$(echo "$password" | $CMD_PREFIX so-user "$op" --email "$email" --firstName "$firstName" --lastName "$lastName" --note "$note" --role "$role" --skip-sync)
         exit_code=$?
         ;;
       add|enable|disable|delete)
@@ -143,12 +143,12 @@ function manage_salt() {
     state)
       log "Performing '$op' for '$state' on minion '$minion'"
       state=$(echo "$request" | jq -r .state)
-      response=$($CMD_PREFIX salt --async "$minion" state.apply "$state" queue=True)
+      response=$($CMD_PREFIX salt --async "$minion" state.apply "$state" queue=2)
       exit_code=$?
       ;;
     highstate)
       log "Performing '$op' on minion $minion"
-      response=$($CMD_PREFIX salt --async "$minion" state.highstate queue=True)
+      response=$($CMD_PREFIX salt --async "$minion" state.highstate queue=2)
       exit_code=$?
       ;;
     activejobs)

setup/so-functions

@@ -399,6 +399,11 @@ collect_mngr_hostname() {
 		done
 	else
 		MSRVIP=$(getent hosts "$MSRV" | awk 'NR==1{print $1}')
+		whiptail_manager_ip "$MSRVIP"
+		while ! valid_ip4 "$MSRVIP" || [[ $MSRVIP == "$MAINIP" || $MSRVIP == "127.0.0.1" ]]; do
+			whiptail_invalid_input
+			whiptail_manager_ip "$MSRVIP"
+		done
 	fi
 }