Merge remote-tracking branch 'remotes/origin/dev' into issue/3933

salt/common/tools/sbin/so-kibana-space-defaults

@@ -1,5 +1,5 @@
 . /usr/sbin/so-common
-
+{% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %}
 wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}"
 ## This hackery will be removed if using Elastic Auth ##
 
@@ -9,5 +9,9 @@ SESSIONCOOKIE=$({{ ELASTICCURL }} -c - -X GET http://localhost:5601/ | grep sid
 # Disable certain Features from showing up in the Kibana UI
 echo
 echo "Setting up default Space:"
+{% if HIGHLANDER %}
+{{ ELASTICCURL }} -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["enterpriseSearch"]} ' >> /opt/so/log/kibana/misc.log
+{% else %}
 {{ ELASTICCURL }} -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet"]} ' >> /opt/so/log/kibana/misc.log
+{% endif %}
 echo

salt/common/tools/sbin/so-user

@@ -101,6 +101,9 @@ function validatePassword() {
   if [[ $len -lt 6 ]]; then
     fail "Password does not meet the minimum requirements"
   fi
+  if [[ $len -gt 72 ]]; then
+    fail "Password is too long (max: 72)"
+  fi
   check_password_and_exit "$password"
 }
 
@@ -237,8 +240,12 @@ function syncElastic() {
   if [[ -f "$databasePath" && -f "$socRolesFile" ]]; then
     # Append the SOC users 
     echo "select '{\"user\":\"' || ici.identifier || '\", \"data\":' || ic.config || '}'" \
-      "from identity_credential_identifiers ici, identity_credentials ic " \
+      "from identity_credential_identifiers ici, identity_credentials ic, identities i " \
-      "where ici.identity_credential_id=ic.id and instr(ic.config, 'hashed_password') " \
+      "where " \
+      "      ici.identity_credential_id=ic.id " \
+      "  and ic.identity_id=i.id " \
+      "  and instr(ic.config, 'hashed_password') " \
+      "  and i.state == 'active' " \
       "order by ici.identifier;" | \
       sqlite3 "$databasePath" | \
       jq -r '.user + ":" + .data.hashed_password' \
@@ -381,6 +388,19 @@ EOF
   fi
 }
 
+function migrateLockedUsers() {
+  # This is a migration function to convert locked users from prior to 2.3.90
+  # to inactive users using the newer Kratos functionality. This should only
+  # find locked users once.
+  lockedEmails=$(curl -s http://localhost:4434/identities | jq -r '.[] | select(.traits.status == "locked") | .traits.email')
+  if [[ -n "$lockedEmails" ]]; then
+    echo "Disabling locked users..."
+    for email in $lockedEmails; do
+      updateStatus "$email" locked
+    done
+  fi
+}
+
 function updateStatus() {
   email=$1
   status=$2
@@ -391,24 +411,18 @@ function updateStatus() {
   response=$(curl -Ss -L "${kratosUrl}/identities/$identityId")
   [[ $? != 0 ]] && fail "Unable to communicate with Kratos"
 
-  oldConfig=$(echo "select config from identity_credentials where identity_id='${identityId}';" | sqlite3 "$databasePath")
+  schemaId=$(echo "$response" | jq -r .schema_id)
+
+  # Capture traits and remove obsolete 'status' trait if exists
+  traitBlock=$(echo "$response" | jq -c .traits | sed -re 's/,?"status":".*?"//')
+
+  state="active"
   if [[ "$status" == "locked" ]]; then
-    config=$(echo $oldConfig | sed -e 's/hashed/locked/')
+    state="inactive"
-    echo "update identity_credentials set config=CAST('${config}' as BLOB) where identity_id='${identityId}';" | sqlite3 "$databasePath"
+  fi
-    [[ $? != 0 ]] && fail "Unable to lock credential record"
+  body="{ \"schema_id\": \"$schemaId\", \"state\": \"$state\", \"traits\": $traitBlock }"
-
+  response=$(curl -fSsL -XPUT "${kratosUrl}/identities/$identityId" -d "$body")
-    echo "delete from sessions where identity_id='${identityId}';" | sqlite3 "$databasePath"
+  [[ $? != 0 ]] && fail "Unable to update user"
-    [[ $? != 0 ]] && fail "Unable to invalidate sessions"    
-  else
-    config=$(echo $oldConfig | sed -e 's/locked/hashed/')
-    echo "update identity_credentials set config=CAST('${config}' as BLOB) where identity_id='${identityId}';" | sqlite3 "$databasePath"
-    [[ $? != 0 ]] && fail "Unable to unlock credential record"
-  fi  
-
-  updatedJson=$(echo "$response" | jq ".traits.status = \"$status\" | del(.verifiable_addresses) | del(.id) | del(.schema_url) | del(.created_at) | del(.updated_at)")
-  response=$(curl -Ss -XPUT -L ${kratosUrl}/identities/$identityId -d "$updatedJson")
-  [[ $? != 0 ]] && fail "Unable to mark user as locked"
-
 }
 
 function updateUser() {
@@ -547,6 +561,11 @@ case "${operation}" in
     echo "Password is acceptable"
     ;;
 
+  "migrate")
+    migrateLockedUsers
+    echo "User migration complete"
+    ;;
+
   *)
     fail "Unsupported operation: $operation"
     ;;

salt/common/tools/sbin/soup

@@ -1037,6 +1037,9 @@ main() {
     echo "Checking sudoers file."
     check_sudoers
 
+    echo "Checking for necessary user migrations."
+    so-user migrate
+
     if [[ -n $lsl_msg ]]; then
       case $lsl_msg in
         'distributed')

salt/elasticsearch/config.map.jinja

@@ -1,4 +1,5 @@
 {% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %}
+{% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %}
 
 {% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %}
   {% do ESCONFIG.elasticsearch.config.xpack.security.authc.anonymous.update({'username': 'anonymous_user', 'roles': 'superuser', 'authz_exception': 'true'}) %}
@@ -8,6 +9,9 @@
     {% if grains.id.split('_') | last in ['manager','managersearch'] %}
     {% if salt['pillar.get']('nodestab', {}) %}
       {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %}
+      {% if HIGHLANDER %}
+          {% do ESCONFIG.elasticsearch.config.node.roles.append('ml') %}
+      {% endif %}
       {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [grains.master]}}) %}
       {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
         {% do ESCONFIG.elasticsearch.config.discovery.seed_hosts.append(SN.split('_')|first) %}
@@ -18,9 +22,15 @@
     {% endif %}
   {% else %}
     {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['data', 'ingest']}) %}
+    {% if HIGHLANDER %}
+          {% do ESCONFIG.elasticsearch.config.node.roles.extend(['ml', 'master']) %}
+    {% endif %}
     {% do ESCONFIG.elasticsearch.config.node.attr.update({'box_type': 'hot'}) %}
     {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [grains.master]}}) %}
   {% endif %}
+    {% if HIGHLANDER %}
+        {% do ESCONFIG.elasticsearch.config.xpack.ml.update({'enabled': true}) %}
+    {% endif %}
 {% endif %}
 
 {% set ESCONFIG = salt['pillar.get']('elasticsearch:config', default=ESCONFIG.elasticsearch.config, merge=True) %}

salt/soc/files/kratos/schema.json

@@ -12,6 +12,7 @@
           "format": "email",
           "title": "E-Mail",
           "minLength": 6,
+          "maxLength": 100,
           "ory.sh/kratos": {
             "credentials": {
               "password": {
@@ -25,16 +26,19 @@
         },
         "firstName": {
           "type": "string",
-          "title": "First Name"
+          "title": "First Name",
+          "maxLength": 100
         },
         "lastName": {
           "type": "string",
-          "title": "Last Name"
+          "title": "Last Name",
+          "maxLength": 100
         },
-        "status": {
+        "note": {
           "type": "string",
-          "title": "Status"
+          "title": "Note",
-        }      
+          "maxLength": 100
+        }
       },
       "required": [
         "email"

setup/so-functions

@@ -1668,6 +1668,10 @@ manager_global() {
 		"  url_base: '$REDIRECTIT'"\
 		"  managerip: '$MAINIP'" > "$global_pillar"
 	
+	if [[ $HIGHLANDER == 'True' ]]; then
+	    printf '%s\n'\
+		"  highlander: True"\ >> "$global_pillar"
+	fi
 	if [[ $is_airgap ]]; then
 		printf '%s\n'\
 			"  airgap: True"\ >> "$global_pillar"