merge with 120 dev and fix conflicts

2025-12-15 21:52:47 +01:00 · 2025-01-23 10:56:48 -05:00
parent 9db3cd901c ca0c1170ab
commit 213df68d04
92 changed files with 3711 additions and 528531 deletions
--- a/salt/manager/tools/sbin/so-client
+++ b/salt/manager/tools/sbin/so-client
@@ -0,0 +1,411 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+if [[ -f /usr/sbin/so-common ]]; then
+  source /usr/sbin/so-common
+else
+  source $(dirname $0)/../../../common/tools/sbin/so-common
+fi
+
+function usage() {
+  cat <<USAGE_EOF
+  Usage: $0 <operation> [supporting parameters]
+  
+  where <operation> is one of the following:
+  
+      list: Lists all client IDs and permissions currently defined in the oauth2 system
+  
+       add: Adds a new client to the oauth2 system and outputs the generated secret
+            Required parameters: 
+              --name <name>
+            Optional parameters: 
+              --note <note>             (defaults to blank)
+              --json                    output as JSON
+
+    delete: Deletes a client from the oauth2 system
+            Required parameters: 
+              --id <id>
+  
+   addperm: Grants a permission to an existing client
+            Required parameters: 
+              --id <id>
+              --permission <permission>
+  
+   delperm: Removes a permission from an existing client
+            Required parameters: 
+              --id <id>
+              --permission <permission>
+
+    update: Updates a client name and note.
+            Required parameters:
+              --id <id>
+              --name <name>
+              --note <note>
+              --searchusername <run-as username>
+  
+  generate-secret: Regenerates a client's secret and outputs the new secret. 
+            Required parameters: 
+              --id <id>
+            Optional parameters: 
+              --json                     output as JSON
+
+USAGE_EOF
+  exit 1
+}
+
+if [[ $# -lt 1 || $1 == --help || $1 == -h || $1 == -? || $1 == --h ]]; then
+  usage
+fi
+
+operation=$1
+shift
+
+searchUsername=__MISSING__
+note=__MISSING__
+
+while [[ $# -gt 0 ]]; do
+  param=$1
+  shift
+  case "$param" in
+    --id)
+      id=$(echo $1 | sed 's/"/\\"/g')
+      [[ ${#id} -gt 55 ]] && fail "id cannot be longer than 55 characters"
+      shift
+      ;;
+    --permission)
+      perm=$(echo $1 | sed 's/"/\\"/g')
+      [[ ${#perm} -gt 50 ]] && fail "permission cannot be longer than 50 characters"
+      shift
+      ;;
+    --name)
+      name=$(echo $1 | sed 's/"/\\"/g')
+      [[ ${#name} -gt 50 ]] && fail "name cannot be longer than 50 characters"
+      shift
+      ;;
+    --note) 
+      note=$(echo $1 | sed 's/"/\\"/g')
+      [[ ${#note} -gt 100 ]] && fail "note cannot be longer than 100 characters"
+      shift
+      ;;
+    --searchusername) 
+      searchUsername=$(echo $1 | sed 's/"/\\"/g')
+      [[ ${#searchUsername} -gt 50 ]] && fail "search username cannot be longer than 50 characters"
+      shift
+      ;;
+    --json) 
+      json=1
+      ;;
+    *) 
+      echo "Encountered unexpected parameter: $param"
+      usage
+      ;;
+  esac
+done
+
+hydraUrl=${HYDRA_URL:-http://127.0.0.1:4445}
+socRolesFile=${SOC_ROLES_FILE:-/opt/so/conf/soc/soc_clients_roles}
+soUID=${SOCORE_UID:-939}
+soGID=${SOCORE_GID:-939}
+
+function lock() {
+  # Obtain file descriptor lock
+  exec 99>/var/tmp/so-client.lock || fail "Unable to create lock descriptor; if the system was not shutdown gracefully you may need to remove /var/tmp/so-client.lock manually."
+  flock -w 10 99 || fail "Another process is using so-client; if the system was not shutdown gracefully you may need to remove /var/tmp/so-client.lock manually."
+  trap 'rm -f /var/tmp/so-client.lock' EXIT
+}
+
+function fail() {
+  msg=$1
+  echo "$1"
+  exit 1
+}
+
+function require() {
+  cmd=$1
+  which "$1" 2>&1 > /dev/null
+  [[ $? != 0 ]] && fail "This script requires the following command be installed: ${cmd}"
+}
+
+# Verify this environment is capable of running this script
+function verifyEnvironment() {
+  require "jq"
+  require "curl"
+  response=$(curl -Ss -L ${hydraUrl}/)
+  [[ "$response" != *"Error 404"* ]] && fail "Unable to communicate with Hydra; specify URL via HYDRA_URL environment variable"
+}
+
+function createFile() {
+  filename=$1
+  uid=$2
+  gid=$3
+
+  mkdir -p $(dirname "$filename")
+  truncate -s 0 "$filename"
+  chmod 600 "$filename"
+  chown "${uid}:${gid}" "$filename"
+}
+
+function ensureRoleFileExists() {
+  if [[ ! -f "$socRolesFile" ]]; then
+    # Generate the new roles file
+    rolesTmpFile="${socRolesFile}.tmp"
+    createFile "$rolesTmpFile" "$soUID" "$soGID"
+
+    if [[ -d "$socRolesFile" ]]; then
+      echo "Removing invalid roles directory created by Docker"
+      rm -fr "$socRolesFile"
+    fi
+    mv "${rolesTmpFile}" "${socRolesFile}"
+  fi
+}
+
+function listClients() {
+  response=$(curl -Ss -L -f ${hydraUrl}/admin/clients)
+  [[ $? != 0 ]] && fail "Unable to communicate with Hydra"
+
+  clientIds=$(echo "${response}" | jq -r ".[] | .client_id" | sort)
+  for clientId in $clientIds; do
+    perms=$(grep ":$clientId\$" "$socRolesFile" | cut -d: -f1 | tr '\n' ' ')
+    echo "$clientId: $perms"
+  done
+}
+
+function addClientPermission() {
+  id=$1
+  perm=$2
+
+  adjustClientPermission "$id" "$perm" "add"
+}
+
+function deleteClientPermission() {
+  id=$1
+  perm=$2
+
+  adjustClientPermission "$id" "$perm" "del"
+}
+
+function adjustClientPermission() {
+  identityId=$1
+  perm=$2
+  op=$3
+
+  [[ ${identityId} == "" ]] && fail "Client not found"
+
+  ensureRoleFileExists
+
+  filename="$socRolesFile"
+  hasPerm=0
+  grep "^$perm:" "$socRolesFile" | grep -q "$identityId" && hasPerm=1
+  if [[ "$op" == "add" ]]; then
+    if [[ "$hasPerm" == "1" ]]; then
+      echo "Client '$identityId' already has the permission: $perm"
+      return 1
+    else
+      echo "$perm:$identityId" >> "$filename"
+    fi
+  elif [[ "$op" == "del" ]]; then
+    if [[ "$hasPerm" -ne 1 ]]; then
+      fail "Client '$identityId' does not have the permission: $perm"
+    else
+      sed -e "\!^$perm:$identityId\$!d" "$filename" > "$filename.tmp"
+      cat "$filename".tmp > "$filename"
+      rm -f "$filename".tmp
+    fi
+  else
+    fail "Unsupported permission adjustment operation: $op"
+  fi
+  return 0
+}
+
+function convertNameToId() {
+    name=$1
+
+    name=${name//[^[:alnum:]]/_}
+    echo "socl_$name" | tr '[:upper:]' '[:lower:]'
+}
+
+function createClient() {
+  name=$1
+  note=$2
+
+  id=$(convertNameToId "$name")
+  now=$(date -u +%FT%TZ)
+  secret=$(get_random_value)
+  body=$(cat <<EOF
+{
+  "access_token_strategy": "opaque",
+  "client_id": "$id",
+  "client_secret": "$secret",
+  "client_name": "$name",
+  "grant_types": [ "client_credentials" ],
+  "response_types": [ "code" ],
+  "metadata": {
+    "note": "$note",
+    "searchUsername": ""
+  }
+}
+EOF
+  )
+  
+  response=$(curl -Ss -L --fail-with-body -X POST ${hydraUrl}/admin/clients -d "$body")
+  if [[ $? != 0 ]]; then
+    error=$(echo $response | jq .error)
+    fail "Failed to submit request to Hydra: $error"
+  fi
+}
+
+function update() {
+  clientId=$1
+  name=$2
+  note=$3
+  username=$4
+  
+  body=$(cat <<EOF
+[
+    {
+        "op": "replace",
+        "path": "/client_name",
+        "value": "$name"
+    },
+    {
+        "op": "replace",
+        "path": "/metadata",
+        "value": {
+          "note": "$note",
+          "searchUsername": "$username"
+        }
+    }
+]
+EOF
+  )
+  
+  response=$(curl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
+  if [[ $? != 0 ]]; then
+    error=$(echo $response | jq .error)
+    fail "Failed to submit request to Hydra: $error"
+  fi
+}
+
+function generateSecret() {
+  clientId=$1
+  
+  secret=$(get_random_value)
+  body=$(cat <<EOF
+[
+    {
+        "op": "replace",
+        "path": "/client_secret",
+        "value": "$secret"
+    }
+]
+EOF
+  )
+  
+  response=$(curl -Ss -L --fail-with-body -X PATCH ${hydraUrl}/admin/clients/$id -d "$body")
+  if [[ $? != 0 ]]; then
+    error=$(echo $response | jq .error)
+    fail "Failed to submit request to Hydra: $error"
+  fi
+}
+
+function deleteClient() {
+  identityId=$1
+
+  [[ ${identityId} == "" ]] && fail "Client not found"
+
+  response=$(curl -Ss -XDELETE -L --fail-with-body "${hydraUrl}/admin/clients/$identityId")
+  if [[ $? != 0 ]]; then
+    error=$(echo $response | jq .error)
+    fail "Failed to submit request to Hydra: $error"
+  fi
+
+  rolesTmpFile="${socRolesFile}.tmp"
+  createFile "$rolesTmpFile" "$soUID" "$soGID"
+  grep -v "$identityId" "$socRolesFile" > "$rolesTmpFile"
+  cat "$rolesTmpFile" > "$socRolesFile"
+}
+
+case "${operation}" in
+  "add")
+    verifyEnvironment
+    [[ "$name" == "" ]] && fail "A short client name must be provided"
+
+    lock
+    createClient "$name" "$note"
+    if [[ "$json" == "1" ]]; then
+      echo "{\"id\":\"$id\",\"secret\":\"$secret\"}"
+    else
+      echo "Successfully added client ID $id with generated secret: $secret"
+    fi
+    ;;
+
+  "list")
+    verifyEnvironment
+    listClients
+    ;;
+
+  "addperm")
+    verifyEnvironment
+    [[ "$id" == "" ]] && fail "Id must be provided"
+    [[ "$perm" == "" ]] && fail "Permission must be provided"
+
+    lock
+    if addClientPermission "$id" "$perm"; then
+      echo "Successfully added permission to client"
+    fi
+    ;;
+
+  "delperm")
+    verifyEnvironment
+    [[ "$id" == "" ]] && fail "Id must be provided"
+    [[ "$perm" == "" ]] && fail "Permission must be provided"
+
+    lock
+    deleteClientPermission "$id" "$perm"
+    echo "Successfully removed permission from client"
+    ;;
+
+  "update")
+    verifyEnvironment
+    [[ "$id" == "" ]] && fail "Id must be provided"
+    [[ "$name" == "" ]] && fail "Name must be provided"
+    [[ "$note" == "__MISSING__" ]] && fail "Note must be provided"
+    [[ "$searchUsername" == "__MISSING__" ]] && fail "Search Username must be provided"
+
+    lock
+    update "$id" "$name" "$note" "$searchUsername"
+    echo "Successfully updated client"
+    ;;
+
+  "generate-secret")
+    verifyEnvironment
+    [[ "$id" == "" ]] && fail "Id must be provided"
+
+    lock
+    generateSecret "$id"
+    if [[ "$json" == "1" ]]; then
+      echo "{\"secret\":\"$secret\"}"
+    else
+      echo "Successfully generated secret: $secret"
+    fi
+    ;;
+
+  "delete")
+    verifyEnvironment
+    [[ "$id" == "" ]] && fail "Id must be provided"
+
+    lock
+    deleteClient "$id"
+    echo "Successfully deleted client."
+    ;;
+  *)
+    fail "Unsupported operation: $operation"
+    usage
+    ;;
+esac
+
+exit 0
--- a/salt/manager/tools/sbin/so-user
+++ b/salt/manager/tools/sbin/so-user
@@ -100,23 +100,23 @@ while [[ $# -gt 0 ]]; do
  shift
  case "$param" in
    --email)
-      email=$1
+      email=$(echo $1 | sed 's/"/\\"/g')
      shift
      ;;
    --role)
-      role=$1
+      role=$(echo $1 | sed 's/"/\\"/g')
      shift
      ;;
    --firstName)
-      firstName=$1
+      firstName=$(echo $1 | sed 's/"/\\"/g')
      shift
      ;;
    --lastName) 
-      lastName=$1
+      lastName=$(echo $1 | sed 's/"/\\"/g')
      shift
      ;;
    --note) 
-      note=$1
+      note=$(echo $1 | sed 's/"/\\"/g')
      shift
      ;;
    --skip-sync) 
@@ -241,6 +241,10 @@ function updatePassword() {
    [[ $? != 0 ]] && fail "Unable to clear aal2 identity IDs"
    echo "delete from identity_credentials where identity_id='${identityId}' and identity_credential_type_id in (select id from identity_credential_types where name in ('totp', 'webauthn', 'oidc'));" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath"
    [[ $? != 0 ]] && fail "Unable to clear aal2 identity credentials"
+    echo "delete from session_devices where session_id in (select id from sessions where identity_id='${identityId}');" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath"
+    [[ $? != 0 ]] && fail "Unable to clear session devices"
+    echo "delete from sessions where identity_id='${identityId}';" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath"
+    [[ $? != 0 ]] && fail "Unable to clear sessions"
    echo "update identities set available_aal='aal1' where id='${identityId}';" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath"
    [[ $? != 0 ]] && fail "Unable to reset aal"
  fi
@@ -357,7 +361,6 @@ function syncElastic() {
      random_crypt=$(get_random_value 53)
      user_data_formatted=$(echo "${user_data_formatted}" | sed -r "s/^(.+:)\$/\\1\$2a\$12${random_crypt}/")
    fi
-
    echo "${user_data_formatted}" >> "$usersTmpFile"

    # Append the user roles
@@ -373,7 +376,6 @@ function syncElastic() {
        sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath" >> "$rolesTmpFile"
      [[ $? != 0 ]] && fail "Unable to read role identities from database"
    done < "$socRolesFile"
-
  else
    echo "Database file or soc roles file does not exist yet, skipping users export"
  fi
--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -404,7 +404,8 @@ preupgrade_changes() {
    [[ "$INSTALLEDVERSION" == 2.4.80 ]] && up_to_2.4.90
    [[ "$INSTALLEDVERSION" == 2.4.90 ]] && up_to_2.4.100
    [[ "$INSTALLEDVERSION" == 2.4.100 ]] && up_to_2.4.110
-    [[ "$INSTALLEDVERSION" == 2.4.110 ]] && up_to_2.4.120
+    [[ "$INSTALLEDVERSION" == 2.4.110 ]] && up_to_2.4.111
+    [[ "$INSTALLEDVERSION" == 2.4.111 ]] && up_to_2.4.120
    true
 }

@@ -426,7 +427,8 @@ postupgrade_changes() {
    [[ "$POSTVERSION"  == 2.4.80 ]] && post_to_2.4.90
    [[ "$POSTVERSION"  == 2.4.90 ]] && post_to_2.4.100
    [[ "$POSTVERSION"  == 2.4.100 ]] && post_to_2.4.110
-    [[ "$POSTVERSION"  == 2.4.110 ]] && post_to_2.4.120
+    [[ "$POSTVERSION" == 2.4.110 ]] && post_to_2.4.111
+    [[ "$POSTVERSION"  == 2.4.111 ]] && post_to_2.4.120
    true
 }

@@ -519,8 +521,17 @@ post_to_2.4.110() {
  POSTVERSION=2.4.110
 }

-post_to_2.4.120() {
+post_to_2.4.111() {
  echo "Nothing to apply"
+  POSTVERSION=2.4.111
+}
+
+post_to_2.4.120() {
+  update_elasticsearch_index_settings
+
+  # Manually rollover suricata alerts index to ensure data_stream.dataset expected mapping is set to 'suricata'
+  rollover_index "logs-suricata.alerts-so"
+
  POSTVERSION=2.4.120
 }

@@ -714,9 +725,17 @@ up_to_2.4.110() {
  INSTALLEDVERSION=2.4.110
 }

+up_to_2.4.111() {
+  echo "Nothing to do for 2.4.111"
+
+  INSTALLEDVERSION=2.4.111
+}
+
 up_to_2.4.120() {
+  add_hydra_pillars
+
  # this is needed for the new versionlock state
-  mkdir /opt/so/saltstack/local/pillar/versionlock
+  mkdir -p /opt/so/saltstack/local/pillar/versionlock
  touch /opt/so/saltstack/local/pillar/versionlock/adv_versionlock.sls /opt/so/saltstack/local/pillar/versionlock/soc_versionlock.sls

  # New Grid Integration added this release
@@ -725,6 +744,26 @@ up_to_2.4.120() {
  INSTALLEDVERSION=2.4.120
 }

+add_hydra_pillars() {
+  mkdir -p /opt/so/saltstack/local/pillar/hydra
+  touch /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls
+  chmod 660 /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls
+  touch /opt/so/saltstack/local/pillar/hydra/adv_hydra.sls
+  HYDRAKEY=$(get_random_value)
+  HYDRASALT=$(get_random_value)
+	printf '%s\n'\
+		"hydra:"\
+		"  config:"\
+		"    secrets:"\
+		"      system:"\
+		"        - '$HYDRAKEY'"\
+    "    oidc:"\
+    "        subject_identifiers:"\
+    "            pairwise:"\
+    "                salt: '$HYDRASALT'"\
+		"" > /opt/so/saltstack/local/pillar/hydra/soc_hydra.sls
+}
+
 add_detection_test_pillars() {
  if [[ -n "$SOUP_INTERNAL_TESTING" ]]; then
    echo "Adding detection pillar values for automated testing"
@@ -771,6 +810,22 @@ ASSIST_EOF
  fi
 }

+rollover_index() {
+  idx=$1
+  exists=$(so-elasticsearch-query $idx -o /dev/null -w "%{http_code}")
+  if [[ $exists -eq 200 ]]; then
+    rollover=$(so-elasticsearch-query $idx/_rollover -o /dev/null -w "%{http_code}" -XPOST)
+
+    if [[ $rollover -eq 200 ]]; then
+      echo "Successfully triggered rollover for $idx..."
+    else
+      echo "Could not trigger rollover for $idx..."
+    fi
+  else
+    echo "Could not find index $idx..."
+  fi
+}
+
 suricata_idstools_migration() {
  #Backup the pillars for idstools
  mkdir -p /nsm/backup/detections-migration/idstools
@@ -922,7 +977,7 @@ update_airgap_rules() {
  rsync -av $UPDATE_DIR/agrules/detect-sigma/* /nsm/rules/detect-sigma/
  rsync -av $UPDATE_DIR/agrules/detect-yara/* /nsm/rules/detect-yara/
  # Copy the securityonion-resorces repo over for SOC Detection Summaries and checkout the published summaries branch
-  rsync -av --chown=socore:socore $UPDATE_DIR/agrules/securityonion-resources /opt/so/conf/soc/ai_summary_repos
+  rsync -av --delete --chown=socore:socore $UPDATE_DIR/agrules/securityonion-resources /opt/so/conf/soc/ai_summary_repos
  git config --global --add safe.directory /opt/so/conf/soc/ai_summary_repos/securityonion-resources
  git -C /opt/so/conf/soc/ai_summary_repos/securityonion-resources checkout generated-summaries-published
  # Copy the securityonion-resorces repo over to nsm
@@ -938,6 +993,30 @@ update_airgap_repo() {
  createrepo /nsm/repo
 }

+update_elasticsearch_index_settings() {
+  # Update managed indices to reflect latest index template
+  for idx in "so-detection" "so-detectionhistory" "so-case" "so-casehistory"; do
+    ilm_name=$idx
+    if [ "$idx" = "so-detectionhistory" ]; then
+      ilm_name="so-detection"
+    elif [ "$idx" = "so-casehistory" ]; then
+      ilm_name="so-case"
+    fi
+    JSON_STRING=$( jq -n --arg ILM_NAME "$ilm_name" '{"settings": {"index.auto_expand_replicas":"0-2","index.lifecycle.name":($ILM_NAME + "-logs")}}')
+
+    echo "Checking if index \"$idx\" exists"
+    exists=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -o /dev/null -w "%{http_code}" -k -L -H "Content-Type: application/json" "https://localhost:9200/$idx")
+    if [ $exists -eq 200 ]; then
+      echo "$idx index found..."
+      echo "Updating $idx index settings"
+      curl -K /opt/so/conf/elasticsearch/curl.config -s -k -L -H "Content-Type: application/json" "https://localhost:9200/$idx/_settings" -d "$JSON_STRING" -XPUT
+      echo -e "\n"
+    else
+      echo -e "Skipping $idx... index does not exist\n"
+    fi
+  done
+}
+
 update_salt_mine() {
    echo "Populating the mine with mine_functions for each host."
    set +e
@@ -1003,12 +1082,12 @@ upgrade_salt() {
    # if oracle run with -r to ignore repos set by bootstrap
    if [[ $OS == 'oracle' ]]; then
      run_check_net_err \
-      "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \
+      "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -r -F -M stable \"$NEWSALTVERSION\"" \
      "Could not update salt, please check $SOUP_LOG for details."
    # if another rhel family variant we want to run without -r to allow the bootstrap script to manage repos
    else
      run_check_net_err \
-      "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -F -M -x python3 stable \"$NEWSALTVERSION\"" \
+      "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -F -M stable \"$NEWSALTVERSION\"" \
      "Could not update salt, please check $SOUP_LOG for details."
    fi
    set -e
@@ -1028,7 +1107,7 @@ upgrade_salt() {
    echo ""
    set +e
    run_check_net_err \
-    "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -F -M -x python3 stable \"$NEWSALTVERSION\"" \
+    "sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -F -M stable \"$NEWSALTVERSION\"" \
    "Could not update salt, please check $SOUP_LOG for details."
    set -e
    echo "Applying apt hold for Salt."
@@ -1334,6 +1413,10 @@ main() {

    echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
    (wait_for_salt_minion "$MINIONID" "5" '/dev/stdout' || fail "Salt minion was not running or ready.") 2>&1 | tee -a "$SOUP_LOG"
+
+    # Stop long-running scripts to allow potentially updated scripts to load on the next execution.
+    killall salt-relay.sh
+
    highstate
    postupgrade_changes
    [[ $is_airgap -eq 0 ]] && unmount_update