merge with 2.4.120, fix merge conflicts

2025-12-18 23:13:20 +01:00 · 2024-10-25 15:09:38 -04:00
parent 0476585370 a76a2d8e9f
commit feb700393e
520 changed files with 12109 additions and 250 deletions
--- a/salt/manager/init.sls
+++ b/salt/manager/init.sls
@@ -45,6 +45,12 @@ yara_log_dir:
      - user
      - group

+{% if GLOBALS.os_family == 'RedHat' %}
+install_createrepo:
+  pkg.installed:
+    - name: createrepo_c
+{% endif %}
+
 repo_conf_dir:
  file.directory:
    - name: /opt/so/conf/reposync
--- a/salt/manager/soc_manager.yaml
+++ b/salt/manager/soc_manager.yaml
@@ -1,7 +1,7 @@
 manager:
  reposync:
    enabled:
-      description: This is the daily task of syncing the Security Onion OS packages. It is recommended that you leave this enabled.
+      description: This is the daily task of syncing the Security Onion OS packages. It is recommended that this setting remain enabled to ensure important updates are applied to the grid on an automated, scheduled basis.
      global: True
      helpLink: soup.html
    hour:
--- a/salt/manager/tools/sbin/so-minion
+++ b/salt/manager/tools/sbin/so-minion
@@ -9,6 +9,10 @@ if [ -f /usr/sbin/so-common ]; then
 	. /usr/sbin/so-common
 fi

+if [ -f /usr/sbin/so-elastic-fleet-common ]; then
+	. /usr/sbin/so-elastic-fleet-common
+fi
+
 function usage() {
 	echo "Usage: $0 -o=<operation> -m=[id]"
 	echo ""
@@ -388,23 +392,31 @@ function add_elastic_fleet_package_registry_to_minion() {

 function create_fleet_policy() {

-	JSON_STRING=$( jq -n \
-					--arg NAME "FleetServer_$LSHOSTNAME" \
-					--arg DESC "Fleet Server - $LSHOSTNAME" \
-					'{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":1209600,"has_fleet_server":true}'
-					)
+	# First, set the default output to Elasticsearch
+	# This is required because of the license output bug
+	JSON_STRING=$(jq -n \
+	'{ 
+		"name": "so-manager_elasticsearch", 
+		"type": "elasticsearch",
+		"is_default": true, 
+		"is_default_monitoring": false
+	}')

-	# Create Fleet Sever Policy
-	curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_elasticsearch" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"

-	JSON_STRING_UPDATE=$( jq -n \
-					--arg NAME "FleetServer_$LSHOSTNAME" \
-					--arg DESC "Fleet Server - $LSHOSTNAME" \
-					'{"name":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":120,"data_output_id":"so-manager_elasticsearch"}'
-					)
+	# Create the Fleet Server Policy
+	elastic_fleet_policy_create "FleetServer_$LSHOSTNAME" "Fleet Server - $LSHOSTNAME" "false" "120"

-	# Update Fleet Policy - ES Output
-	curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/FleetServer_$LSHOSTNAME" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING_UPDATE"
+    # Modify the default integration policy to update the policy_id with the correct naming
+	UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "FleetServer_$LSHOSTNAME" --arg name "fleet_server-$LSHOSTNAME" '
+    .policy_id = $policy_id |
+    .name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json)
+
+	# Add the Fleet Server Integration to the new Fleet Policy
+	elastic_fleet_integration_create "$UPDATED_INTEGRATION_POLICY"
+
+	# Set the default output back to the default
+	/sbin/so-elastic-fleet-outputs-update
 }

 function update_fleet_host_urls() {
--- a/salt/manager/tools/sbin/so-user
+++ b/salt/manager/tools/sbin/so-user
@@ -173,7 +173,7 @@ function verifyEnvironment() {
 }

 function findIdByEmail() {
-  email=$1
+  email=${1,,}

  response=$(curl -Ss -L ${kratosUrl}/identities)
  identityId=$(echo "${response}" | jq -r ".[] | select(.verifiable_addresses[0].value == \"$email\") | .id")
@@ -195,12 +195,13 @@ function validatePassword() {

 function validateEmail() {
  email=$1
+  requireLower=$2
  # (?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])
  if [[ ! "$email" =~ ^[[:alnum:]._%+-]+@[[:alnum:].-]+\.[[:alpha:]]{2,}$ ]]; then
    fail "Email address is invalid"
  fi

-  if [[ "$email" =~ [A-Z] ]]; then
+  if [[ "$requireLower" == "true" && "$email" =~ [A-Z] ]]; then
    fail "Email addresses cannot contain uppercase letters"
  fi
 }
@@ -234,10 +235,14 @@ function updatePassword() {
    passwordHash=$(hashPassword "$password")
    # Update DB with new hash
    echo "update identity_credentials set config=CAST('{\"hashed_password\":\"$passwordHash\"}' as BLOB), created_at=datetime('now'), updated_at=datetime('now') where identity_id='${identityId}' and identity_credential_type_id=(select id from identity_credential_types where name='password');" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath"
-    # Deactivate MFA
-    echo "delete from identity_credential_identifiers where identity_credential_id=(select id from identity_credentials where identity_id='${identityId}' and identity_credential_type_id=(select id from identity_credential_types where name in ('totp', 'webauthn', 'oidc')));" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath"
-    echo "delete from identity_credentials where identity_id='${identityId}' and identity_credential_type_id=(select id from identity_credential_types where name in ('totp', 'webauthn', 'oidc'));" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath"
    [[ $? != 0 ]] && fail "Unable to update password"
+    # Deactivate MFA
+    echo "delete from identity_credential_identifiers where identity_credential_id in (select id from identity_credentials where identity_id='${identityId}' and identity_credential_type_id in (select id from identity_credential_types where name in ('totp', 'webauthn', 'oidc')));" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath"
+    [[ $? != 0 ]] && fail "Unable to clear aal2 identity IDs"
+    echo "delete from identity_credentials where identity_id='${identityId}' and identity_credential_type_id in (select id from identity_credential_types where name in ('totp', 'webauthn', 'oidc'));" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath"
+    [[ $? != 0 ]] && fail "Unable to clear aal2 identity credentials"
+    echo "update identities set available_aal='aal1' where id='${identityId}';" | sqlite3 -cmd ".timeout ${databaseTimeout}" "$databasePath"
+    [[ $? != 0 ]] && fail "Unable to reset aal"
  fi
 }

@@ -577,7 +582,7 @@ case "${operation}" in
    [[ "$email" == "" ]] && fail "Email address must be provided"

    lock
-    validateEmail "$email"
+    validateEmail "$email" true
    updatePassword
    createUser "$email" "${role:-$DEFAULT_ROLE}" "${firstName}" "${lastName}" "${note}"
    syncAll
@@ -683,13 +688,13 @@ case "${operation}" in
    ;;

  "validate")
-    validateEmail "$email"
+    validateEmail "$email" true
    updatePassword
    echo "Email and password are acceptable"
    ;;

  "valemail")
-    validateEmail "$email"
+    validateEmail "$email" true
    echo "Email is acceptable"
    ;;

--- a/salt/manager/tools/sbin/soup
+++ b/salt/manager/tools/sbin/soup
@@ -19,6 +19,8 @@ SOUP_LOG=/root/soup.log
 WHATWOULDYOUSAYYAHDOHERE=soup
 whiptail_title='Security Onion UPdater'
 NOTIFYCUSTOMELASTICCONFIG=false
+TOPFILE=/opt/so/saltstack/default/salt/top.sls
+BACKUPTOPFILE=/opt/so/saltstack/default/salt/top.sls.backup
 # used to display messages to the user at the end of soup
 declare -a FINAL_MESSAGE_QUEUE=()

@@ -32,10 +34,7 @@ check_err() {
  if [[ $exit_code -ne 0 ]]; then

    set +e
-    systemctl_func "start" "$cron_service_name"
-    systemctl_func "start" "salt-master"
-    systemctl_func "start" "salt-minion"
-    enable_highstate
+    failed_soup_restore_items

    printf '%s' "Soup failed with error $exit_code: "
    case $exit_code in
@@ -347,8 +346,6 @@ highstate() {

 masterlock() {
  echo "Locking Salt Master"
-  TOPFILE=/opt/so/saltstack/default/salt/top.sls
-  BACKUPTOPFILE=/opt/so/saltstack/default/salt/top.sls.backup
  mv -v $TOPFILE $BACKUPTOPFILE
  echo "base:" > $TOPFILE
  echo "  $MINIONID:" >> $TOPFILE
@@ -358,8 +355,12 @@ masterlock() {
 }

 masterunlock() {
-  echo "Unlocking Salt Master"
-  mv -v $BACKUPTOPFILE $TOPFILE
+  if [ -f $BACKUPTOPFILE ]; then
+    echo "Unlocking Salt Master"
+    mv -v $BACKUPTOPFILE $TOPFILE
+  else
+    echo "Salt Master does not need unlocked."
+  fi
 }

 phases_pillar_2_4_80() {
@@ -402,6 +403,8 @@ preupgrade_changes() {
    [[ "$INSTALLEDVERSION" == 2.4.70 ]] && up_to_2.4.80
    [[ "$INSTALLEDVERSION" == 2.4.80 ]] && up_to_2.4.90
    [[ "$INSTALLEDVERSION" == 2.4.90 ]] && up_to_2.4.100
+    [[ "$INSTALLEDVERSION" == 2.4.100 ]] && up_to_2.4.110
+    [[ "$INSTALLEDVERSION" == 2.4.110 ]] && up_to_2.4.120
    true
 }

@@ -422,6 +425,8 @@ postupgrade_changes() {
    [[ "$POSTVERSION"  == 2.4.70 ]] && post_to_2.4.80
    [[ "$POSTVERSION"  == 2.4.80 ]] && post_to_2.4.90
    [[ "$POSTVERSION"  == 2.4.90 ]] && post_to_2.4.100
+    [[ "$POSTVERSION"  == 2.4.100 ]] && post_to_2.4.110
+    [[ "$POSTVERSION"  == 2.4.110 ]] && post_to_2.4.120
    true
 }

@@ -453,8 +458,6 @@ post_to_2.4.20() {
 }

 post_to_2.4.30() {
-  echo "Regenerating Elastic Agent Installers"
-  /sbin/so-elastic-agent-gen-installers
  # there is an occasional error with this state: pki_public_ca_crt: TypeError: list indices must be integers or slices, not str
  set +e
  salt-call state.apply ca queue=True
@@ -479,8 +482,7 @@ post_to_2.4.50() {
 }

 post_to_2.4.60() {
-  echo "Regenerating Elastic Agent Installers..."
-  so-elastic-agent-gen-installers
+  echo "Nothing to apply"
  POSTVERSION=2.4.60
 }

@@ -507,10 +509,21 @@ post_to_2.4.90() {
 }

 post_to_2.4.100() {
-  echo "Nothing to apply"
+  echo "Regenerating Elastic Agent Installers"
+  /sbin/so-elastic-agent-gen-installers
  POSTVERSION=2.4.100
 }

+post_to_2.4.110() {
+  echo "Nothing to apply"
+  POSTVERSION=2.4.110
+}
+
+post_to_2.4.120() {
+  echo "Nothing to apply"
+  POSTVERSION=2.4.120
+}
+
 repo_sync() {
  echo "Sync the local repo."
  su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
@@ -527,11 +540,17 @@ stop_salt_master() {
    pkill -9 -ef "/usr/bin/python3 /bin/salt" >> $SOUP_LOG 2>&1

    echo ""
-    echo "Storing salt-master pid."
+    echo "Storing salt-master PID."
    MASTERPID=$(pgrep -f '/opt/saltstack/salt/bin/python3.10 /usr/bin/salt-master MainProcess')
-    echo "Found salt-master PID $MASTERPID"
-    systemctl_func "stop" "salt-master"
-    timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option."
+    if [ ! -z "$MASTERPID" ]; then
+      echo "Found salt-master PID $MASTERPID"
+      systemctl_func "stop" "salt-master"
+      if ps -p "$MASTERPID" > /dev/null 2>&1; then
+        timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option."
+      fi
+    else
+      echo "The salt-master PID was not found. The process '/usr/bin/salt-master MainProcess' is not running."
+    fi
    set -e
 }

@@ -587,18 +606,7 @@ up_to_2.4.20() {
 }

 up_to_2.4.30() {
-
-  # Remove older defend integration json & installed integration
-  rm -f /opt/so/conf/elastic-fleet/integrations/endpoints-initial/elastic-defend-endpoints.json
-
-  . $UPDATE_DIR/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
-  elastic_fleet_integration_remove endpoints-initial elastic-defend-endpoints
-
-  rm -f /opt/so/state/eaintegrations.txt
-
-  # Elastic Update for this release, so download Elastic Agent files
-  determine_elastic_agent_upgrade
-  rm -f /opt/so/state/estemplates*.txt
+  echo "Nothing to do for 2.4.30"

  INSTALLEDVERSION=2.4.30
 }
@@ -688,16 +696,32 @@ up_to_2.4.90() {
  so-yaml.py remove /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password
  so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.password "$kafkatrimpass"
  so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.trustpass "$kafkatrust"
+  echo "If the Detection index exists, update the refresh_interval"
  so-elasticsearch-query so-detection*/_settings -X PUT -d '{"index":{"refresh_interval":"1s"}}'

  INSTALLEDVERSION=2.4.90
 }
+
 up_to_2.4.100() {
  # Elastic Update for this release, so download Elastic Agent files
  determine_elastic_agent_upgrade
  INSTALLEDVERSION=2.4.100
 }

+up_to_2.4.110() {
+  echo "Nothing to do for 2.4.110"
+
+  INSTALLEDVERSION=2.4.110
+}
+
+up_to_2.4.120() {
+  # this is needed for the new versionlock state
+  mkdir /opt/so/saltstack/local/pillar/versionlock
+  touch /opt/so/saltstack/local/pillar/versionlock/adv_versionlock.sls /opt/so/saltstack/local/pillar/versionlock/soc_versionlock.sls
+
+  INSTALLEDVERSION=2.4.120
+}
+
 add_detection_test_pillars() {
  if [[ -n "$SOUP_INTERNAL_TESTING" ]]; then
    echo "Adding detection pillar values for automated testing"
@@ -849,11 +873,15 @@ determine_elastic_agent_upgrade() {
  if [[ $is_airgap -eq 0 ]]; then
    update_elastic_agent_airgap
  else
-    update_elastic_agent
+    set +e
+    # the new elasticsearch defaults.yaml file is not yet placed in /opt/so/saltstack/default/salt/elasticsearch yet
+    update_elastic_agent "$UPDATE_DIR"
+    set -e
  fi
 }

 update_elastic_agent_airgap() {
+  get_elastic_agent_vars "/tmp/soagupdate/SecurityOnion"
  rsync -av /tmp/soagupdate/fleet/* /nsm/elastic-fleet/artifacts/
  tar -xf "$ELASTIC_AGENT_FILE" -C "$ELASTIC_AGENT_EXPANSION_DIR"
 }
@@ -890,6 +918,12 @@ update_airgap_rules() {
  rsync -av $UPDATE_DIR/agrules/suricata/* /nsm/rules/suricata/
  rsync -av $UPDATE_DIR/agrules/detect-sigma/* /nsm/rules/detect-sigma/
  rsync -av $UPDATE_DIR/agrules/detect-yara/* /nsm/rules/detect-yara/
+  # Copy the securityonion-resorces repo over for SOC Detection Summaries and checkout the published summaries branch
+  rsync -av --chown=socore:socore $UPDATE_DIR/agrules/securityonion-resources /opt/so/conf/soc/ai_summary_repos
+  git config --global --add safe.directory /opt/so/conf/soc/ai_summary_repos/securityonion-resources
+  git -C /opt/so/conf/soc/ai_summary_repos/securityonion-resources checkout generated-summaries-published
+  # Copy the securityonion-resorces repo over to nsm
+  rsync -av $UPDATE_DIR/agrules/securityonion-resources/* /nsm/securityonion-resources/
 }

 update_airgap_repo() {
@@ -897,7 +931,7 @@ update_airgap_repo() {
  echo "Syncing new updates to /nsm/repo"
  rsync -av $AGREPO/* /nsm/repo/
  echo "Creating repo"
-  dnf -y install yum-utils createrepo
+  dnf -y install yum-utils createrepo_c
  createrepo /nsm/repo
 }

@@ -957,7 +991,9 @@ upgrade_salt() {
  if [[ $is_rpm ]]; then
    echo "Removing yum versionlock for Salt."
    echo ""
-    yum versionlock delete "salt-*"
+    yum versionlock delete "salt"
+    yum versionlock delete "salt-minion"
+    yum versionlock delete "salt-master"
    echo "Updating Salt packages."
    echo ""
    set +e
@@ -975,7 +1011,9 @@ upgrade_salt() {
    set -e
    echo "Applying yum versionlock for Salt."
    echo ""
-    yum versionlock add "salt-*"
+    yum versionlock add "salt-0:$NEWSALTVERSION-0.*"
+    yum versionlock add "salt-minion-0:$NEWSALTVERSION-0.*"
+    yum versionlock add "salt-master-0:$NEWSALTVERSION-0.*"
  # Else do Ubuntu things
  elif [[ $is_deb ]]; then
    echo "Removing apt hold for Salt."
@@ -1071,6 +1109,16 @@ apply_hotfix() {
  fi
 }

+failed_soup_restore_items() {
+  local services=("$cron_service_name" "salt-master" "salt-minion")
+  for SERVICE_NAME in "${services[@]}"; do
+    if ! systemctl is-active --quiet "$SERVICE_NAME"; then
+      systemctl_func "start" "$SERVICE_NAME"
+    fi
+  done
+  enable_highstate
+  masterunlock
+}

 #upgrade salt to 3004.1
 #2_3_10_hotfix_1() {
@@ -1110,6 +1158,8 @@ main() {
  echo ""
  require_manager

+  failed_soup_restore_items
+
  check_pillar_items

  echo "Checking to see if this is an airgap install."
@@ -1416,6 +1466,8 @@ Please review the following for more information about the update process and re
 $DOC_BASE_URL/soup.html
 https://blog.securityonion.net

+WARNING: If you run soup via an SSH session and that SSH session terminates, then any processes running in that session would terminate. You should avoid leaving soup unattended especially if the machine you are SSHing from is configured to sleep after a period of time. You might also consider using something like screen or tmux so that if your SSH session terminates, the processes will continue running on the server.
+
 EOF

  cat << EOF