From 1cb34b089cc6e678504e514a161c128d22dfbce3 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 20 Apr 2026 14:38:55 -0400 Subject: [PATCH] Restore 3/dev soup and add postgres users to post_to_3.1.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit feature/postgres had rewritten the 3.1.0 upgrade block, dropping the elastic upgrade work 3/dev landed for 9.0.8→9.3.3: elasticsearch_backup_index_templates, the component template state cleanup, and the /usr/sbin/so-kibana-space-defaults post-upgrade call. It also carried an older ES upgrade mapping (8.18.8→9.0.8) that was superseded on 3/dev (9.0.8→9.3.3 for 3.0.0-20260331), and a handful of latent shell-quoting regressions in verify_es_version_compatibility and the intermediate-upgrade helpers. Adopt the 3/dev soup verbatim and only add the new Telegraf Postgres provisioning to post_to_3.1.0 on top of so-kibana-space-defaults. --- salt/manager/tools/sbin/soup | 94 +++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 45 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 5ed66134f..c25358418 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -362,9 +362,8 @@ preupgrade_changes() { # This function is to add any new pillar items if needed. echo "Checking to see if changes are needed." - [[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0 - [[ "$INSTALLEDVERSION" == 3.0.0 ]] && up_to_3.1.0 - + [[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0 + [[ "$INSTALLEDVERSION" == "3.0.0" ]] && up_to_3.1.0 true } @@ -373,8 +372,7 @@ postupgrade_changes() { echo "Running post upgrade processes." [[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0 - [[ "$POSTVERSION" =~ 3.0.0 ]] && post_to_3.1.0 - + [[ "$POSTVERSION" == "3.0.0" ]] && post_to_3.1.0 true } @@ -385,7 +383,7 @@ check_minimum_version() { fi } -### 3.0.0 Start ### +### 3.0.0 Scripts ### convert_suricata_yes_no() { echo "Starting suricata yes/no values to true/false conversion." @@ -449,7 +447,6 @@ migrate_pcap_to_suricata() { } up_to_3.0.0() { - determine_elastic_agent_upgrade migrate_pcap_to_suricata INSTALLEDVERSION=3.0.0 @@ -473,13 +470,26 @@ post_to_3.0.0() { ### 3.0.0 End ### -### 3.1.0 Start ### +### 3.1.0 Scripts ### + +elasticsearch_backup_index_templates() { + echo "Backing up current elasticsearch index templates in /opt/so/conf/elasticsearch/templates/index/ to /nsm/backup/3.0.0_elasticsearch_index_templates.tar.gz" + tar -czf /nsm/backup/3.0.0_elasticsearch_index_templates.tar.gz -C /opt/so/conf/elasticsearch/templates/index/ . +} + up_to_3.1.0() { - echo "Nothing to do" + determine_elastic_agent_upgrade + elasticsearch_backup_index_templates + # Clear existing component template state file. + rm -f /opt/so/state/esfleet_component_templates.json + + INSTALLEDVERSION=3.1.0 } post_to_3.1.0() { + /usr/sbin/so-kibana-space-defaults + # Provision per-minion Telegraf Postgres users for every minion known to the # manager. postgres.auth iterates manage.up to generate any missing passwords; # postgres.telegraf_users reconciles the roles and schemas inside the so-postgres @@ -493,6 +503,7 @@ post_to_3.1.0() { ### 3.1.0 End ### + repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." @@ -752,12 +763,12 @@ verify_es_version_compatibility() { local is_active_intermediate_upgrade=1 # supported upgrade paths for SO-ES versions declare -A es_upgrade_map=( - ["8.18.8"]="9.0.8" + ["9.0.8"]="9.3.3" ) # Elasticsearch MUST upgrade through these versions declare -A es_to_so_version=( - ["8.18.8"]="2.4.190-20251024" + ["9.0.8"]="3.0.0-20260331" ) # Get current Elasticsearch version @@ -769,26 +780,17 @@ verify_es_version_compatibility() { exit 160 fi - if ! target_es_version_raw=$(so-yaml.py get $UPDATE_DIR/salt/elasticsearch/defaults.yaml elasticsearch.version); then - # so-yaml.py failed to get the ES version from upgrade versions elasticsearch/defaults.yaml file. Likely they are upgrading to an SO version older than 2.4.110 prior to the ES version pinning and should be OKAY to continue with the upgrade. + if ! target_es_version=$(so-yaml.py get -r $UPDATE_DIR/salt/elasticsearch/defaults.yaml elasticsearch.version); then + echo "Couldn't determine the target Elasticsearch version (post soup version) to ensure compatibility with current Elasticsearch version. Exiting" - # if so-yaml.py failed to get the ES version AND the version we are upgrading to is newer than 2.4.110 then we should bail - if [[ $(cat $UPDATE_DIR/VERSION | cut -d'.' -f3) > 110 ]]; then - echo "Couldn't determine the target Elasticsearch version (post soup version) to ensure compatibility with current Elasticsearch version. Exiting" - - exit 160 - fi - - # allow upgrade to version < 2.4.110 without checking ES version compatibility - return 0 - else - target_es_version=$(sed -n '1p' <<< "$target_es_version_raw") + exit 160 fi for statefile in "${es_required_version_statefile_base}"-*; do [[ -f $statefile ]] || continue - local es_required_version_statefile_value=$(cat "$statefile") + local es_required_version_statefile_value + es_required_version_statefile_value=$(cat "$statefile") if [[ "$es_required_version_statefile_value" == "$target_es_version" ]]; then echo "Intermediate upgrade to ES $target_es_version is in progress. Skipping Elasticsearch version compatibility check." @@ -797,7 +799,7 @@ verify_es_version_compatibility() { fi # use sort to check if es_required_statefile_value is < the current es_version. - if [[ "$(printf '%s\n' $es_required_version_statefile_value $es_version | sort -V | head -n1)" == "$es_required_version_statefile_value" ]]; then + if [[ "$(printf '%s\n' "$es_required_version_statefile_value" "$es_version" | sort -V | head -n1)" == "$es_required_version_statefile_value" ]]; then rm -f "$statefile" continue fi @@ -808,8 +810,7 @@ verify_es_version_compatibility() { echo -e "\n##############################################################################################################################\n" echo "A previously required intermediate Elasticsearch upgrade was detected. Verifying that all Searchnodes/Heavynodes have successfully upgraded Elasticsearch to $es_required_version_statefile_value before proceeding with soup to avoid potential data loss! This command can take up to an hour to complete." - timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$statefile" - if [[ $? -ne 0 ]]; then + if ! timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$statefile"; then echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n" echo "A previous required intermediate Elasticsearch upgrade to $es_required_version_statefile_value has yet to successfully complete across the grid. Please allow time for all Searchnodes/Heavynodes to have upgraded Elasticsearch to $es_required_version_statefile_value before running soup again to avoid potential data loss!" @@ -826,6 +827,7 @@ verify_es_version_compatibility() { return 0 fi + # shellcheck disable=SC2076 # Do not want a regex here eg usage " 8.18.8 9.0.8 " =~ " 9.0.8 " if [[ " ${es_upgrade_map[$es_version]} " =~ " $target_es_version " || "$es_version" == "$target_es_version" ]]; then # supported upgrade return 0 @@ -834,7 +836,7 @@ verify_es_version_compatibility() { if [[ -z "$compatible_versions" ]]; then # If current ES version is not explicitly defined in the upgrade map, we know they have an intermediate upgrade to do. # We default to the lowest ES version defined in es_to_so_version as $first_es_required_version - local first_es_required_version=$(printf '%s\n' "${!es_to_so_version[@]}" | sort -V | head -n1) + first_es_required_version=$(printf '%s\n' "${!es_to_so_version[@]}" | sort -V | head -n1) next_step_so_version=${es_to_so_version[$first_es_required_version]} required_es_upgrade_version="$first_es_required_version" else @@ -853,7 +855,7 @@ verify_es_version_compatibility() { if [[ $is_airgap -eq 0 ]]; then run_airgap_intermediate_upgrade else - if [[ ! -z $ISOLOC ]]; then + if [[ -n $ISOLOC ]]; then originally_requested_iso_location="$ISOLOC" fi # Make sure ISOLOC is not set. Network installs that used soup -f would have ISOLOC set. @@ -885,7 +887,8 @@ wait_for_salt_minion_with_restart() { } run_airgap_intermediate_upgrade() { - local originally_requested_so_version=$(cat $UPDATE_DIR/VERSION) + local originally_requested_so_version + originally_requested_so_version=$(cat "$UPDATE_DIR/VERSION") # preserve ISOLOC value, so we can try to use it post intermediate upgrade local originally_requested_iso_location="$ISOLOC" @@ -897,7 +900,8 @@ run_airgap_intermediate_upgrade() { while [[ -z "$next_iso_location" ]] || [[ ! -f "$next_iso_location" && ! -b "$next_iso_location" ]]; do # List removable devices if any are present - local removable_devices=$(lsblk -no PATH,SIZE,TYPE,MOUNTPOINTS,RM | awk '$NF==1') + local removable_devices + removable_devices=$(lsblk -no PATH,SIZE,TYPE,MOUNTPOINTS,RM | awk '$NF==1') if [[ -n "$removable_devices" ]]; then echo "PATH SIZE TYPE MOUNTPOINTS RM" echo "$removable_devices" @@ -918,21 +922,21 @@ run_airgap_intermediate_upgrade() { echo "Using $next_iso_location for required intermediary upgrade." exec bash <