From 7b5980bfe54e9bbc73e1f871296ab5e7e0bae205 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 30 Jul 2025 16:04:10 -0400 Subject: [PATCH 1/6] setup bridge for hypervisor using $MNIC --- setup/so-functions | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index c2eb0b349..91d717c11 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1187,14 +1187,18 @@ get_minion_type() { } hypervisor_local_states() { - # these states need to run before the first highstate so that we dont deal with the salt-minion restarting - # and we need these setup prior to the highstate - if [ $is_hypervisor ] || [ $is_managerhype ]; then - salt-call state.apply libvirt.64962 --local --file-root=../salt/ -l info - salt-call state.apply libvirt.bridge --local --file-root=../salt/ -l info pillar='{"host": {"mainint": "enp1s0"}}' - fi + # these states need to run before the first highstate so that we dont deal with the salt-minion restarting + # and we need these setup prior to the highstate + info "Check if hypervisor or managerhype" + if [ $is_hypervisor ] || [ $is_managerhype ]; then + info "Running libvirt states for hypervisor" + logCmd "salt-call state.apply libvirt.64962 --local --file-root=../salt/ -l info" + info "Setting up bridge for $MNIC" + salt-call state.apply libvirt.bridge --local --file-root=../salt/ -l info pillar="{\"host\": {\"mainint\": \"$MNIC\"}}" + fi } + install_cleanup() { if [ -f "$temp_install_dir" ]; then info "Installer removing the following files:" From 8720a4540a4cfa8dade196554609fd6dd1ae74c0 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 30 Jul 2025 16:36:40 -0400 Subject: [PATCH 2/6] remove extra line --- setup/so-functions | 1 - 1 file changed, 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index 91d717c11..522446be4 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1198,7 +1198,6 @@ hypervisor_local_states() { fi } - install_cleanup() { if [ -f "$temp_install_dir" ]; then info "Installer removing the following files:" From 15cbc626c4fba7a3c9f91bf4fddba548436c340c Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Wed, 30 Jul 2025 16:37:19 -0400 Subject: [PATCH 3/6] resolve for already configured RAID --- salt/hypervisor/tools/sbin/so-nvme-raid1.sh | 120 ++++++++++++++------ 1 file changed, 88 insertions(+), 32 deletions(-) diff --git a/salt/hypervisor/tools/sbin/so-nvme-raid1.sh b/salt/hypervisor/tools/sbin/so-nvme-raid1.sh index 79ccbb33c..ab97e3c88 100644 --- a/salt/hypervisor/tools/sbin/so-nvme-raid1.sh +++ b/salt/hypervisor/tools/sbin/so-nvme-raid1.sh @@ -12,7 +12,7 @@ # - Detects and reports existing RAID configurations # - Thoroughly cleans target drives of any existing data/configurations # - Creates GPT partition tables with RAID-type partitions -# - Establishes RAID-1 array (/dev/md0) for data redundancy +# - Establishes RAID-1 array (${RAID_DEVICE}) for data redundancy # - Formats the array with XFS filesystem for performance # - Automatically mounts at /nsm and configures for boot persistence # - Provides monitoring information for resync operations @@ -37,6 +37,11 @@ # Exit on any error set -e +# Configuration variables +RAID_ARRAY_NAME="md0" +RAID_DEVICE="/dev/${RAID_ARRAY_NAME}" +MOUNT_POINT="/nsm" + # Function to log messages log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" @@ -50,36 +55,68 @@ check_root() { fi } +# Function to find MD arrays using specific devices +find_md_arrays_using_devices() { + local target_devices=("$@") + local found_arrays=() + + # Parse /proc/mdstat to find arrays using our target devices + if [ -f "/proc/mdstat" ]; then + while IFS= read -r line; do + if [[ $line =~ ^(md[0-9]+) ]]; then + local array_name="${BASH_REMATCH[1]}" + local array_path="/dev/$array_name" + + # Check if this array uses any of our target devices + for device in "${target_devices[@]}"; do + if echo "$line" | grep -q "${device##*/}"; then + found_arrays+=("$array_path") + break + fi + done + fi + done < /proc/mdstat + fi + + printf '%s\n' "${found_arrays[@]}" +} + # Function to check if RAID is already set up check_existing_raid() { - if [ -e "/dev/md0" ]; then - if mdadm --detail /dev/md0 &>/dev/null; then - local raid_state=$(mdadm --detail /dev/md0 | grep "State" | awk '{print $3}') - local mount_point="/nsm" - - log "Found existing RAID array /dev/md0 (State: $raid_state)" - - if mountpoint -q "$mount_point"; then - log "RAID is already mounted at $mount_point" - log "Current RAID details:" - mdadm --detail /dev/md0 + local target_devices=("/dev/nvme0n1p1" "/dev/nvme1n1p1") + local found_arrays=($(find_md_arrays_using_devices "${target_devices[@]}")) + + # Check if we found any arrays using our target devices + if [ ${#found_arrays[@]} -gt 0 ]; then + for array_path in "${found_arrays[@]}"; do + if mdadm --detail "$array_path" &>/dev/null; then + local raid_state=$(mdadm --detail "$array_path" | grep "State" | awk '{print $3}') + local mount_point="/nsm" - # Check if resyncing - if grep -q "resync" /proc/mdstat; then - log "RAID is currently resyncing:" - grep resync /proc/mdstat - log "You can monitor progress with: watch -n 60 cat /proc/mdstat" - else - log "RAID is fully synced and operational" + log "Found existing RAID array $array_path (State: $raid_state)" + + if mountpoint -q "$mount_point"; then + log "RAID is already mounted at $mount_point" + log "Current RAID details:" + mdadm --detail "$array_path" + + # Check if resyncing + if grep -q "resync" /proc/mdstat; then + log "RAID is currently resyncing:" + grep resync /proc/mdstat + log "You can monitor progress with: watch -n 60 cat /proc/mdstat" + else + log "RAID is fully synced and operational" + fi + + # Show disk usage + log "Current disk usage:" + df -h "$mount_point" + + exit 0 fi - - # Show disk usage - log "Current disk usage:" - df -h "$mount_point" - - exit 0 fi - fi + done fi # Check if any of the target devices are in use @@ -90,10 +127,29 @@ check_existing_raid() { fi if mdadm --examine "$device" &>/dev/null || mdadm --examine "${device}p1" &>/dev/null; then + # Find the actual array name for this device + local device_arrays=($(find_md_arrays_using_devices "${device}p1")) + local array_name="" + + if [ ${#device_arrays[@]} -gt 0 ]; then + array_name="${device_arrays[0]}" + else + # Fallback: try to find array name from /proc/mdstat + local partition_name="${device##*/}p1" + array_name=$(grep -l "$partition_name" /proc/mdstat 2>/dev/null | head -1) + if [ -n "$array_name" ]; then + array_name=$(grep "^md[0-9]" /proc/mdstat | grep "$partition_name" | awk '{print "/dev/" $1}' | head -1) + fi + # Final fallback + if [ -z "$array_name" ]; then + array_name="$RAID_DEVICE" + fi + fi + log "Error: $device appears to be part of an existing RAID array" log "To reuse this device, you must first:" log "1. Unmount any filesystems" - log "2. Stop the RAID array: mdadm --stop /dev/md0" + log "2. Stop the RAID array: mdadm --stop $array_name" log "3. Zero the superblock: mdadm --zero-superblock ${device}p1" exit 1 fi @@ -183,20 +239,20 @@ main() { fi log "Creating RAID array" - mdadm --create /dev/md0 --level=1 --raid-devices=2 \ + mdadm --create "$RAID_DEVICE" --level=1 --raid-devices=2 \ --metadata=1.2 \ /dev/nvme0n1p1 /dev/nvme1n1p1 \ --force --run log "Creating XFS filesystem" - mkfs.xfs -f /dev/md0 + mkfs.xfs -f "$RAID_DEVICE" log "Creating mount point" mkdir -p /nsm log "Updating fstab" - sed -i '/\/dev\/md0/d' /etc/fstab - echo "/dev/md0 /nsm xfs defaults,nofail 0 0" >> /etc/fstab + sed -i "\|${RAID_DEVICE}|d" /etc/fstab + echo "${RAID_DEVICE} ${MOUNT_POINT} xfs defaults,nofail 0 0" >> /etc/fstab log "Reloading systemd daemon" systemctl daemon-reload @@ -209,7 +265,7 @@ main() { log "RAID setup complete" log "RAID array details:" - mdadm --detail /dev/md0 + mdadm --detail "$RAID_DEVICE" if grep -q "resync" /proc/mdstat; then log "RAID is currently resyncing. You can monitor progress with:" From 930c8147e79a17bf576e410156e08b95398bfb53 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 1 Aug 2025 08:52:21 -0400 Subject: [PATCH 4/6] simplify cpu and memory regex --- .../hypervisor/soc_hypervisor.yaml.jinja | 36 +------------------ 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja b/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja index 07577a1fb..926263b9d 100644 --- a/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja +++ b/salt/soc/dyanno/hypervisor/soc_hypervisor.yaml.jinja @@ -70,41 +70,7 @@ Base domain has not been initialized. {%- endmacro -%} {%- macro update_resource_field(field, free_value, total_value, unit_label) -%} -{%- set resource_regex = '' -%} -{%- if free_value < 10 -%} -{%- set resource_regex = '^[1-' ~ free_value ~ ']$' -%} -{%- elif free_value < 100 -%} -{%- set tens_digit = free_value // 10 -%} -{%- set ones_digit = free_value % 10 -%} -{%- if ones_digit == 0 -%} -{%- set resource_regex = '^([1-9]|[1-' ~ (tens_digit-1) ~ '][0-9]|' ~ tens_digit ~ '0)$' -%} -{%- else -%} -{%- set resource_regex = '^([1-9]|[1-' ~ (tens_digit-1) ~ '][0-9]|' ~ tens_digit ~ '[0-' ~ ones_digit ~ '])$' -%} -{%- endif -%} -{%- elif free_value < 1000 -%} -{%- set hundreds_digit = free_value // 100 -%} -{%- set tens_digit = (free_value % 100) // 10 -%} -{%- set ones_digit = free_value % 10 -%} -{%- if hundreds_digit == 1 -%} -{%- if tens_digit == 0 and ones_digit == 0 -%} -{%- set resource_regex = '^([1-9]|[1-9][0-9]|100)$' -%} -{%- elif tens_digit == 0 -%} -{%- set resource_regex = '^([1-9]|[1-9][0-9]|10[0-' ~ ones_digit ~ '])$' -%} -{%- elif ones_digit == 0 -%} -{%- set resource_regex = '^([1-9]|[1-9][0-9]|10[0-9]|1[1-' ~ tens_digit ~ ']0)$' -%} -{%- else -%} -{%- set resource_regex = '^([1-9]|[1-9][0-9]|10[0-9]|1[1-' ~ (tens_digit-1) ~ '][0-9]|1' ~ tens_digit ~ '[0-' ~ ones_digit ~ '])$' -%} -{%- endif -%} -{%- else -%} -{%- if tens_digit == 0 and ones_digit == 0 -%} -{%- set resource_regex = '^([1-9]|[1-9][0-9]|[1-9][0-9][0-9]|[1-' ~ (hundreds_digit-1) ~ '][0-9][0-9]|' ~ hundreds_digit ~ '00)$' -%} -{%- elif ones_digit == 0 -%} -{%- set resource_regex = '^([1-9]|[1-9][0-9]|[1-9][0-9][0-9]|[1-' ~ (hundreds_digit-1) ~ '][0-9][0-9]|' ~ hundreds_digit ~ '[0-' ~ tens_digit ~ ']0)$' -%} -{%- else -%} -{%- set resource_regex = '^([1-9]|[1-9][0-9]|[1-9][0-9][0-9]|[1-' ~ (hundreds_digit-1) ~ '][0-9][0-9]|' ~ hundreds_digit ~ '[0-' ~ (tens_digit-1) ~ '][0-9]|' ~ hundreds_digit ~ tens_digit ~ '[0-' ~ ones_digit ~ '])$' -%} -{%- endif -%} -{%- endif -%} -{%- endif -%} +{%- set resource_regex = '^[0-9]{1,3}$' -%} {%- do field.update({ 'label': field.label | replace('FREE', free_value | string) | replace('TOTAL', total_value | string), 'regex': resource_regex, From e9e3252bb5114ed5ddd7996fe1f2fcbfbbfadef4 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 1 Aug 2025 08:53:45 -0400 Subject: [PATCH 5/6] nvme script move nsm if mounted --- salt/hypervisor/tools/sbin/so-nvme-raid1.sh | 92 +++++++++++++++++---- 1 file changed, 75 insertions(+), 17 deletions(-) diff --git a/salt/hypervisor/tools/sbin/so-nvme-raid1.sh b/salt/hypervisor/tools/sbin/so-nvme-raid1.sh index ab97e3c88..9b9fcba2b 100644 --- a/salt/hypervisor/tools/sbin/so-nvme-raid1.sh +++ b/salt/hypervisor/tools/sbin/so-nvme-raid1.sh @@ -95,20 +95,83 @@ check_existing_raid() { log "Found existing RAID array $array_path (State: $raid_state)" - if mountpoint -q "$mount_point"; then - log "RAID is already mounted at $mount_point" + # Check what's currently mounted at /nsm + local current_mount=$(findmnt -n -o SOURCE "$mount_point" 2>/dev/null || echo "") + + if [ -n "$current_mount" ]; then + if [ "$current_mount" = "$array_path" ]; then + log "RAID array $array_path is already correctly mounted at $mount_point" + log "Current RAID details:" + mdadm --detail "$array_path" + + # Check if resyncing + if grep -q "resync" /proc/mdstat; then + log "RAID is currently resyncing:" + grep resync /proc/mdstat + log "You can monitor progress with: watch -n 60 cat /proc/mdstat" + else + log "RAID is fully synced and operational" + fi + + # Show disk usage + log "Current disk usage:" + df -h "$mount_point" + + exit 0 + else + log "Found $mount_point mounted on $current_mount, but RAID array $array_path exists" + log "Will unmount current filesystem and remount on RAID array" + + # Unmount current filesystem + log "Unmounting $mount_point" + umount "$mount_point" + + # Remove old fstab entry + log "Removing old fstab entry for $current_mount" + sed -i "\|$current_mount|d" /etc/fstab + + # Mount the RAID array + log "Mounting RAID array $array_path at $mount_point" + mount "$array_path" "$mount_point" + + # Update fstab + log "Updating fstab for RAID array" + sed -i "\|${array_path}|d" /etc/fstab + echo "${array_path} ${mount_point} xfs defaults,nofail 0 0" >> /etc/fstab + + log "RAID array is now mounted at $mount_point" + log "Current RAID details:" + mdadm --detail "$array_path" + + # Check if resyncing + if grep -q "resync" /proc/mdstat; then + log "RAID is currently resyncing:" + grep resync /proc/mdstat + log "You can monitor progress with: watch -n 60 cat /proc/mdstat" + else + log "RAID is fully synced and operational" + fi + + # Show disk usage + log "Current disk usage:" + df -h "$mount_point" + + exit 0 + fi + else + # /nsm not mounted, mount the RAID array + log "Mounting RAID array $array_path at $mount_point" + mount "$array_path" "$mount_point" + + # Update fstab + log "Updating fstab for RAID array" + sed -i "\|${array_path}|d" /etc/fstab + echo "${array_path} ${mount_point} xfs defaults,nofail 0 0" >> /etc/fstab + + log "RAID array is now mounted at $mount_point" log "Current RAID details:" mdadm --detail "$array_path" - # Check if resyncing - if grep -q "resync" /proc/mdstat; then - log "RAID is currently resyncing:" - grep resync /proc/mdstat - log "You can monitor progress with: watch -n 60 cat /proc/mdstat" - else - log "RAID is fully synced and operational" - fi - # Show disk usage log "Current disk usage:" df -h "$mount_point" @@ -120,12 +183,7 @@ check_existing_raid() { fi # Check if any of the target devices are in use - for device in "/dev/nvme0n1" "/dev/nvme1n1"; do - if lsblk -o NAME,MOUNTPOINT "$device" | grep -q "nsm"; then - log "Error: $device is already mounted at /nsm" - exit 1 - fi - + for device in "/dev/nvme0n1" "/dev/nvme1n1"; do if mdadm --examine "$device" &>/dev/null || mdadm --examine "${device}p1" &>/dev/null; then # Find the actual array name for this device local device_arrays=($(find_md_arrays_using_devices "${device}p1")) From 9d96a117530ea07334304606d3c648e163177bb8 Mon Sep 17 00:00:00 2001 From: Josh Patterson Date: Fri, 1 Aug 2025 08:55:38 -0400 Subject: [PATCH 6/6] update usage --- salt/hypervisor/tools/sbin/so-nvme-raid1.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/hypervisor/tools/sbin/so-nvme-raid1.sh b/salt/hypervisor/tools/sbin/so-nvme-raid1.sh index 9b9fcba2b..cc9916a4c 100644 --- a/salt/hypervisor/tools/sbin/so-nvme-raid1.sh +++ b/salt/hypervisor/tools/sbin/so-nvme-raid1.sh @@ -30,7 +30,7 @@ # # WARNING: This script will DESTROY all data on the target drives! # -# USAGE: sudo ./raid_setup.sh +# USAGE: sudo ./so-nvme-raid1.sh # #################################################################