mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-03-24 13:32:37 +01:00
1405 lines
52 KiB
Bash
Executable File
1405 lines
52 KiB
Bash
Executable File
#!/bin/bash
|
||
|
||
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
|
||
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
|
||
# https://securityonion.net/license; you may not use this file except in compliance with the
|
||
# Elastic License 2.0.
|
||
|
||
|
||
. /usr/sbin/so-common
|
||
. /usr/sbin/so-image-common
|
||
|
||
UPDATE_DIR=/tmp/sogh/securityonion
|
||
DEFAULT_SALT_DIR=/opt/so/saltstack/default
|
||
INSTALLEDVERSION=$(cat /etc/soversion)
|
||
POSTVERSION=$INSTALLEDVERSION
|
||
INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
|
||
BATCHSIZE=5
|
||
SOUP_LOG=/root/soup.log
|
||
WHATWOULDYOUSAYYAHDOHERE=soup
|
||
whiptail_title='Security Onion UPdater'
|
||
NOTIFYCUSTOMELASTICCONFIG=false
|
||
TOPFILE=/opt/so/saltstack/default/salt/top.sls
|
||
BACKUPTOPFILE=/opt/so/saltstack/default/salt/top.sls.backup
|
||
SALTUPGRADED=false
|
||
SALT_CLOUD_INSTALLED=false
|
||
SALT_CLOUD_CONFIGURED=false
|
||
# used to display messages to the user at the end of soup
|
||
declare -a FINAL_MESSAGE_QUEUE=()
|
||
|
||
|
||
check_err() {
|
||
local exit_code=$1
|
||
local err_msg="Unhandled error occured, please check $SOUP_LOG for details."
|
||
|
||
[[ $ERR_HANDLED == true ]] && exit $exit_code
|
||
|
||
if [[ $exit_code -ne 0 ]]; then
|
||
|
||
set +e
|
||
failed_soup_restore_items
|
||
|
||
printf '%s' "Soup failed with error $exit_code: "
|
||
case $exit_code in
|
||
2)
|
||
echo 'No such file or directory'
|
||
;;
|
||
5)
|
||
echo 'Interrupted system call'
|
||
;;
|
||
12)
|
||
echo 'Out of memory'
|
||
;;
|
||
28)
|
||
echo 'No space left on device'
|
||
echo "Likely ran out of space on disk, please review hardware requirements for Security Onion: $DOC_BASE_URL/hardware"
|
||
;;
|
||
30)
|
||
echo 'Read-only file system'
|
||
;;
|
||
35)
|
||
echo 'Resource temporarily unavailable'
|
||
;;
|
||
64)
|
||
echo 'Machine is not on the network'
|
||
;;
|
||
67)
|
||
echo 'Link has been severed'
|
||
;;
|
||
100)
|
||
echo 'Network is down'
|
||
;;
|
||
101)
|
||
echo 'Network is unreachable'
|
||
;;
|
||
102)
|
||
echo 'Network reset'
|
||
;;
|
||
110)
|
||
echo 'Connection timed out'
|
||
;;
|
||
111)
|
||
echo 'Connection refused'
|
||
;;
|
||
112)
|
||
echo 'Host is down'
|
||
;;
|
||
113)
|
||
echo 'No route to host'
|
||
;;
|
||
160)
|
||
echo 'Incompatible Elasticsearch upgrade'
|
||
;;
|
||
161)
|
||
echo 'Required intermediate Elasticsearch upgrade not complete'
|
||
;;
|
||
170)
|
||
echo "Intermediate upgrade completed successfully to $next_step_so_version, but next soup to Security Onion $originally_requested_so_version could not be started automatically."
|
||
echo "Start soup again manually to continue the upgrade to Security Onion $originally_requested_so_version."
|
||
;;
|
||
*)
|
||
echo 'Unhandled error'
|
||
echo "$err_msg"
|
||
;;
|
||
esac
|
||
if [[ $exit_code -ge 64 && $exit_code -le 113 ]]; then
|
||
echo "$err_msg"
|
||
fi
|
||
|
||
exit $exit_code
|
||
fi
|
||
|
||
}
|
||
|
||
airgap_mounted() {
|
||
# Let's see if the ISO is already mounted.
|
||
if [[ -f /tmp/soagupdate/SecurityOnion/VERSION ]]; then
|
||
echo "The ISO is already mounted"
|
||
else
|
||
if [[ -z $ISOLOC ]]; then
|
||
echo "This is airgap. Ask for a location."
|
||
echo ""
|
||
cat << EOF
|
||
In order for soup to proceed, the path to the downloaded Security Onion ISO file, or the path to the CD-ROM or equivalent device containing the ISO media must be provided.
|
||
For example, if you have copied the new Security Onion ISO file to your home directory, then the path might look like /home/myuser/securityonion-2.x.y.iso.
|
||
Or, if you have burned the new ISO onto an optical disk then the path might look like /dev/cdrom.
|
||
|
||
EOF
|
||
read -rp 'Enter the path to the new Security Onion ISO content: ' ISOLOC
|
||
fi
|
||
if [[ -f $ISOLOC ]]; then
|
||
# Mounting the ISO image
|
||
mkdir -p /tmp/soagupdate
|
||
mount -t iso9660 -o loop $ISOLOC /tmp/soagupdate
|
||
# Make sure mounting was successful
|
||
if [ ! -f /tmp/soagupdate/SecurityOnion/VERSION ]; then
|
||
echo "Something went wrong trying to mount the ISO."
|
||
echo "Ensure you verify the ISO that you downloaded."
|
||
exit 0
|
||
else
|
||
echo "ISO has been mounted!"
|
||
fi
|
||
elif [[ -f $ISOLOC/SecurityOnion/VERSION ]]; then
|
||
ln -s $ISOLOC /tmp/soagupdate
|
||
echo "Found the update content"
|
||
elif [[ -b $ISOLOC ]]; then
|
||
mkdir -p /tmp/soagupdate
|
||
mount $ISOLOC /tmp/soagupdate
|
||
if [ ! -f /tmp/soagupdate/SecurityOnion/VERSION ]; then
|
||
echo "Something went wrong trying to mount the device."
|
||
echo "Ensure you verify the ISO that you downloaded."
|
||
exit 0
|
||
else
|
||
echo "Device has been mounted! $(cat /tmp/soagupdate/SecurityOnion/VERSION)"
|
||
fi
|
||
else
|
||
echo "Could not find Security Onion ISO content at ${ISOLOC}"
|
||
echo "Ensure the path you entered is correct, and that you verify the ISO that you downloaded."
|
||
exit 0
|
||
fi
|
||
fi
|
||
}
|
||
|
||
airgap_update_dockers() {
|
||
if [[ $is_airgap -eq 0 ]] || [[ ! -z "$ISOLOC" ]]; then
|
||
# Let's copy the tarball
|
||
if [[ ! -f $AGDOCKER/registry.tar ]]; then
|
||
echo "Unable to locate registry. Exiting"
|
||
exit 0
|
||
else
|
||
echo "Stopping the registry docker"
|
||
docker stop so-dockerregistry
|
||
docker rm so-dockerregistry
|
||
echo "Copying the new dockers over"
|
||
tar xf "$AGDOCKER/registry.tar" -C /nsm/docker-registry/docker
|
||
echo "Add Registry back"
|
||
docker load -i "$AGDOCKER/registry_image.tar"
|
||
echo "Restart registry container"
|
||
salt-call state.apply registry queue=True
|
||
fi
|
||
fi
|
||
}
|
||
|
||
backup_old_states_pillars() {
|
||
|
||
tar czf /nsm/backup/$(echo $INSTALLEDVERSION)_$(date +%Y%m%d-%H%M%S)_soup_default_states_pillars.tar.gz /opt/so/saltstack/default/
|
||
tar czf /nsm/backup/$(echo $INSTALLEDVERSION)_$(date +%Y%m%d-%H%M%S)_soup_local_states_pillars.tar.gz /opt/so/saltstack/local/
|
||
|
||
}
|
||
|
||
update_registry() {
|
||
docker stop so-dockerregistry
|
||
docker rm so-dockerregistry
|
||
salt-call state.apply registry queue=True
|
||
}
|
||
|
||
check_airgap() {
|
||
# See if this is an airgap install
|
||
AIRGAP=$(cat /opt/so/saltstack/local/pillar/global/soc_global.sls | grep airgap: | awk '{print $2}' | tr '[:upper:]' '[:lower:]')
|
||
if [[ "$AIRGAP" == "true" ]]; then
|
||
is_airgap=0
|
||
UPDATE_DIR=/tmp/soagupdate/SecurityOnion
|
||
AGDOCKER=/tmp/soagupdate/docker
|
||
AGREPO=/tmp/soagupdate/minimal/Packages
|
||
else
|
||
is_airgap=1
|
||
fi
|
||
}
|
||
|
||
check_local_mods() {
|
||
local salt_local=/opt/so/saltstack/local
|
||
|
||
local_mod_arr=()
|
||
|
||
while IFS= read -r -d '' local_file; do
|
||
stripped_path=${local_file#"$salt_local"}
|
||
default_file="${DEFAULT_SALT_DIR}${stripped_path}"
|
||
if [[ -f $default_file ]]; then
|
||
file_diff=$(diff "$default_file" "$local_file" )
|
||
if [[ $(echo "$file_diff" | grep -Ec "^[<>]") -gt 0 ]]; then
|
||
local_mod_arr+=( "$local_file" )
|
||
fi
|
||
fi
|
||
done< <(find $salt_local -type f -print0)
|
||
|
||
if [[ ${#local_mod_arr} -gt 0 ]]; then
|
||
echo "Potentially breaking changes found in the following files (check ${DEFAULT_SALT_DIR} for original copy):"
|
||
for file_str in "${local_mod_arr[@]}"; do
|
||
echo " $file_str"
|
||
done
|
||
echo ""
|
||
echo "To reference this list later, check $SOUP_LOG"
|
||
sleep 10
|
||
fi
|
||
}
|
||
|
||
check_pillar_items() {
|
||
local pillar_output=$(salt-call pillar.items -lerror --out=json)
|
||
|
||
cond=$(jq '.local | has("_errors")' <<< "$pillar_output")
|
||
if [[ "$cond" == "true" ]]; then
|
||
printf "\nThere is an issue rendering the manager's pillars. Please correct the issues in the sls files mentioned below before running SOUP again.\n\n"
|
||
jq '.local._errors[]' <<< "$pillar_output"
|
||
exit 0
|
||
else
|
||
printf "\nThe manager's pillars can be rendered. We can proceed with SOUP.\n\n"
|
||
fi
|
||
}
|
||
|
||
check_saltmaster_status() {
|
||
set +e
|
||
echo "Waiting on the Salt Master service to be ready."
|
||
check_salt_master_status || fail "Can't access salt master or it is not ready. Check $SOUP_LOG for details."
|
||
set -e
|
||
}
|
||
|
||
check_sudoers() {
|
||
if grep -q "so-setup" /etc/sudoers; then
|
||
echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"."
|
||
fi
|
||
}
|
||
|
||
check_os_updates() {
|
||
# Check to see if there are OS updates
|
||
echo "Checking for OS updates."
|
||
NEEDUPDATES="We have detected missing operating system (OS) updates. Do you want to install these OS updates now? This could take a while depending on the size of your grid and how many packages are missing, but it is recommended to keep your system updated."
|
||
OSUPDATES=$(dnf -q list updates | grep -v docker | grep -v containerd | grep -v salt | grep -v Available | wc -l)
|
||
if [[ "$OSUPDATES" -gt 0 ]]; then
|
||
if [[ -z $UNATTENDED ]]; then
|
||
echo "$NEEDUPDATES"
|
||
echo ""
|
||
read -rp "Press U to update OS packages (recommended), C to continue without updates, or E to exit: " confirm
|
||
if [[ "$confirm" == [cC] ]]; then
|
||
echo "Continuing without updating packages"
|
||
elif [[ "$confirm" == [uU] ]]; then
|
||
echo "Applying Grid Updates. The following patch.os salt state may take a while depending on how many packages need to be updated."
|
||
update_flag=true
|
||
else
|
||
echo "Exiting soup"
|
||
exit 0
|
||
fi
|
||
else
|
||
update_flag=true
|
||
fi
|
||
else
|
||
echo "Looks like you have an updated OS"
|
||
fi
|
||
|
||
if [[ $update_flag == true ]]; then
|
||
set +e
|
||
run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.'
|
||
set -e
|
||
fi
|
||
}
|
||
|
||
clean_dockers() {
|
||
# Place Holder for cleaning up old docker images
|
||
echo "Trying to clean up old dockers."
|
||
docker system prune -a -f --volumes
|
||
|
||
}
|
||
|
||
clone_to_tmp() {
|
||
# Clean old files
|
||
rm -rf /tmp/sogh
|
||
# Make a temp location for the files
|
||
mkdir -p /tmp/sogh
|
||
cd /tmp/sogh
|
||
SOUP_BRANCH="-b 2.4/main"
|
||
if [ -n "$BRANCH" ]; then
|
||
SOUP_BRANCH="-b $BRANCH"
|
||
fi
|
||
git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git
|
||
cd /tmp
|
||
if [ ! -f $UPDATE_DIR/VERSION ]; then
|
||
echo "Update was unable to pull from Github. Please check your Internet access."
|
||
exit 0
|
||
fi
|
||
}
|
||
|
||
enable_highstate() {
|
||
echo "Enabling highstate."
|
||
salt-call state.enable highstate -l info --local
|
||
echo ""
|
||
}
|
||
|
||
get_soup_script_hashes() {
|
||
CURRENTSOUP=$(md5sum /usr/sbin/soup | awk '{print $1}')
|
||
GITSOUP=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/soup | awk '{print $1}')
|
||
CURRENTCMN=$(md5sum /usr/sbin/so-common | awk '{print $1}')
|
||
GITCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-common | awk '{print $1}')
|
||
CURRENTIMGCMN=$(md5sum /usr/sbin/so-image-common | awk '{print $1}')
|
||
GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}')
|
||
CURRENTSOFIREWALL=$(md5sum /usr/sbin/so-firewall | awk '{print $1}')
|
||
GITSOFIREWALL=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-firewall | awk '{print $1}')
|
||
}
|
||
|
||
highstate() {
|
||
# Run a highstate.
|
||
salt-call state.highstate -l info queue=True
|
||
}
|
||
|
||
masterlock() {
|
||
echo "Locking Salt Master"
|
||
mv -v $TOPFILE $BACKUPTOPFILE
|
||
echo "base:" > $TOPFILE
|
||
echo " $MINIONID:" >> $TOPFILE
|
||
echo " - ca" >> $TOPFILE
|
||
echo " - elasticsearch" >> $TOPFILE
|
||
}
|
||
|
||
masterunlock() {
|
||
if [ -f $BACKUPTOPFILE ]; then
|
||
echo "Unlocking Salt Master"
|
||
mv -v $BACKUPTOPFILE $TOPFILE
|
||
else
|
||
echo "Salt Master does not need unlocked."
|
||
fi
|
||
}
|
||
|
||
|
||
preupgrade_changes() {
|
||
# This function is to add any new pillar items if needed.
|
||
echo "Checking to see if changes are needed."
|
||
|
||
[[ "$INSTALLEDVERSION" =~ ^2\.4\.21[0-9]+$ ]] && up_to_3.0.0
|
||
true
|
||
}
|
||
|
||
postupgrade_changes() {
|
||
# This function is to add any new pillar items if needed.
|
||
echo "Running post upgrade processes."
|
||
|
||
[[ "$POSTVERSION" =~ ^2\.4\.21[0-9]+$ ]] && post_to_3.0.0
|
||
true
|
||
}
|
||
|
||
check_minimum_version() {
|
||
if [[ ! "$INSTALLEDVERSION" =~ ^(2\.4\.21[0-9]+|3\.) ]]; then
|
||
echo "You must be on at least Security Onion 2.4.210 to upgrade. Currently installed version: $INSTALLEDVERSION"
|
||
exit 1
|
||
fi
|
||
}
|
||
|
||
### 3.0.0 Scripts ###
|
||
|
||
up_to_3.0.0() {
|
||
determine_elastic_agent_upgrade
|
||
migrate_pcap_to_suricata
|
||
|
||
INSTALLEDVERSION=3.0.0
|
||
}
|
||
|
||
migrate_pcap_to_suricata() {
|
||
local MINIONDIR=/opt/so/saltstack/local/pillar/minions
|
||
local PCAPFILE=/opt/so/saltstack/local/pillar/pcap/soc_pcap.sls
|
||
|
||
for pillar_file in "$PCAPFILE" "$MINIONDIR"/*.sls; do
|
||
[[ -f "$pillar_file" ]] || continue
|
||
pcap_enabled=$(so-yaml.py get -r "$pillar_file" pcap.enabled 2>/dev/null) || continue
|
||
so-yaml.py add "$pillar_file" suricata.pcap.enabled "$pcap_enabled"
|
||
so-yaml.py remove "$pillar_file" pcap
|
||
done
|
||
}
|
||
|
||
post_to_3.0.0() {
|
||
for idx in "logs-idh-so" "logs-redis.log-default"; do
|
||
rollover_index "$idx"
|
||
done
|
||
|
||
# Remove ILM for so-case and so-detection indices
|
||
for idx in "so-case" "so-casehistory" "so-detection" "so-detectionhistory"; do
|
||
so-elasticsearch-query $idx/_ilm/remove -XPOST
|
||
done
|
||
|
||
POSTVERSION=3.0.0
|
||
}
|
||
|
||
### 3.0.0 End ###
|
||
|
||
repo_sync() {
|
||
echo "Sync the local repo."
|
||
su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
|
||
}
|
||
|
||
stop_salt_master() {
|
||
# kill all salt jobs across the grid because the hang indefinitely if they are queued and salt-master restarts
|
||
set +e
|
||
echo ""
|
||
echo "Killing all Salt jobs across the grid."
|
||
salt \* saltutil.kill_all_jobs >> $SOUP_LOG 2>&1
|
||
echo ""
|
||
echo "Killing any queued Salt jobs on the manager."
|
||
pkill -9 -ef "/usr/bin/python3 /bin/salt" >> $SOUP_LOG 2>&1
|
||
|
||
echo ""
|
||
echo "Storing salt-master PID."
|
||
MASTERPID=$(pgrep -f '/opt/saltstack/salt/bin/python3.10 /usr/bin/salt-master MainProcess')
|
||
if [ ! -z "$MASTERPID" ]; then
|
||
echo "Found salt-master PID $MASTERPID"
|
||
systemctl_func "stop" "salt-master"
|
||
if ps -p "$MASTERPID" > /dev/null 2>&1; then
|
||
timeout 30 tail --pid=$MASTERPID -f /dev/null || echo "salt-master still running at $(date +"%T.%6N") after waiting 30s. We cannot kill due to systemd restart option."
|
||
fi
|
||
else
|
||
echo "The salt-master PID was not found. The process '/usr/bin/salt-master MainProcess' is not running."
|
||
fi
|
||
set -e
|
||
}
|
||
|
||
stop_salt_minion() {
|
||
echo "Disabling highstate to prevent from running if salt-minion restarts."
|
||
salt-call state.disable highstate -l info --local
|
||
echo ""
|
||
|
||
# kill all salt jobs before stopping salt-minion
|
||
set +e
|
||
echo ""
|
||
echo "Killing Salt jobs on this node."
|
||
salt-call saltutil.kill_all_jobs --local
|
||
|
||
echo "Storing salt-minion pid."
|
||
MINIONPID=$(pgrep -f '/opt/saltstack/salt/bin/python3.10 /usr/bin/salt-minion' | head -1)
|
||
echo "Found salt-minion PID $MINIONPID"
|
||
systemctl_func "stop" "salt-minion"
|
||
|
||
timeout 30 tail --pid=$MINIONPID -f /dev/null || echo "Killing salt-minion at $(date +"%T.%6N") after waiting 30s" && pkill -9 -ef /usr/bin/salt-minion
|
||
set -e
|
||
}
|
||
|
||
determine_elastic_agent_upgrade() {
|
||
if [[ $is_airgap -eq 0 ]]; then
|
||
update_elastic_agent_airgap
|
||
else
|
||
set +e
|
||
# the new elasticsearch defaults.yaml file is not yet placed in /opt/so/saltstack/default/salt/elasticsearch yet
|
||
update_elastic_agent "$UPDATE_DIR"
|
||
set -e
|
||
fi
|
||
}
|
||
|
||
update_elastic_agent_airgap() {
|
||
get_elastic_agent_vars "/tmp/soagupdate/SecurityOnion"
|
||
rsync -av /tmp/soagupdate/fleet/* /nsm/elastic-fleet/artifacts/
|
||
tar -xf "$ELASTIC_AGENT_FILE" -C "$ELASTIC_AGENT_EXPANSION_DIR"
|
||
}
|
||
|
||
verify_upgradespace() {
|
||
CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//')
|
||
if [ "$CURRENTSPACE" -lt "10" ]; then
|
||
echo "You are low on disk space."
|
||
return 1
|
||
else
|
||
return 0
|
||
fi
|
||
}
|
||
|
||
upgrade_space() {
|
||
if ! verify_upgradespace; then
|
||
clean_dockers
|
||
if ! verify_upgradespace; then
|
||
echo "There is not enough space to perform the upgrade. Please free up space and try again"
|
||
exit 0
|
||
fi
|
||
else
|
||
echo "You have enough space for upgrade. Proceeding with soup."
|
||
fi
|
||
}
|
||
|
||
unmount_update() {
|
||
cd /tmp
|
||
umount /tmp/soagupdate
|
||
}
|
||
|
||
update_airgap_rules() {
|
||
# Copy the rules over to update them for airgap.
|
||
rsync -a --delete $UPDATE_DIR/agrules/suricata/ /nsm/rules/suricata/etopen/
|
||
rsync -a $UPDATE_DIR/agrules/detect-sigma/* /nsm/rules/detect-sigma/
|
||
rsync -a $UPDATE_DIR/agrules/detect-yara/* /nsm/rules/detect-yara/
|
||
# Copy the securityonion-resorces repo over for SOC Detection Summaries and checkout the published summaries branch
|
||
rsync -a --delete --chown=socore:socore $UPDATE_DIR/agrules/securityonion-resources /opt/so/conf/soc/ai_summary_repos
|
||
git config --global --add safe.directory /opt/so/conf/soc/ai_summary_repos/securityonion-resources
|
||
git -C /opt/so/conf/soc/ai_summary_repos/securityonion-resources checkout generated-summaries-published
|
||
# Copy the securityonion-resorces repo over to nsm
|
||
rsync -a $UPDATE_DIR/agrules/securityonion-resources/* /nsm/securityonion-resources/
|
||
}
|
||
|
||
update_airgap_repo() {
|
||
# Update the files in the repo
|
||
echo "Syncing new updates to /nsm/repo"
|
||
rsync -a $AGREPO/* /nsm/repo/
|
||
echo "Creating repo"
|
||
dnf -y install yum-utils createrepo_c
|
||
createrepo /nsm/repo
|
||
}
|
||
|
||
update_salt_mine() {
|
||
echo "Populating the mine with mine_functions for each host."
|
||
set +e
|
||
salt \* mine.update -b 50
|
||
set -e
|
||
}
|
||
|
||
update_version() {
|
||
# Update the version to the latest
|
||
echo "Updating the Security Onion version file."
|
||
echo $NEWVERSION > /etc/soversion
|
||
echo $HOTFIXVERSION > /etc/sohotfix
|
||
sed -i "s/soversion:.*/soversion: $NEWVERSION/" /opt/so/saltstack/local/pillar/global/soc_global.sls
|
||
}
|
||
|
||
upgrade_check() {
|
||
# Let's make sure we actually need to update.
|
||
NEWVERSION=$(cat $UPDATE_DIR/VERSION)
|
||
HOTFIXVERSION=$(cat $UPDATE_DIR/HOTFIX)
|
||
if [ ! -f /etc/sohotfix ]; then
|
||
touch /etc/sohotfix
|
||
fi
|
||
[[ -f /etc/sohotfix ]] && CURRENTHOTFIX=$(cat /etc/sohotfix)
|
||
if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then
|
||
echo "Checking to see if there are hotfixes needed"
|
||
if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then
|
||
echo "You are already running the latest version of Security Onion."
|
||
exit 0
|
||
else
|
||
echo "We need to apply a hotfix"
|
||
is_hotfix=true
|
||
fi
|
||
else
|
||
is_hotfix=false
|
||
fi
|
||
|
||
}
|
||
|
||
upgrade_check_salt() {
|
||
NEWSALTVERSION=$(grep "version:" $UPDATE_DIR/salt/salt/master.defaults.yaml | grep -o "[0-9]\+\.[0-9]\+")
|
||
if [ "$INSTALLEDSALTVERSION" == "$NEWSALTVERSION" ]; then
|
||
echo "You are already running the correct version of Salt for Security Onion."
|
||
else
|
||
echo "Salt needs to be upgraded to $NEWSALTVERSION."
|
||
UPGRADESALT=1
|
||
fi
|
||
}
|
||
|
||
upgrade_salt() {
|
||
echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION."
|
||
echo ""
|
||
# Check if salt-cloud is installed
|
||
if rpm -q salt-cloud &>/dev/null; then
|
||
SALT_CLOUD_INSTALLED=true
|
||
fi
|
||
# Check if salt-cloud is configured
|
||
if [[ -f /etc/salt/cloud.profiles.d/socloud.conf ]]; then
|
||
SALT_CLOUD_CONFIGURED=true
|
||
fi
|
||
|
||
echo "Removing yum versionlock for Salt."
|
||
echo ""
|
||
yum versionlock delete "salt"
|
||
yum versionlock delete "salt-minion"
|
||
yum versionlock delete "salt-master"
|
||
# Remove salt-cloud versionlock if installed
|
||
if [[ $SALT_CLOUD_INSTALLED == true ]]; then
|
||
yum versionlock delete "salt-cloud"
|
||
fi
|
||
echo "Updating Salt packages."
|
||
echo ""
|
||
set +e
|
||
# Run with -r to ignore repos set by bootstrap
|
||
if [[ $SALT_CLOUD_INSTALLED == true ]]; then
|
||
run_check_net_err \
|
||
"sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -r -L -F -M stable \"$NEWSALTVERSION\"" \
|
||
"Could not update salt, please check $SOUP_LOG for details."
|
||
else
|
||
run_check_net_err \
|
||
"sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -X -r -F -M stable \"$NEWSALTVERSION\"" \
|
||
"Could not update salt, please check $SOUP_LOG for details."
|
||
fi
|
||
set -e
|
||
echo "Applying yum versionlock for Salt."
|
||
echo ""
|
||
yum versionlock add "salt-0:$NEWSALTVERSION-0.*"
|
||
yum versionlock add "salt-minion-0:$NEWSALTVERSION-0.*"
|
||
yum versionlock add "salt-master-0:$NEWSALTVERSION-0.*"
|
||
# Add salt-cloud versionlock if installed
|
||
if [[ $SALT_CLOUD_INSTALLED == true ]]; then
|
||
yum versionlock add "salt-cloud-0:$NEWSALTVERSION-0.*"
|
||
fi
|
||
|
||
echo "Checking if Salt was upgraded."
|
||
echo ""
|
||
# Check that Salt was upgraded
|
||
SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk '{print $2}')
|
||
if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then
|
||
echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG."
|
||
echo "Once the issue is resolved, run soup again."
|
||
echo "Exiting."
|
||
echo ""
|
||
exit 1
|
||
else
|
||
SALTUPGRADED=true
|
||
echo "Salt upgrade success."
|
||
echo ""
|
||
fi
|
||
|
||
}
|
||
|
||
verify_latest_update_script() {
|
||
get_soup_script_hashes
|
||
if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then
|
||
echo "This version of the soup script is up to date. Proceeding."
|
||
else
|
||
echo "You are not running the latest soup version. Updating soup and its components. This might take multiple runs to complete."
|
||
|
||
salt-call state.apply common.soup_scripts queue=True -lerror --file-root=$UPDATE_DIR/salt --local --out-file=/dev/null
|
||
|
||
# Verify that soup scripts updated as expected
|
||
get_soup_script_hashes
|
||
if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then
|
||
echo "Succesfully updated soup scripts."
|
||
else
|
||
echo "There was a problem updating soup scripts. Trying to rerun script update."
|
||
salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local
|
||
fi
|
||
|
||
echo ""
|
||
echo "The soup script has been modified. Please run soup again to continue the upgrade."
|
||
exit 0
|
||
fi
|
||
|
||
}
|
||
|
||
verify_es_version_compatibility() {
|
||
|
||
local es_required_version_statefile_base="/opt/so/state/so_es_required_upgrade_version"
|
||
local es_verification_script="/tmp/so_intermediate_upgrade_verification.sh"
|
||
local is_active_intermediate_upgrade=1
|
||
# supported upgrade paths for SO-ES versions
|
||
declare -A es_upgrade_map=(
|
||
["8.18.8"]="9.0.8"
|
||
)
|
||
|
||
# Elasticsearch MUST upgrade through these versions
|
||
declare -A es_to_so_version=(
|
||
["8.18.8"]="2.4.190-20251024"
|
||
)
|
||
|
||
# Get current Elasticsearch version
|
||
if es_version_raw=$(so-elasticsearch-query / --fail --retry 5 --retry-delay 10); then
|
||
es_version=$(echo "$es_version_raw" | jq -r '.version.number' )
|
||
else
|
||
echo "Could not determine current Elasticsearch version to validate compatibility with post soup Elasticsearch version."
|
||
|
||
exit 160
|
||
fi
|
||
|
||
if ! target_es_version_raw=$(so-yaml.py get $UPDATE_DIR/salt/elasticsearch/defaults.yaml elasticsearch.version); then
|
||
# so-yaml.py failed to get the ES version from upgrade versions elasticsearch/defaults.yaml file. Likely they are upgrading to an SO version older than 2.4.110 prior to the ES version pinning and should be OKAY to continue with the upgrade.
|
||
|
||
# if so-yaml.py failed to get the ES version AND the version we are upgrading to is newer than 2.4.110 then we should bail
|
||
if [[ $(cat $UPDATE_DIR/VERSION | cut -d'.' -f3) > 110 ]]; then
|
||
echo "Couldn't determine the target Elasticsearch version (post soup version) to ensure compatibility with current Elasticsearch version. Exiting"
|
||
|
||
exit 160
|
||
fi
|
||
|
||
# allow upgrade to version < 2.4.110 without checking ES version compatibility
|
||
return 0
|
||
else
|
||
target_es_version=$(sed -n '1p' <<< "$target_es_version_raw")
|
||
fi
|
||
|
||
for statefile in "${es_required_version_statefile_base}"-*; do
|
||
[[ -f $statefile ]] || continue
|
||
|
||
local es_required_version_statefile_value=$(cat "$statefile")
|
||
|
||
if [[ "$es_required_version_statefile_value" == "$target_es_version" ]]; then
|
||
echo "Intermediate upgrade to ES $target_es_version is in progress. Skipping Elasticsearch version compatibility check."
|
||
is_active_intermediate_upgrade=0
|
||
continue
|
||
fi
|
||
|
||
# use sort to check if es_required_statefile_value is < the current es_version.
|
||
if [[ "$(printf '%s\n' $es_required_version_statefile_value $es_version | sort -V | head -n1)" == "$es_required_version_statefile_value" ]]; then
|
||
rm -f "$statefile"
|
||
continue
|
||
fi
|
||
|
||
if [[ ! -f "$es_verification_script" ]]; then
|
||
create_intermediate_upgrade_verification_script "$es_verification_script"
|
||
fi
|
||
|
||
echo -e "\n##############################################################################################################################\n"
|
||
echo "A previously required intermediate Elasticsearch upgrade was detected. Verifying that all Searchnodes/Heavynodes have successfully upgraded Elasticsearch to $es_required_version_statefile_value before proceeding with soup to avoid potential data loss! This command can take up to an hour to complete."
|
||
timeout --foreground 4000 bash "$es_verification_script" "$es_required_version_statefile_value" "$statefile"
|
||
if [[ $? -ne 0 ]]; then
|
||
echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
|
||
|
||
echo "A previous required intermediate Elasticsearch upgrade to $es_required_version_statefile_value has yet to successfully complete across the grid. Please allow time for all Searchnodes/Heavynodes to have upgraded Elasticsearch to $es_required_version_statefile_value before running soup again to avoid potential data loss!"
|
||
|
||
echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
|
||
|
||
exit 161
|
||
fi
|
||
echo -e "\n##############################################################################################################################\n"
|
||
done
|
||
|
||
# if current soup is an intermediate upgrade we can skip the upgrade map check below
|
||
if [[ $is_active_intermediate_upgrade -eq 0 ]]; then
|
||
return 0
|
||
fi
|
||
|
||
if [[ " ${es_upgrade_map[$es_version]} " =~ " $target_es_version " || "$es_version" == "$target_es_version" ]]; then
|
||
# supported upgrade
|
||
return 0
|
||
else
|
||
compatible_versions=${es_upgrade_map[$es_version]}
|
||
if [[ -z "$compatible_versions" ]]; then
|
||
# If current ES version is not explicitly defined in the upgrade map, we know they have an intermediate upgrade to do.
|
||
# We default to the lowest ES version defined in es_to_so_version as $first_es_required_version
|
||
local first_es_required_version=$(printf '%s\n' "${!es_to_so_version[@]}" | sort -V | head -n1)
|
||
next_step_so_version=${es_to_so_version[$first_es_required_version]}
|
||
required_es_upgrade_version="$first_es_required_version"
|
||
else
|
||
next_step_so_version=${es_to_so_version[${compatible_versions##* }]}
|
||
required_es_upgrade_version="${compatible_versions##* }"
|
||
fi
|
||
echo -e "\n##############################################################################################################################\n"
|
||
echo -e "You are currently running Security Onion $INSTALLEDVERSION. You will need to update to version $next_step_so_version before updating to $(cat $UPDATE_DIR/VERSION).\n"
|
||
|
||
es_required_version_statefile="${es_required_version_statefile_base}-${required_es_upgrade_version}"
|
||
echo "$required_es_upgrade_version" > "$es_required_version_statefile"
|
||
|
||
# We expect to upgrade to the latest compatiable minor version of ES
|
||
create_intermediate_upgrade_verification_script "$es_verification_script"
|
||
|
||
if [[ $is_airgap -eq 0 ]]; then
|
||
run_airgap_intermediate_upgrade
|
||
else
|
||
if [[ ! -z $ISOLOC ]]; then
|
||
originally_requested_iso_location="$ISOLOC"
|
||
fi
|
||
# Make sure ISOLOC is not set. Network installs that used soup -f would have ISOLOC set.
|
||
unset ISOLOC
|
||
|
||
run_network_intermediate_upgrade
|
||
fi
|
||
fi
|
||
|
||
}
|
||
|
||
wait_for_salt_minion_with_restart() {
|
||
local minion="$1"
|
||
local max_wait="${2:-60}"
|
||
local interval="${3:-3}"
|
||
local logfile="$4"
|
||
|
||
wait_for_salt_minion "$minion" "$max_wait" "$interval" "$logfile"
|
||
local result=$?
|
||
|
||
if [[ $result -ne 0 ]]; then
|
||
echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion not ready, attempting restart..."
|
||
systemctl_func "restart" "salt-minion"
|
||
wait_for_salt_minion "$minion" "$max_wait" "$interval" "$logfile"
|
||
result=$?
|
||
fi
|
||
|
||
return $result
|
||
}
|
||
|
||
run_airgap_intermediate_upgrade() {
|
||
local originally_requested_so_version=$(cat $UPDATE_DIR/VERSION)
|
||
# preserve ISOLOC value, so we can try to use it post intermediate upgrade
|
||
local originally_requested_iso_location="$ISOLOC"
|
||
|
||
# make sure a fresh ISO gets mounted
|
||
unmount_update
|
||
|
||
echo "You can download the $next_step_so_version ISO image from https://download.securityonion.net/file/securityonion/securityonion-$next_step_so_version.iso"
|
||
echo -e "\nIf you have the next ISO / USB ready, enter the path now eg. /dev/sdd, /home/onion/securityonion-$next_step_so_version.iso:"
|
||
|
||
while [[ -z "$next_iso_location" ]] || [[ ! -f "$next_iso_location" && ! -b "$next_iso_location" ]]; do
|
||
# List removable devices if any are present
|
||
local removable_devices=$(lsblk -no PATH,SIZE,TYPE,MOUNTPOINTS,RM | awk '$NF==1')
|
||
if [[ -n "$removable_devices" ]]; then
|
||
echo "PATH SIZE TYPE MOUNTPOINTS RM"
|
||
echo "$removable_devices"
|
||
fi
|
||
|
||
read -rp "Device/ISO Path (or 'exit' to quit): " next_iso_location
|
||
if [[ "${next_iso_location,,}" == "exit" ]]; then
|
||
echo "Exiting soup. Before reattempting to upgrade to $originally_requested_so_version, please first upgrade to $next_step_so_version to ensure Elasticsearch can properly update through the required versions."
|
||
|
||
exit 160
|
||
fi
|
||
|
||
if [[ ! -f "$next_iso_location" && ! -b "$next_iso_location" ]]; then
|
||
echo "$next_iso_location is not a valid file or block device."
|
||
next_iso_location=""
|
||
fi
|
||
done
|
||
|
||
echo "Using $next_iso_location for required intermediary upgrade."
|
||
exec bash <<EOF
|
||
ISOLOC=$next_iso_location soup -y && \
|
||
ISOLOC=$next_iso_location soup -y && \
|
||
|
||
echo -e "\n##############################################################################################################################\n" && \
|
||
echo -e "Verifying Elasticsearch was successfully upgraded to $required_es_upgrade_version across the grid. This part can take a while as Searchnodes/Heavynodes sync up with the Manager! \n\nOnce verification completes the next soup will begin automatically. If verification takes longer than 1 hour it will stop waiting and your grid will remain at $next_step_so_version. Allowing for all Searchnodes/Heavynodes to upgrade Elasticsearch to the required version on their own time.\n" && \
|
||
|
||
timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh $required_es_upgrade_version $es_required_version_statefile && \
|
||
|
||
echo -e "\n##############################################################################################################################\n" && \
|
||
|
||
# automatically start the next soup if the original ISO isn't using the same block device we just used
|
||
if [[ -n "$originally_requested_iso_location" ]] && [[ "$originally_requested_iso_location" != "$next_iso_location" ]]; then
|
||
umount /tmp/soagupdate
|
||
ISOLOC=$originally_requested_iso_location soup -y && \
|
||
ISOLOC=$originally_requested_iso_location soup -y
|
||
else
|
||
echo "Could not automatically start next soup to $originally_requested_so_version. Soup will now exit here at $(cat /etc/soversion)" && \
|
||
|
||
exit 170
|
||
fi
|
||
|
||
echo -e "\n##############################################################################################################################\n"
|
||
EOF
|
||
}
|
||
|
||
run_network_intermediate_upgrade() {
|
||
# preserve BRANCH value if set originally
|
||
if [[ -n "$BRANCH" ]]; then
|
||
local originally_requested_so_branch="$BRANCH"
|
||
else
|
||
local originally_requested_so_branch="2.4/main"
|
||
fi
|
||
|
||
echo "Starting automated intermediate upgrade to $next_step_so_version."
|
||
echo "After completion, the system will automatically attempt to upgrade to the latest version."
|
||
echo -e "\n##############################################################################################################################\n"
|
||
exec bash << EOF
|
||
BRANCH=$next_step_so_version soup -y && \
|
||
BRANCH=$next_step_so_version soup -y && \
|
||
|
||
echo -e "\n##############################################################################################################################\n" && \
|
||
echo -e "Verifying Elasticsearch was successfully upgraded to $required_es_upgrade_version across the grid. This part can take a while as Searchnodes/Heavynodes sync up with the Manager! \n\nOnce verification completes the next soup will begin automatically. If verification takes longer than 1 hour it will stop waiting and your grid will remain at $next_step_so_version. Allowing for all Searchnodes/Heavynodes to upgrade Elasticsearch to the required version on their own time.\n" && \
|
||
|
||
timeout --foreground 4000 bash /tmp/so_intermediate_upgrade_verification.sh $required_es_upgrade_version $es_required_version_statefile && \
|
||
|
||
echo -e "\n##############################################################################################################################\n" && \
|
||
if [[ -n "$originally_requested_iso_location" ]]; then
|
||
# nonairgap soup that used -f originally, runs intermediate upgrade using network + BRANCH, later coming back to the original ISO for the last soup
|
||
ISOLOC=$originally_requested_iso_location soup -y && \
|
||
ISOLOC=$originally_requested_iso_location soup -y
|
||
else
|
||
BRANCH=$originally_requested_so_branch soup -y && \
|
||
BRANCH=$originally_requested_so_branch soup -y
|
||
fi
|
||
echo -e "\n##############################################################################################################################\n"
|
||
EOF
|
||
}
|
||
|
||
create_intermediate_upgrade_verification_script() {
|
||
# After an intermediate upgrade, verify that ALL nodes running Elasticsearch are at the expected version BEFORE proceeding to the next upgrade step. This is a CRITICAL step
|
||
local verification_script="$1"
|
||
|
||
cat << 'EOF' > "$verification_script"
|
||
#!/bin/bash
|
||
|
||
SOUP_INTERMEDIATE_UPGRADE_FAILURES_LOG_FILE="/root/so_intermediate_upgrade_verification_failures.log"
|
||
CURRENT_TIME=$(date +%Y%m%d.%H%M%S)
|
||
EXPECTED_ES_VERSION="$1"
|
||
|
||
if [[ -z "$EXPECTED_ES_VERSION" ]]; then
|
||
echo -e "\nExpected Elasticsearch version not provided. Usage: $0 <expected_es_version>"
|
||
exit 1
|
||
fi
|
||
|
||
if [[ -f "$SOUP_INTERMEDIATE_UPGRADE_FAILURES_LOG_FILE" ]]; then
|
||
mv "$SOUP_INTERMEDIATE_UPGRADE_FAILURES_LOG_FILE" "$SOUP_INTERMEDIATE_UPGRADE_FAILURES_LOG_FILE.$CURRENT_TIME"
|
||
fi
|
||
|
||
check_heavynodes_es_version() {
|
||
# Check if heavynodes are in this grid
|
||
if ! salt-key -l accepted | grep -q 'heavynode$'; then
|
||
|
||
# No heavynodes, skip version check
|
||
echo "No heavynodes detected in this Security Onion deployment. Skipping heavynode Elasticsearch version verification."
|
||
return 0
|
||
fi
|
||
|
||
echo -e "\nOne or more heavynodes detected. Verifying their Elasticsearch versions."
|
||
|
||
local retries=20
|
||
local retry_count=0
|
||
local delay=180
|
||
|
||
while [[ $retry_count -lt $retries ]]; do
|
||
# keep stderr with variable for logging
|
||
heavynode_versions=$(salt -C 'G@role:so-heavynode' cmd.run 'so-elasticsearch-query / --retry 3 --retry-delay 10 | jq ".version.number"' shell=/bin/bash --out=json 2> /dev/null)
|
||
local exit_status=$?
|
||
|
||
# Check that all heavynodes returned good data
|
||
if [[ $exit_status -ne 0 ]]; then
|
||
echo "Failed to retrieve Elasticsearch version from one or more heavynodes... Retrying in $delay seconds. Attempt $((retry_count + 1)) of $retries."
|
||
((retry_count++))
|
||
sleep $delay
|
||
|
||
continue
|
||
else
|
||
if echo "$heavynode_versions" | jq -s --arg expected "\"$EXPECTED_ES_VERSION\"" --exit-status 'all(.[]; . | to_entries | all(.[]; .value == $expected))' > /dev/null; then
|
||
echo -e "\nAll heavynodes are at the expected Elasticsearch version $EXPECTED_ES_VERSION."
|
||
|
||
return 0
|
||
else
|
||
echo "One or more heavynodes are not at the expected Elasticsearch version $EXPECTED_ES_VERSION. Rechecking in $delay seconds. Attempt $((retry_count + 1)) of $retries."
|
||
((retry_count++))
|
||
sleep $delay
|
||
|
||
continue
|
||
fi
|
||
fi
|
||
done
|
||
|
||
echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
|
||
echo "One or more heavynodes is not at the expected Elasticsearch version $EXPECTED_ES_VERSION."
|
||
echo "Current versions:"
|
||
echo "$heavynode_versions" | jq -s 'add'
|
||
echo "$heavynode_versions" | jq -s 'add' >> "$SOUP_INTERMEDIATE_UPGRADE_FAILURES_LOG_FILE"
|
||
echo -e "\n Stopping automatic upgrade to latest Security Onion version. Heavynodes must ALL be at Elasticsearch version $EXPECTED_ES_VERSION before proceeding with the next upgrade step to avoid potential data loss!"
|
||
echo -e "\n Heavynodes will upgrade themselves to Elasticsearch $EXPECTED_ES_VERSION on their own, but this process can take a long time depending on network link between Manager and Heavynodes."
|
||
echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
|
||
|
||
return 1
|
||
}
|
||
|
||
check_searchnodes_es_version() {
|
||
local retries=20
|
||
local retry_count=0
|
||
local delay=180
|
||
|
||
while [[ $retry_count -lt $retries ]]; do
|
||
# keep stderr with variable for logging
|
||
cluster_versions=$(so-elasticsearch-query _nodes/_all/version --retry 5 --retry-delay 10 --fail 2>&1)
|
||
local exit_status=$?
|
||
|
||
if [[ $exit_status -ne 0 ]]; then
|
||
echo "Failed to retrieve Elasticsearch versions from searchnodes... Retrying in $delay seconds. Attempt $((retry_count + 1)) of $retries."
|
||
((retry_count++))
|
||
sleep $delay
|
||
|
||
continue
|
||
else
|
||
if echo "$cluster_versions" | jq --arg expected "$EXPECTED_ES_VERSION" --exit-status '.nodes | to_entries | all(.[].value.version; . == $expected)' > /dev/null; then
|
||
echo "All Searchnodes are at the expected Elasticsearch version $EXPECTED_ES_VERSION."
|
||
|
||
return 0
|
||
else
|
||
echo "One or more Searchnodes is not at the expected Elasticsearch version $EXPECTED_ES_VERSION. Rechecking in $delay seconds. Attempt $((retry_count + 1)) of $retries."
|
||
((retry_count++))
|
||
sleep $delay
|
||
|
||
continue
|
||
fi
|
||
fi
|
||
done
|
||
|
||
echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
|
||
echo "One or more Searchnodes is not at the expected Elasticsearch version $EXPECTED_ES_VERSION."
|
||
echo "Current versions:"
|
||
echo "$cluster_versions" | jq '.nodes | to_entries | map({(.value.name): .value.version}) | sort | add'
|
||
echo "$cluster_versions" >> "$SOUP_INTERMEDIATE_UPGRADE_FAILURES_LOG_FILE"
|
||
echo -e "\nStopping automatic upgrade to latest version. Searchnodes must ALL be at Elasticsearch version $EXPECTED_ES_VERSION before proceeding with the next upgrade step to avoid potential data loss!"
|
||
echo -e "\nSearchnodes will upgrade themselves to Elasticsearch $EXPECTED_ES_VERSION on their own, but this process can take a while depending on cluster size / network link between Manager and Searchnodes."
|
||
echo -e "\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
|
||
|
||
echo "$cluster_versions" > "$SOUP_INTERMEDIATE_UPGRADE_FAILURES_LOG_FILE"
|
||
|
||
return 1
|
||
|
||
}
|
||
|
||
# Need to add a check for heavynodes and ensure all heavynodes get their own "cluster" upgraded before moving on to final upgrade.
|
||
check_searchnodes_es_version || exit 1
|
||
check_heavynodes_es_version || exit 1
|
||
|
||
# Remove required version state file after successful verification
|
||
rm -f "$2"
|
||
|
||
exit 0
|
||
|
||
EOF
|
||
}
|
||
|
||
# Keeping this block in case we need to do a hotfix that requires salt update
|
||
apply_hotfix() {
|
||
echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)"
|
||
}
|
||
|
||
failed_soup_restore_items() {
|
||
local services=("$cron_service_name" "salt-master" "salt-minion")
|
||
for SERVICE_NAME in "${services[@]}"; do
|
||
if ! systemctl is-active --quiet "$SERVICE_NAME"; then
|
||
systemctl_func "start" "$SERVICE_NAME"
|
||
fi
|
||
done
|
||
enable_highstate
|
||
masterunlock
|
||
}
|
||
|
||
main() {
|
||
trap 'check_err $?' EXIT
|
||
|
||
if [ -n "$BRANCH" ]; then
|
||
echo "SOUP will use the $BRANCH branch."
|
||
echo ""
|
||
fi
|
||
|
||
echo "### Preparing soup at $(date) ###"
|
||
echo ""
|
||
set_os
|
||
|
||
if [[ ! $is_oracle ]]; then
|
||
fail "This OS is not supported. Security Onion requires Oracle Linux 9."
|
||
fi
|
||
|
||
check_salt_master_status 1 || fail "Could not talk to salt master: Please run 'systemctl status salt-master' to ensure the salt-master service is running and check the log at /opt/so/log/salt/master."
|
||
|
||
echo "Checking to see if this is a manager."
|
||
echo ""
|
||
require_manager
|
||
|
||
failed_soup_restore_items
|
||
|
||
check_pillar_items
|
||
|
||
echo "Checking to see if this is an airgap install."
|
||
echo ""
|
||
check_airgap
|
||
if [[ $is_airgap -eq 0 && $UNATTENDED == true && -z $ISOLOC ]]; then
|
||
echo "Missing file argument (-f <FILENAME>) for unattended airgap upgrade."
|
||
exit 0
|
||
fi
|
||
|
||
set_minionid
|
||
MINION_ROLE=$(lookup_role)
|
||
echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
|
||
echo ""
|
||
check_minimum_version
|
||
|
||
if [[ $is_airgap -eq 0 ]]; then
|
||
# Let's mount the ISO since this is airgap
|
||
airgap_mounted
|
||
else
|
||
# if not airgap but -f was used
|
||
if [[ ! -z "$ISOLOC" ]]; then
|
||
airgap_mounted
|
||
AGDOCKER=/tmp/soagupdate/docker
|
||
fi
|
||
echo "Cloning Security Onion github repo into $UPDATE_DIR."
|
||
echo "Removing previous upgrade sources."
|
||
rm -rf $UPDATE_DIR
|
||
echo "Cloning the Security Onion Repo."
|
||
clone_to_tmp
|
||
fi
|
||
echo "Verifying we have the latest soup script."
|
||
verify_latest_update_script
|
||
|
||
echo "Verifying Elasticsearch version compatibility before upgrading."
|
||
verify_es_version_compatibility
|
||
|
||
echo "Let's see if we need to update Security Onion."
|
||
upgrade_check
|
||
upgrade_space
|
||
|
||
echo "Checking for Salt Master and Minion updates."
|
||
upgrade_check_salt
|
||
set -e
|
||
|
||
if [[ $is_airgap -eq 0 ]]; then
|
||
update_airgap_repo
|
||
dnf clean all
|
||
check_os_updates
|
||
elif [[ $OS == 'oracle' ]]; then
|
||
# sync remote repo down to local if not airgap
|
||
repo_sync
|
||
dnf clean all
|
||
check_os_updates
|
||
fi
|
||
|
||
if [ "$is_hotfix" == "true" ]; then
|
||
echo "Applying $HOTFIXVERSION hotfix"
|
||
# since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars
|
||
if [[ ! "$MINION_ROLE" == "import" ]]; then
|
||
backup_old_states_pillars
|
||
fi
|
||
copy_new_files
|
||
create_local_directories "/opt/so/saltstack/default"
|
||
apply_hotfix
|
||
echo "Hotfix applied"
|
||
update_version
|
||
enable_highstate
|
||
highstate
|
||
else
|
||
echo ""
|
||
echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
|
||
echo ""
|
||
|
||
systemctl_func "stop" "$cron_service_name"
|
||
|
||
echo "Updating dockers to $NEWVERSION."
|
||
if [[ $is_airgap -eq 0 ]]; then
|
||
airgap_update_dockers
|
||
# if not airgap but -f was used
|
||
elif [[ ! -z "$ISOLOC" ]]; then
|
||
airgap_update_dockers
|
||
unmount_update
|
||
else
|
||
update_registry
|
||
set +e
|
||
update_docker_containers 'soup' '' '' '/dev/stdout' 2>&1
|
||
set -e
|
||
fi
|
||
|
||
stop_salt_minion
|
||
|
||
stop_salt_master
|
||
|
||
#update_repo
|
||
|
||
# Does salt need upgraded. If so update it.
|
||
if [[ $UPGRADESALT -eq 1 ]]; then
|
||
echo "Upgrading Salt"
|
||
# Update the repo files so it can actually upgrade
|
||
upgrade_salt
|
||
fi
|
||
|
||
preupgrade_changes
|
||
echo ""
|
||
|
||
if [[ $is_airgap -eq 0 ]]; then
|
||
echo "Updating Rule Files to the Latest."
|
||
update_airgap_rules
|
||
echo "Updating Playbooks to the Latest."
|
||
airgap_playbooks "$UPDATE_DIR"
|
||
fi
|
||
|
||
# since we don't run the backup.config_backup state on import we wont snapshot previous version states and pillars
|
||
if [[ ! "$MINION_ROLE" == "import" ]]; then
|
||
echo ""
|
||
echo "Creating snapshots of default and local Salt states and pillars and saving to /nsm/backup/"
|
||
backup_old_states_pillars
|
||
fi
|
||
|
||
echo ""
|
||
echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR."
|
||
copy_new_files
|
||
echo ""
|
||
create_local_directories "/opt/so/saltstack/default"
|
||
update_version
|
||
|
||
echo ""
|
||
echo "Locking down Salt Master for upgrade at $(date +"%T.%6N")."
|
||
masterlock
|
||
|
||
systemctl_func "start" "salt-master"
|
||
|
||
# Testing that salt-master is up by checking that is it connected to itself
|
||
check_saltmaster_status
|
||
|
||
# update the salt-minion configs here and start the minion
|
||
# since highstate are disabled above, minion start should not trigger a highstate
|
||
echo ""
|
||
echo "Ensuring salt-minion configs are up-to-date."
|
||
salt-call state.apply salt.minion -l info queue=True
|
||
echo ""
|
||
|
||
# ensure the mine is updated and populated before highstates run, following the salt-master restart
|
||
update_salt_mine
|
||
|
||
if [[ $SALT_CLOUD_CONFIGURED == true && $SALTUPGRADED == true ]]; then
|
||
echo "Updating salt-cloud config to use the new Salt version"
|
||
salt-call state.apply salt.cloud.config concurrent=True
|
||
fi
|
||
|
||
enable_highstate
|
||
|
||
echo ""
|
||
echo "Running a highstate. This could take several minutes."
|
||
set +e
|
||
wait_for_salt_minion_with_restart "$MINIONID" "60" "3" "$SOUP_LOG" || fail "Salt minion was not running or ready."
|
||
highstate
|
||
set -e
|
||
|
||
stop_salt_master
|
||
|
||
masterunlock
|
||
|
||
systemctl_func "start" "salt-master"
|
||
|
||
check_saltmaster_status
|
||
|
||
echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
|
||
wait_for_salt_minion_with_restart "$MINIONID" "60" "3" "$SOUP_LOG" || fail "Salt minion was not running or ready."
|
||
|
||
# Stop long-running scripts to allow potentially updated scripts to load on the next execution.
|
||
if pgrep salt-relay.sh > /dev/null 2>&1; then
|
||
echo "Stopping salt-relay.sh"
|
||
killall salt-relay.sh
|
||
else
|
||
echo "salt-relay.sh is not running"
|
||
fi
|
||
|
||
# ensure the mine is updated and populated before highstates run, following the salt-master restart
|
||
update_salt_mine
|
||
|
||
highstate
|
||
check_saltmaster_status
|
||
postupgrade_changes
|
||
[[ $is_airgap -eq 0 ]] && unmount_update
|
||
|
||
echo ""
|
||
echo "Upgrade to $NEWVERSION complete."
|
||
|
||
# Everything beyond this is post-upgrade checking, don't fail past this point if something here causes an error
|
||
set +e
|
||
|
||
echo "Checking the number of minions."
|
||
NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | grep -v adv_ | wc -l)
|
||
if [[ $UPGRADESALT -eq 1 ]] && [[ $NUM_MINIONS -gt 1 ]]; then
|
||
if [[ $is_airgap -eq 0 ]]; then
|
||
echo ""
|
||
echo "Cleaning repos on remote Security Onion nodes."
|
||
salt -C 'not *_eval and not *_manager* and not *_standalone and G@os:OEL' cmd.run "dnf clean all"
|
||
echo ""
|
||
fi
|
||
fi
|
||
|
||
#echo "Checking for local modifications."
|
||
#check_local_mods
|
||
|
||
echo "Checking sudoers file."
|
||
check_sudoers
|
||
|
||
systemctl_func "start" "$cron_service_name"
|
||
|
||
if [[ -n $lsl_msg ]]; then
|
||
case $lsl_msg in
|
||
'distributed')
|
||
echo "[INFO] The value of log_size_limit in any heavy node minion pillars may be incorrect."
|
||
echo " -> We recommend checking and adjusting the values as necessary."
|
||
echo " -> Minion pillar directory: /opt/so/saltstack/local/pillar/minions/"
|
||
;;
|
||
'single-node')
|
||
# We can assume the lsl_details array has been set if lsl_msg has this value
|
||
echo "[WARNING] The value of log_size_limit (${lsl_details[0]}) does not match the recommended value of ${lsl_details[1]}."
|
||
echo " -> We recommend checking and adjusting the value as necessary."
|
||
echo " -> File: /opt/so/saltstack/local/pillar/minions/${lsl_details[2]}.sls"
|
||
;;
|
||
esac
|
||
fi
|
||
|
||
if [[ $NUM_MINIONS -gt 1 ]]; then
|
||
|
||
cat << EOF
|
||
|
||
|
||
|
||
This appears to be a distributed deployment. Other nodes should update themselves at the next Salt highstate (typically within 15 minutes). Do not manually restart anything until you know that all the search/heavy nodes in your deployment are updated. This is especially important if you are using true clustering for Elasticsearch.
|
||
|
||
Each minion is on a random 15 minute check-in period and things like network bandwidth can be a factor in how long the actual upgrade takes. If you have a heavy node on a slow link, it is going to take a while to get the containers to it. Depending on what changes happened between the versions, Elasticsearch might not be able to talk to said heavy node until the update is complete.
|
||
|
||
If it looks like you’re missing data after the upgrade, please avoid restarting services and instead make sure at least one search node has completed its upgrade. The best way to do this is to run 'sudo salt-call state.highstate' from a search node and make sure there are no errors. Typically if it works on one node it will work on the rest. Sensor nodes are less complex and will update as they check in so you can monitor those from the Grid section of SOC.
|
||
|
||
For more information, please see $DOC_BASE_URL/soup#distributed-deployments.
|
||
|
||
EOF
|
||
|
||
fi
|
||
fi
|
||
|
||
if [ "$NOTIFYCUSTOMELASTICCONFIG" = true ] ; then
|
||
|
||
cat << EOF
|
||
|
||
|
||
A custom Elasticsearch configuration has been found at /opt/so/saltstack/local/elasticsearch/files/elasticsearch.yml. This file is no longer referenced in Security Onion versions >= 2.3.80.
|
||
|
||
If you still need those customizations, you'll need to manually migrate them to the new Elasticsearch config as shown at $DOC_BASE_URL/elasticsearch.
|
||
|
||
EOF
|
||
|
||
fi
|
||
|
||
# check if the FINAL_MESSAGE_QUEUE is not empty
|
||
if (( ${#FINAL_MESSAGE_QUEUE[@]} != 0 )); then
|
||
echo "The following additional information applies specifically to your grid:"
|
||
for m in "${FINAL_MESSAGE_QUEUE[@]}"; do
|
||
echo "$m"
|
||
echo
|
||
done
|
||
fi
|
||
|
||
echo "### soup has been served at $(date) ###"
|
||
}
|
||
|
||
while getopts ":b:f:y" opt; do
|
||
case ${opt} in
|
||
b )
|
||
BATCHSIZE="$OPTARG"
|
||
if ! [[ "$BATCHSIZE" =~ ^[1-9][0-9]*$ ]]; then
|
||
echo "Batch size must be a number greater than 0."
|
||
exit 1
|
||
fi
|
||
;;
|
||
y )
|
||
if [[ ! -f /opt/so/state/yeselastic.txt ]]; then
|
||
echo "Cannot run soup in unattended mode. You must run soup manually to accept the Elastic License."
|
||
exit 1
|
||
else
|
||
UNATTENDED=true
|
||
fi
|
||
;;
|
||
f )
|
||
ISOLOC="$OPTARG"
|
||
;;
|
||
\? )
|
||
echo "Usage: soup [-b] [-y] [-f <iso location>]"
|
||
exit 1
|
||
;;
|
||
: )
|
||
echo "Invalid option: $OPTARG requires an argument"
|
||
exit 1
|
||
;;
|
||
esac
|
||
done
|
||
shift $((OPTIND - 1))
|
||
|
||
if [ -f $SOUP_LOG ]; then
|
||
CURRENT_TIME=$(date +%Y%m%d.%H%M%S)
|
||
mv $SOUP_LOG $SOUP_LOG.$INSTALLEDVERSION.$CURRENT_TIME
|
||
fi
|
||
|
||
if [[ -z $UNATTENDED ]]; then
|
||
cat << EOF
|
||
|
||
SOUP - Security Onion UPdater
|
||
|
||
Please review the following for more information about the update process and recent updates:
|
||
$DOC_BASE_URL/soup
|
||
https://blog.securityonion.net
|
||
|
||
WARNING: If you run soup via an SSH session and that SSH session terminates, then any processes running in that session would terminate. You should avoid leaving soup unattended especially if the machine you are SSHing from is configured to sleep after a period of time. You might also consider using something like screen or tmux so that if your SSH session terminates, the processes will continue running on the server.
|
||
|
||
EOF
|
||
|
||
cat << EOF
|
||
Press Enter to continue or Ctrl-C to cancel.
|
||
EOF
|
||
|
||
read -r input
|
||
fi
|
||
|
||
main "$@" | tee -a $SOUP_LOG
|