mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2025-12-06 17:22:49 +01:00
Merge pull request #4289 from Security-Onion-Solutions/foxtrot
Soup error handling, reorder sensoroni state
This commit is contained in:
William Wernert
GitHub
@@ -302,6 +302,7 @@ retry() {
|
||||
cmd=$3
|
||||
expectedOutput=$4
|
||||
attempt=0
|
||||
local exitcode=0
|
||||
while [[ $attempt -lt $maxAttempts ]]; do
|
||||
attempt=$((attempt+1))
|
||||
echo "Executing command with retry support: $cmd"
|
||||
@@ -321,7 +322,29 @@ retry() {
|
||||
sleep $sleepDelay
|
||||
done
|
||||
echo "Command continues to fail; giving up."
|
||||
return 1
|
||||
return $exitcode
|
||||
}
|
||||
|
||||
run_check_net_err() {
|
||||
local cmd=$1
|
||||
local err_msg=${2:-"Unknown error occured, please check /root/$WHATWOULDYOUSAYYAHDOHERE.log for details."} # Really need to rename that variable
|
||||
local no_retry=$3
|
||||
|
||||
local exit_code
|
||||
if [[ -z $no_retry ]]; then
|
||||
retry 5 60 "$cmd"
|
||||
exit_code=$?
|
||||
else
|
||||
eval "$cmd"
|
||||
exit_code=$?
|
||||
fi
|
||||
|
||||
if [[ $exit_code -ne 0 ]]; then
|
||||
ERR_HANDLED=true
|
||||
[[ -z $no_retry ]] || echo "Command failed with error $exit_code"
|
||||
echo "$err_msg"
|
||||
exit $exit_code
|
||||
fi
|
||||
}
|
||||
|
||||
set_os() {
|
||||
|
||||
@@ -32,13 +32,15 @@ def get_image_version(string) -> str:
|
||||
ver = string.split(':')[-1]
|
||||
if ver == 'latest':
|
||||
# Version doesn't like "latest", so use a high semver
|
||||
return '999999.9.9'
|
||||
return '99999.9.9'
|
||||
else:
|
||||
try:
|
||||
Version(ver)
|
||||
except InvalidVersion:
|
||||
# Strip the last substring following a hyphen for automated branches
|
||||
ver = '-'.join(ver.split('-')[:-1])
|
||||
# Also return a very high semver for any version
|
||||
# with a dash in it since it will likely be a dev version of some kind
|
||||
if '-' in ver:
|
||||
return '999999.9.9'
|
||||
return ver
|
||||
|
||||
|
||||
@@ -73,8 +75,12 @@ def main(quiet):
|
||||
for tag in group:
|
||||
if not quiet: print(f'Removing image {tag}')
|
||||
client.images.remove(tag)
|
||||
except InvalidVersion as e:
|
||||
print(f'so-{get_so_image_basename(t_list[0])}: {e.args[0]}', file=sys.stderr)
|
||||
except (docker.errors.APIError, InvalidVersion) as e:
|
||||
print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
|
||||
exit(1)
|
||||
except Exception as e:
|
||||
print('Unhandled exception occurred:')
|
||||
print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
|
||||
exit(1)
|
||||
|
||||
if no_prunable and not quiet:
|
||||
@@ -86,4 +92,4 @@ if __name__ == "__main__":
|
||||
main_parser.add_argument('-q', '--quiet', action='store_const', const=True, required=False)
|
||||
args = main_parser.parse_args(sys.argv[1:])
|
||||
|
||||
main(args.quiet)
|
||||
main(args.quiet)
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
# NOTE: This script depends on so-common
|
||||
IMAGEREPO=security-onion-solutions
|
||||
|
||||
# shellcheck disable=SC2120
|
||||
container_list() {
|
||||
MANAGERCHECK=$1
|
||||
|
||||
@@ -128,13 +129,13 @@ update_docker_containers() {
|
||||
mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1
|
||||
|
||||
# Let's make sure we have the public key
|
||||
retry 50 10 "curl -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" >> "$LOG_FILE" 2>&1
|
||||
run_check_net_err \
|
||||
"curl --retry 5 --retry-delay 60 -sSL https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -o $SIGNPATH/KEYS" \
|
||||
"Could not pull signature key file, please ensure connectivity to https://raw.gihubusercontent.com" \
|
||||
noretry >> "$LOG_FILE" 2>&1
|
||||
result=$?
|
||||
if [[ $result -eq 0 ]]; then
|
||||
cat $SIGNPATH/KEYS | gpg --import - >> "$LOG_FILE" 2>&1
|
||||
else
|
||||
echo "Failed to pull signature key file: $result"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Download the containers from the interwebs
|
||||
@@ -148,14 +149,15 @@ update_docker_containers() {
|
||||
|
||||
# Pull down the trusted docker image
|
||||
local image=$i:$VERSION$IMAGE_TAG_SUFFIX
|
||||
retry 50 10 "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" >> "$LOG_FILE" 2>&1
|
||||
run_check_net_err \
|
||||
"docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" \
|
||||
"Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1
|
||||
|
||||
# Get signature
|
||||
retry 50 10 "curl -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" >> "$LOG_FILE" 2>&1
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "Unable to pull signature file for $image" >> "$LOG_FILE" 2>&1
|
||||
exit 1
|
||||
fi
|
||||
run_check_net_err \
|
||||
"curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \
|
||||
"Could not pull signature file for $image, please ensure connectivity to https://sigs.securityonion.net " \
|
||||
noretry >> "$LOG_FILE" 2>&1
|
||||
# Dump our hash values
|
||||
DOCKERINSPECT=$(docker inspect $CONTAINER_REGISTRY/$IMAGEREPO/$image)
|
||||
|
||||
|
||||
@@ -20,12 +20,79 @@
|
||||
UPDATE_DIR=/tmp/sogh/securityonion
|
||||
INSTALLEDVERSION=$(cat /etc/soversion)
|
||||
POSTVERSION=$INSTALLEDVERSION
|
||||
INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'})
|
||||
INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
|
||||
BATCHSIZE=5
|
||||
SOUP_LOG=/root/soup.log
|
||||
INFLUXDB_MIGRATION_LOG=/opt/so/log/influxdb/soup_migration.log
|
||||
WHATWOULDYOUSAYYAHDOHERE=soup
|
||||
|
||||
check_err() {
|
||||
local exit_code=$1
|
||||
local err_msg="Unhandled error occured, please check $SOUP_LOG for details."
|
||||
|
||||
[[ $ERR_HANDLED == true ]] && exit $exit_code
|
||||
if [[ $exit_code -ne 0 ]]; then
|
||||
printf '%s' "Soup failed with error $exit_code: "
|
||||
case $exit_code in
|
||||
2)
|
||||
echo 'No such file or directory'
|
||||
;;
|
||||
5)
|
||||
echo 'Interrupted system call'
|
||||
;;
|
||||
12)
|
||||
echo 'Out of memory'
|
||||
;;
|
||||
28)
|
||||
echo 'No space left on device'
|
||||
echo 'Likely ran out of space on disk, please review hardware requirements for Security Onion: https://docs.securityonion.net/en/2.3/hardware.html'
|
||||
;;
|
||||
30)
|
||||
echo 'Read-only file system'
|
||||
;;
|
||||
35)
|
||||
echo 'Resource temporarily unavailable'
|
||||
;;
|
||||
64)
|
||||
echo 'Machine is not on the network'
|
||||
;;
|
||||
67)
|
||||
echo 'Link has been severed'
|
||||
;;
|
||||
100)
|
||||
echo 'Network is down'
|
||||
;;
|
||||
101)
|
||||
echo 'Network is unreachable'
|
||||
;;
|
||||
102)
|
||||
echo 'Network reset'
|
||||
;;
|
||||
110)
|
||||
echo 'Connection timed out'
|
||||
;;
|
||||
111)
|
||||
echo 'Connection refused'
|
||||
;;
|
||||
112)
|
||||
echo 'Host is down'
|
||||
;;
|
||||
113)
|
||||
echo 'No route to host'
|
||||
;;
|
||||
*)
|
||||
echo 'Unhandled error'
|
||||
echo "$err_msg"
|
||||
;;
|
||||
esac
|
||||
if [[ $exit_code -ge 64 && $exit_code -le 113 ]]; then
|
||||
echo "$err_msg"
|
||||
fi
|
||||
exit $exit_code
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
add_common() {
|
||||
cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
|
||||
cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
|
||||
@@ -47,8 +114,8 @@ airgap_mounted() {
|
||||
echo "Example: /home/user/securityonion-2.X.0.iso"
|
||||
echo "Example: /dev/sdx1"
|
||||
echo ""
|
||||
read -p 'Enter the location of the iso: ' ISOLOC
|
||||
if [ -f $ISOLOC ]; then
|
||||
read -rp 'Enter the location of the iso: ' ISOLOC
|
||||
if [[ -f $ISOLOC ]]; then
|
||||
# Mounting the ISO image
|
||||
mkdir -p /tmp/soagupdate
|
||||
mount -t iso9660 -o loop $ISOLOC /tmp/soagupdate
|
||||
@@ -60,7 +127,7 @@ airgap_mounted() {
|
||||
else
|
||||
echo "ISO has been mounted!"
|
||||
fi
|
||||
elif [ -f $ISOLOC/SecurityOnion/VERSION ]; then
|
||||
elif [[ -f $ISOLOC/SecurityOnion/VERSION ]]; then
|
||||
ln -s $ISOLOC /tmp/soagupdate
|
||||
echo "Found the update content"
|
||||
else
|
||||
@@ -78,9 +145,9 @@ airgap_mounted() {
|
||||
}
|
||||
|
||||
airgap_update_dockers() {
|
||||
if [ $is_airgap -eq 0 ]; then
|
||||
if [[ $is_airgap -eq 0 ]]; then
|
||||
# Let's copy the tarball
|
||||
if [ ! -f $AGDOCKER/registry.tar ]; then
|
||||
if [[ ! -f $AGDOCKER/registry.tar ]]; then
|
||||
echo "Unable to locate registry. Exiting"
|
||||
exit 1
|
||||
else
|
||||
@@ -88,9 +155,9 @@ airgap_update_dockers() {
|
||||
docker stop so-dockerregistry
|
||||
docker rm so-dockerregistry
|
||||
echo "Copying the new dockers over"
|
||||
tar xvf $AGDOCKER/registry.tar -C /nsm/docker-registry/docker
|
||||
tar xvf "$AGDOCKER/registry.tar" -C /nsm/docker-registry/docker
|
||||
echo "Add Registry back"
|
||||
docker load -i $AGDOCKER/registry_image.tar
|
||||
docker load -i "$AGDOCKER/registry_image.tar"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
@@ -102,9 +169,9 @@ update_registry() {
|
||||
}
|
||||
|
||||
check_sudoers() {
|
||||
if grep -q "so-setup" /etc/sudoers; then
|
||||
echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"."
|
||||
fi
|
||||
if grep -q "so-setup" /etc/sudoers; then
|
||||
echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"."
|
||||
fi
|
||||
}
|
||||
|
||||
check_log_size_limit() {
|
||||
@@ -178,7 +245,9 @@ check_os_updates() {
|
||||
echo "Continuing without updating packages"
|
||||
elif [[ "$confirm" == [uU] ]]; then
|
||||
echo "Applying Grid Updates"
|
||||
salt \* -b 5 state.apply patch.os queue=True
|
||||
set +e
|
||||
run_check_net_err "salt '*' -b 5 state.apply patch.os queue=True" 'Could not apply OS updates, please check your network connection.'
|
||||
set -e
|
||||
else
|
||||
echo "Exiting soup"
|
||||
exit 0
|
||||
@@ -206,7 +275,9 @@ clone_to_tmp() {
|
||||
if [ -n "$BRANCH" ]; then
|
||||
SOUP_BRANCH="-b $BRANCH"
|
||||
fi
|
||||
git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git
|
||||
set +e
|
||||
run_check_net_err "git clone $SOUP_BRANCH https://github.com/Security-Onion-Solutions/securityonion.git" "Could not clone repo, please ensure network access to https://github.com"
|
||||
set -e
|
||||
cd /tmp
|
||||
if [ ! -f $UPDATE_DIR/VERSION ]; then
|
||||
echo "Update was unable to pull from github. Please check your internet."
|
||||
@@ -253,6 +324,7 @@ preupgrade_changes_2.3.50_repo() {
|
||||
echo "Checking to see if 2.3.50 repo changes are needed."
|
||||
|
||||
[[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50_repo
|
||||
true
|
||||
}
|
||||
|
||||
preupgrade_changes() {
|
||||
@@ -265,6 +337,7 @@ preupgrade_changes() {
|
||||
[[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20
|
||||
[[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30
|
||||
[[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50
|
||||
true
|
||||
}
|
||||
|
||||
postupgrade_changes() {
|
||||
@@ -275,6 +348,7 @@ postupgrade_changes() {
|
||||
[[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30
|
||||
[[ "$POSTVERSION" == 2.3.30 ]] && post_2.3.30_to_2.3.40
|
||||
[[ "$POSTVERSION" == 2.3.50 ]] && post_2.3.5X_to_2.3.60
|
||||
true
|
||||
}
|
||||
|
||||
post_rc1_to_2.3.21() {
|
||||
@@ -432,7 +506,7 @@ up_2.3.2X_to_2.3.30() {
|
||||
sed -i "/ imagerepo: securityonion/c\ imagerepo: 'security-onion-solutions'" /opt/so/saltstack/local/pillar/global.sls
|
||||
|
||||
# Strelka rule repo pillar addition
|
||||
if [ $is_airgap -eq 0 ]; then
|
||||
if [[ $is_airgap -eq 0 ]]; then
|
||||
# Add manager as default Strelka YARA rule repo
|
||||
sed -i "/^strelka:/a \\ repos: \n - https://$HOSTNAME/repo/rules/strelka" /opt/so/saltstack/local/pillar/global.sls;
|
||||
else
|
||||
@@ -459,7 +533,7 @@ up_2.3.3X_to_2.3.50_repo() {
|
||||
rm -f "/etc/yum.repos.d/$DELREPO.repo"
|
||||
fi
|
||||
done
|
||||
if [ $is_airgap -eq 1 ]; then
|
||||
if [[ $is_airgap -eq 1 ]]; then
|
||||
# Copy the new repo file if not airgap
|
||||
cp $UPDATE_DIR/salt/repo/client/files/centos/securityonion.repo /etc/yum.repos.d/
|
||||
yum clean all
|
||||
@@ -575,7 +649,7 @@ upgrade_check() {
|
||||
# Let's make sure we actually need to update.
|
||||
NEWVERSION=$(cat $UPDATE_DIR/VERSION)
|
||||
HOTFIXVERSION=$(cat $UPDATE_DIR/HOTFIX)
|
||||
CURRENTHOTFIX=$(cat /etc/sohotfix 2>/dev/null)
|
||||
[[ -f /etc/sohotfix ]] && CURRENTHOTFIX=$(cat /etc/sohotfix)
|
||||
if [ "$INSTALLEDVERSION" == "$NEWVERSION" ]; then
|
||||
echo "Checking to see if there are hotfixes needed"
|
||||
if [ "$HOTFIXVERSION" == "$CURRENTHOTFIX" ]; then
|
||||
@@ -592,13 +666,14 @@ upgrade_check() {
|
||||
}
|
||||
|
||||
upgrade_check_salt() {
|
||||
NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk {'print $2'})
|
||||
NEWSALTVERSION=$(grep version: $UPDATE_DIR/salt/salt/master.defaults.yaml | awk '{print $2}')
|
||||
if [ "$INSTALLEDSALTVERSION" == "$NEWSALTVERSION" ]; then
|
||||
echo "You are already running the correct version of Salt for Security Onion."
|
||||
else
|
||||
UPGRADESALT=1
|
||||
fi
|
||||
}
|
||||
|
||||
upgrade_salt() {
|
||||
SALTUPGRADED=True
|
||||
echo "Performing upgrade of Salt from $INSTALLEDSALTVERSION to $NEWSALTVERSION."
|
||||
@@ -610,7 +685,11 @@ upgrade_salt() {
|
||||
yum versionlock delete "salt-*"
|
||||
echo "Updating Salt packages and restarting services."
|
||||
echo ""
|
||||
sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable "$NEWSALTVERSION"
|
||||
set +e
|
||||
run_check_net_err \
|
||||
"sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -r -F -M -x python3 stable \"$NEWSALTVERSION\"" \
|
||||
"Could not update salt, please check $SOUP_LOG for details."
|
||||
set -e
|
||||
echo "Applying yum versionlock for Salt."
|
||||
echo ""
|
||||
yum versionlock add "salt-*"
|
||||
@@ -623,7 +702,11 @@ upgrade_salt() {
|
||||
apt-mark unhold "salt-minion"
|
||||
echo "Updating Salt packages and restarting services."
|
||||
echo ""
|
||||
sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable "$NEWSALTVERSION"
|
||||
set +e
|
||||
run_check_net_err \
|
||||
"sh $UPDATE_DIR/salt/salt/scripts/bootstrap-salt.sh -F -M -x python3 stable \"$NEWSALTVERSION\"" \
|
||||
"Could not update salt, please check $SOUP_LOG for details."
|
||||
set -e
|
||||
echo "Applying apt hold for Salt."
|
||||
echo ""
|
||||
apt-mark hold "salt-common"
|
||||
@@ -648,234 +731,244 @@ verify_latest_update_script() {
|
||||
cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/
|
||||
cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
|
||||
cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
|
||||
salt-call state.apply common queue=True
|
||||
salt-call state.apply -l info common queue=True
|
||||
echo ""
|
||||
echo "soup has been updated. Please run soup again."
|
||||
exit 0
|
||||
fi
|
||||
}
|
||||
|
||||
main () {
|
||||
echo "### Preparing soup at `date` ###"
|
||||
while getopts ":b" opt; do
|
||||
case "$opt" in
|
||||
b ) # process option b
|
||||
shift
|
||||
BATCHSIZE=$1
|
||||
if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then
|
||||
echo "Batch size must be a number greater than 0."
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
\? )
|
||||
echo "Usage: cmd [-b]"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "Checking to see if this is a manager."
|
||||
echo ""
|
||||
require_manager
|
||||
set_minionid
|
||||
echo "Checking to see if this is an airgap install"
|
||||
echo ""
|
||||
check_airgap
|
||||
echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
|
||||
echo ""
|
||||
set_os
|
||||
set_palette
|
||||
check_elastic_license
|
||||
echo ""
|
||||
if [ $is_airgap -eq 0 ]; then
|
||||
# Let's mount the ISO since this is airgap
|
||||
airgap_mounted
|
||||
else
|
||||
echo "Cloning Security Onion github repo into $UPDATE_DIR."
|
||||
echo "Removing previous upgrade sources."
|
||||
rm -rf $UPDATE_DIR
|
||||
clone_to_tmp
|
||||
fi
|
||||
check_os_updates
|
||||
echo ""
|
||||
echo "Verifying we have the latest soup script."
|
||||
verify_latest_update_script
|
||||
echo ""
|
||||
|
||||
echo "Generating new repo archive"
|
||||
generate_and_clean_tarballs
|
||||
if [ -f /usr/sbin/so-image-common ]; then
|
||||
. /usr/sbin/so-image-common
|
||||
else
|
||||
add_common
|
||||
fi
|
||||
|
||||
echo "Let's see if we need to update Security Onion."
|
||||
upgrade_check
|
||||
upgrade_space
|
||||
|
||||
echo "Checking for Salt Master and Minion updates."
|
||||
upgrade_check_salt
|
||||
|
||||
|
||||
if [ "$is_hotfix" == "true" ]; then
|
||||
echo "Applying $HOTFIXVERSION"
|
||||
copy_new_files
|
||||
echo ""
|
||||
update_version
|
||||
salt-call state.highstate -l info queue=True
|
||||
main() {
|
||||
set -e
|
||||
trap 'check_err $?' EXIT
|
||||
|
||||
else
|
||||
echo ""
|
||||
echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
|
||||
echo ""
|
||||
|
||||
echo "Updating dockers to $NEWVERSION."
|
||||
if [ $is_airgap -eq 0 ]; then
|
||||
airgap_update_dockers
|
||||
update_centos_repo
|
||||
yum clean all
|
||||
check_os_updates
|
||||
else
|
||||
update_registry
|
||||
update_docker_containers "soup"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Stopping Salt Minion service."
|
||||
systemctl stop salt-minion
|
||||
echo "Killing any remaining Salt Minion processes."
|
||||
pkill -9 -ef /usr/bin/salt-minion
|
||||
echo ""
|
||||
echo "Stopping Salt Master service."
|
||||
systemctl stop salt-master
|
||||
echo ""
|
||||
|
||||
preupgrade_changes_2.3.50_repo
|
||||
|
||||
# Does salt need upgraded. If so update it.
|
||||
if [ "$UPGRADESALT" == "1" ]; then
|
||||
echo "Upgrading Salt"
|
||||
# Update the repo files so it can actually upgrade
|
||||
upgrade_salt
|
||||
fi
|
||||
|
||||
echo "Checking if Salt was upgraded."
|
||||
echo ""
|
||||
# Check that Salt was upgraded
|
||||
SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk {'print $2'})
|
||||
if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then
|
||||
echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG."
|
||||
echo "Once the issue is resolved, run soup again."
|
||||
echo "Exiting."
|
||||
echo ""
|
||||
exit 1
|
||||
else
|
||||
echo "Salt upgrade success."
|
||||
echo ""
|
||||
fi
|
||||
|
||||
preupgrade_changes
|
||||
echo ""
|
||||
|
||||
if [ $is_airgap -eq 0 ]; then
|
||||
echo "Updating Rule Files to the Latest."
|
||||
update_airgap_rules
|
||||
fi
|
||||
|
||||
# Only update the repo if its airgap
|
||||
if [[ $is_airgap -eq 0 ]] && [[ "$UPGRADESALT" != "1" ]]; then
|
||||
update_centos_repo
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR."
|
||||
copy_new_files
|
||||
echo ""
|
||||
update_version
|
||||
|
||||
echo ""
|
||||
echo "Locking down Salt Master for upgrade"
|
||||
masterlock
|
||||
|
||||
echo ""
|
||||
echo "Starting Salt Master service."
|
||||
systemctl start salt-master
|
||||
|
||||
# Testing that that salt-master is up by checking that is it connected to itself
|
||||
retry 50 10 "salt-call state.show_top -l error" || exit 1
|
||||
|
||||
echo ""
|
||||
echo "Ensuring python modules for Salt are installed and patched."
|
||||
salt-call state.apply salt.python3-influxdb -l info queue=True
|
||||
echo ""
|
||||
|
||||
# Only regenerate osquery packages if Fleet is enabled
|
||||
FLEET_MANAGER=$(lookup_pillar fleet_manager)
|
||||
FLEET_NODE=$(lookup_pillar fleet_node)
|
||||
if [[ "$FLEET_MANAGER" == "True" || "$FLEET_NODE" == "True" ]]; then
|
||||
echo ""
|
||||
echo "Regenerating Osquery Packages.... This will take several minutes."
|
||||
salt-call state.apply fleet.event_gen-packages -l info queue=True
|
||||
echo ""
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
|
||||
salt-call state.highstate -l info queue=True
|
||||
echo ""
|
||||
echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete."
|
||||
|
||||
echo ""
|
||||
echo "Stopping Salt Master to remove ACL"
|
||||
systemctl stop salt-master
|
||||
|
||||
masterunlock
|
||||
|
||||
echo ""
|
||||
echo "Starting Salt Master service."
|
||||
systemctl start salt-master
|
||||
|
||||
# Testing that that salt-master is up by checking that is it connected to itself
|
||||
retry 50 10 "salt-call state.show_top -l error" || exit 1
|
||||
|
||||
echo "Running a highstate. This could take several minutes."
|
||||
salt-call state.highstate -l info queue=True
|
||||
postupgrade_changes
|
||||
unmount_update
|
||||
thehive_maint
|
||||
|
||||
if [ "$UPGRADESALT" == "1" ]; then
|
||||
if [ $is_airgap -eq 0 ]; then
|
||||
echo ""
|
||||
echo "Cleaning repos on remote Security Onion nodes."
|
||||
salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone and G@os:CentOS' cmd.run "yum clean all"
|
||||
echo ""
|
||||
fi
|
||||
fi
|
||||
|
||||
check_sudoers
|
||||
|
||||
if [[ -n $lsl_msg ]]; then
|
||||
case $lsl_msg in
|
||||
'distributed')
|
||||
echo "[INFO] The value of log_size_limit in any heavy node minion pillars may be incorrect."
|
||||
echo " -> We recommend checking and adjusting the values as necessary."
|
||||
echo " -> Minion pillar directory: /opt/so/saltstack/local/pillar/minions/"
|
||||
echo "### Preparing soup at $(date) ###"
|
||||
while getopts ":b" opt; do
|
||||
case "$opt" in
|
||||
b ) # process option b
|
||||
shift
|
||||
BATCHSIZE=$1
|
||||
if ! [[ "$BATCHSIZE" =~ ^[0-9]+$ ]]; then
|
||||
echo "Batch size must be a number greater than 0."
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
'single-node')
|
||||
# We can assume the lsl_details array has been set if lsl_msg has this value
|
||||
echo "[WARNING] The value of log_size_limit (${lsl_details[0]}) does not match the recommended value of ${lsl_details[1]}."
|
||||
echo " -> We recommend checking and adjusting the value as necessary."
|
||||
echo " -> File: /opt/so/saltstack/local/pillar/minions/${lsl_details[2]}.sls"
|
||||
\? )
|
||||
echo "Usage: cmd [-b]"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "Checking to see if this is a manager."
|
||||
echo ""
|
||||
require_manager
|
||||
set_minionid
|
||||
echo "Checking to see if this is an airgap install"
|
||||
echo ""
|
||||
check_airgap
|
||||
echo "Found that Security Onion $INSTALLEDVERSION is currently installed."
|
||||
echo ""
|
||||
set_os
|
||||
set_palette
|
||||
check_elastic_license
|
||||
echo ""
|
||||
if [[ $is_airgap -eq 0 ]]; then
|
||||
# Let's mount the ISO since this is airgap
|
||||
airgap_mounted
|
||||
else
|
||||
echo "Cloning Security Onion github repo into $UPDATE_DIR."
|
||||
echo "Removing previous upgrade sources."
|
||||
rm -rf $UPDATE_DIR
|
||||
clone_to_tmp
|
||||
fi
|
||||
check_os_updates
|
||||
echo ""
|
||||
echo "Verifying we have the latest soup script."
|
||||
verify_latest_update_script
|
||||
echo ""
|
||||
|
||||
echo "Generating new repo archive"
|
||||
generate_and_clean_tarballs
|
||||
if [ -f /usr/sbin/so-image-common ]; then
|
||||
. /usr/sbin/so-image-common
|
||||
else
|
||||
add_common
|
||||
fi
|
||||
|
||||
NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l)
|
||||
echo "Let's see if we need to update Security Onion."
|
||||
upgrade_check
|
||||
upgrade_space
|
||||
|
||||
if [ $NUM_MINIONS -gt 1 ]; then
|
||||
echo "Checking for Salt Master and Minion updates."
|
||||
upgrade_check_salt
|
||||
|
||||
cat << EOF
|
||||
|
||||
if [ "$is_hotfix" == "true" ]; then
|
||||
echo "Applying $HOTFIXVERSION"
|
||||
copy_new_files
|
||||
echo ""
|
||||
update_version
|
||||
salt-call state.highstate -l info queue=True
|
||||
else
|
||||
echo ""
|
||||
echo "Performing upgrade from Security Onion $INSTALLEDVERSION to Security Onion $NEWVERSION."
|
||||
echo ""
|
||||
|
||||
echo "Updating dockers to $NEWVERSION."
|
||||
if [[ $is_airgap -eq 0 ]]; then
|
||||
airgap_update_dockers
|
||||
update_centos_repo
|
||||
yum clean all
|
||||
check_os_updates
|
||||
else
|
||||
update_registry
|
||||
set +e
|
||||
update_docker_containers "soup"
|
||||
set -e
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Stopping Salt Minion service."
|
||||
systemctl stop salt-minion
|
||||
echo "Killing any remaining Salt Minion processes."
|
||||
set +e
|
||||
pkill -9 -ef /usr/bin/salt-minion
|
||||
set -e
|
||||
echo ""
|
||||
echo "Stopping Salt Master service."
|
||||
systemctl stop salt-master
|
||||
echo ""
|
||||
|
||||
preupgrade_changes_2.3.50_repo
|
||||
|
||||
# Does salt need upgraded. If so update it.
|
||||
if [[ $UPGRADESALT -eq 1 ]]; then
|
||||
echo "Upgrading Salt"
|
||||
# Update the repo files so it can actually upgrade
|
||||
upgrade_salt
|
||||
fi
|
||||
|
||||
echo "Checking if Salt was upgraded."
|
||||
echo ""
|
||||
# Check that Salt was upgraded
|
||||
SALTVERSIONPOSTUPGRADE=$(salt --versions-report | grep Salt: | awk '{print $2}')
|
||||
if [[ "$SALTVERSIONPOSTUPGRADE" != "$NEWSALTVERSION" ]]; then
|
||||
echo "Salt upgrade failed. Check of indicators of failure in $SOUP_LOG."
|
||||
echo "Once the issue is resolved, run soup again."
|
||||
echo "Exiting."
|
||||
echo ""
|
||||
exit 1
|
||||
else
|
||||
echo "Salt upgrade success."
|
||||
echo ""
|
||||
fi
|
||||
|
||||
preupgrade_changes
|
||||
echo ""
|
||||
|
||||
if [[ $is_airgap -eq 0 ]]; then
|
||||
echo "Updating Rule Files to the Latest."
|
||||
update_airgap_rules
|
||||
fi
|
||||
|
||||
# Only update the repo if its airgap
|
||||
if [[ $is_airgap -eq 0 && $UPGRADESALT -ne 1 ]]; then
|
||||
update_centos_repo
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Copying new Security Onion code from $UPDATE_DIR to $DEFAULT_SALT_DIR."
|
||||
copy_new_files
|
||||
echo ""
|
||||
update_version
|
||||
|
||||
echo ""
|
||||
echo "Locking down Salt Master for upgrade"
|
||||
masterlock
|
||||
|
||||
echo ""
|
||||
echo "Starting Salt Master service."
|
||||
systemctl start salt-master
|
||||
|
||||
# Testing that salt-master is up by checking that is it connected to itself
|
||||
set +e
|
||||
retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details."
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
echo "Ensuring python modules for Salt are installed and patched."
|
||||
salt-call state.apply salt.python3-influxdb -l info queue=True
|
||||
echo ""
|
||||
|
||||
# Only regenerate osquery packages if Fleet is enabled
|
||||
FLEET_MANAGER=$(lookup_pillar fleet_manager)
|
||||
FLEET_NODE=$(lookup_pillar fleet_node)
|
||||
if [[ "$FLEET_MANAGER" == "True" || "$FLEET_NODE" == "True" ]]; then
|
||||
echo ""
|
||||
echo "Regenerating Osquery Packages.... This will take several minutes."
|
||||
salt-call state.apply fleet.event_gen-packages -l info queue=True
|
||||
echo ""
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes."
|
||||
salt-call state.highstate -l info queue=True
|
||||
echo ""
|
||||
echo "Upgrade from $INSTALLEDVERSION to $NEWVERSION complete."
|
||||
|
||||
echo ""
|
||||
echo "Stopping Salt Master to remove ACL"
|
||||
systemctl stop salt-master
|
||||
|
||||
masterunlock
|
||||
|
||||
echo ""
|
||||
echo "Starting Salt Master service."
|
||||
systemctl start salt-master
|
||||
|
||||
# Testing that salt-master is up by checking that is it connected to itself
|
||||
set +e
|
||||
retry 50 10 "salt-call state.show_top -l error" || fail "salt-master could not be reached. Check $SOUP_LOG for details."
|
||||
set -e
|
||||
|
||||
echo "Running a highstate. This could take several minutes."
|
||||
salt-call state.highstate -l info queue=True
|
||||
postupgrade_changes
|
||||
[[ $is_airgap -eq 0 ]] && unmount_update
|
||||
thehive_maint
|
||||
|
||||
if [[ $UPGRADESALT -eq 1 ]]; then
|
||||
if [[ $is_airgap -eq 0 ]]; then
|
||||
echo ""
|
||||
echo "Cleaning repos on remote Security Onion nodes."
|
||||
salt -C 'not *_eval and not *_helixsensor and not *_manager and not *_managersearch and not *_standalone and G@os:CentOS' cmd.run "yum clean all"
|
||||
echo ""
|
||||
fi
|
||||
fi
|
||||
|
||||
check_sudoers
|
||||
|
||||
if [[ -n $lsl_msg ]]; then
|
||||
case $lsl_msg in
|
||||
'distributed')
|
||||
echo "[INFO] The value of log_size_limit in any heavy node minion pillars may be incorrect."
|
||||
echo " -> We recommend checking and adjusting the values as necessary."
|
||||
echo " -> Minion pillar directory: /opt/so/saltstack/local/pillar/minions/"
|
||||
;;
|
||||
'single-node')
|
||||
# We can assume the lsl_details array has been set if lsl_msg has this value
|
||||
echo "[WARNING] The value of log_size_limit (${lsl_details[0]}) does not match the recommended value of ${lsl_details[1]}."
|
||||
echo " -> We recommend checking and adjusting the value as necessary."
|
||||
echo " -> File: /opt/so/saltstack/local/pillar/minions/${lsl_details[2]}.sls"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
NUM_MINIONS=$(ls /opt/so/saltstack/local/pillar/minions/*_*.sls | wc -l)
|
||||
|
||||
if [[ $NUM_MINIONS -gt 1 ]]; then
|
||||
|
||||
cat << EOF
|
||||
|
||||
|
||||
|
||||
@@ -889,10 +982,10 @@ For more information, please see https://docs.securityonion.net/en/2.3/soup.html
|
||||
|
||||
EOF
|
||||
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "### soup has been served at `date` ###"
|
||||
echo "### soup has been served at $(date) ###"
|
||||
}
|
||||
|
||||
cat << EOF
|
||||
@@ -907,6 +1000,7 @@ Press Enter to continue or Ctrl-C to cancel.
|
||||
|
||||
EOF
|
||||
|
||||
read input
|
||||
read -r input
|
||||
|
||||
main "$@" | tee -a $SOUP_LOG
|
||||
|
||||
11
salt/top.sls
11
salt/top.sls
@@ -38,7 +38,6 @@ base:
|
||||
- patch.os.schedule
|
||||
- motd
|
||||
- salt.minion-check
|
||||
- sensoroni
|
||||
- salt.lasthighstate
|
||||
|
||||
'*_helixsensor and G@saltversion:{{saltversion}}':
|
||||
@@ -47,6 +46,7 @@ base:
|
||||
- ca
|
||||
- ssl
|
||||
- registry
|
||||
- sensoroni
|
||||
- telegraf
|
||||
- firewall
|
||||
- idstools
|
||||
@@ -66,6 +66,7 @@ base:
|
||||
- match: compound
|
||||
- ca
|
||||
- ssl
|
||||
- sensoroni
|
||||
- telegraf
|
||||
- firewall
|
||||
- nginx
|
||||
@@ -92,6 +93,7 @@ base:
|
||||
- ca
|
||||
- ssl
|
||||
- registry
|
||||
- sensoroni
|
||||
- manager
|
||||
- nginx
|
||||
- telegraf
|
||||
@@ -160,6 +162,7 @@ base:
|
||||
- ca
|
||||
- ssl
|
||||
- registry
|
||||
- sensoroni
|
||||
- nginx
|
||||
- telegraf
|
||||
- influxdb
|
||||
@@ -220,6 +223,7 @@ base:
|
||||
- ca
|
||||
- ssl
|
||||
- registry
|
||||
- sensoroni
|
||||
- manager
|
||||
- nginx
|
||||
- telegraf
|
||||
@@ -290,6 +294,7 @@ base:
|
||||
- match: compound
|
||||
- ca
|
||||
- ssl
|
||||
- sensoroni
|
||||
- nginx
|
||||
- telegraf
|
||||
- firewall
|
||||
@@ -320,6 +325,7 @@ base:
|
||||
- ca
|
||||
- ssl
|
||||
- registry
|
||||
- sensoroni
|
||||
- nginx
|
||||
- telegraf
|
||||
- influxdb
|
||||
@@ -382,6 +388,7 @@ base:
|
||||
- match: compound
|
||||
- ca
|
||||
- ssl
|
||||
- sensoroni
|
||||
- nginx
|
||||
- telegraf
|
||||
- firewall
|
||||
@@ -424,6 +431,7 @@ base:
|
||||
- match: compound
|
||||
- ca
|
||||
- ssl
|
||||
- sensoroni
|
||||
- nginx
|
||||
- telegraf
|
||||
- firewall
|
||||
@@ -441,6 +449,7 @@ base:
|
||||
- ca
|
||||
- ssl
|
||||
- registry
|
||||
- sensoroni
|
||||
- manager
|
||||
- nginx
|
||||
- soc
|
||||
|
||||
Reference in New Issue
Block a user