Merge pull request #2478 from Security-Onion-Solutions/feature/wait-for-apt

Feature/wait for apt
2025-12-07 17:52:46 +01:00 · 2020-12-28 18:29:20 -05:00
parent deb38844ba e9a6155e44
commit afe40fe87b
2 changed files with 237 additions and 151 deletions
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -17,8 +17,8 @@

 # Check for prerequisites
 if [ "$(id -u)" -ne 0 ]; then
-  echo "This script must be run using sudo!"
-  exit 1
+	echo "This script must be run using sudo!"
+	exit 1
 fi

 # Define a banner to separate sections
@@ -26,46 +26,46 @@ banner="========================================================================

 header() {
 	echo
-    printf '%s\n' "$banner" "$*" "$banner"
+		printf '%s\n' "$banner" "$*" "$banner"
 }

 lookup_salt_value() {
-  key=$1
-  group=$2
-  kind=$3
+	key=$1
+	group=$2
+	kind=$3

-  if [ -z "$kind" ]; then
-    kind=pillar
-  fi
+	if [ -z "$kind" ]; then
+		kind=pillar
+	fi

-  if [ -n "$group" ]; then
-    group=${group}:
-  fi
+	if [ -n "$group" ]; then
+		group=${group}:
+	fi

-  salt-call --no-color  ${kind}.get ${group}${key} --out=newline_values_only
+	salt-call --no-color  ${kind}.get ${group}${key} --out=newline_values_only
 }

 lookup_pillar() {
-  key=$1
-  pillar=$2
-  if [ -z "$pillar" ]; then
-    pillar=global
-  fi
-  lookup_salt_value "$key" "$pillar" "pillar"
+	key=$1
+	pillar=$2
+	if [ -z "$pillar" ]; then
+		pillar=global
+	fi
+	lookup_salt_value "$key" "$pillar" "pillar"
 }

 lookup_pillar_secret() {
-  lookup_pillar "$1" "secrets"
+	lookup_pillar "$1" "secrets"
 }

 lookup_grain() {
-  lookup_salt_value "$1" "" "grains"
+	lookup_salt_value "$1" "" "grains"
 }

 lookup_role() {
-  id=$(lookup_grain id)
-  pieces=($(echo $id | tr '_' ' '))
-  echo ${pieces[1]}
+	id=$(lookup_grain id)
+	pieces=($(echo $id | tr '_' ' '))
+	echo ${pieces[1]}
 }

 check_container() {
@@ -74,98 +74,143 @@ check_container() {
 }

 check_password() {
-  local password=$1
-  echo "$password" | egrep -v "'|\"|\\$|\\\\" > /dev/null 2>&1
-  return $?
+	local password=$1
+	echo "$password" | egrep -v "'|\"|\\$|\\\\" > /dev/null 2>&1
+	return $?
 }

 set_os() {
-  if [ -f /etc/redhat-release ]; then
-    OS=centos
-  else
-    OS=ubuntu
-  fi
+	if [ -f /etc/redhat-release ]; then
+		OS=centos
+	else
+		OS=ubuntu
+	fi
 }

 set_minionid() {
-  MINIONID=$(lookup_grain id)
+	MINIONID=$(lookup_grain id)
 }

 set_version() {
-  CURRENTVERSION=0.0.0
-  if [ -f /etc/soversion ]; then
-    CURRENTVERSION=$(cat /etc/soversion)
-  fi
-  if [ -z "$VERSION" ]; then
-    if [ -z "$NEWVERSION" ]; then
-      if [ "$CURRENTVERSION" == "0.0.0" ]; then
-        echo "ERROR: Unable to detect Security Onion version; terminating script."
-        exit 1
-      else
-        VERSION=$CURRENTVERSION
-      fi
-    else
-      VERSION="$NEWVERSION"
-    fi
-  fi
+	CURRENTVERSION=0.0.0
+	if [ -f /etc/soversion ]; then
+		CURRENTVERSION=$(cat /etc/soversion)
+	fi
+	if [ -z "$VERSION" ]; then
+		if [ -z "$NEWVERSION" ]; then
+			if [ "$CURRENTVERSION" == "0.0.0" ]; then
+				echo "ERROR: Unable to detect Security Onion version; terminating script."
+				exit 1
+			else
+				VERSION=$CURRENTVERSION
+			fi
+		else
+			VERSION="$NEWVERSION"
+		fi
+	fi
 }

 require_manager() {
-  # Check to see if this is a manager
-  MANAGERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}')
-  if [ $MANAGERCHECK == 'so-eval' ] || [ $MANAGERCHECK == 'so-manager' ] || [ $MANAGERCHECK == 'so-managersearch' ] || [ $MANAGERCHECK == 'so-standalone' ] || [ $MANAGERCHECK == 'so-helix' ] || [ $MANAGERCHECK == 'so-import' ]; then
-    echo "This is a manager, We can proceed."
-  else
-    echo "Please run this command on the manager; the manager controls the grid."
-    exit 1
-  fi
+	# Check to see if this is a manager
+	MANAGERCHECK=$(cat /etc/salt/grains | grep role | awk '{print $2}')
+	if [ $MANAGERCHECK == 'so-eval' ] || [ $MANAGERCHECK == 'so-manager' ] || [ $MANAGERCHECK == 'so-managersearch' ] || [ $MANAGERCHECK == 'so-standalone' ] || [ $MANAGERCHECK == 'so-helix' ] || [ $MANAGERCHECK == 'so-import' ]; then
+		echo "This is a manager, We can proceed."
+	else
+		echo "Please run this command on the manager; the manager controls the grid."
+		exit 1
+	fi
 }

 is_single_node_grid() {
-  role=$(lookup_role)
-  if [ "$role" != "eval" ] && [ "$role" != "standalone" ] && [ "$role" != "import" ]; then
-    return 1
-  fi
-  return 0
+	role=$(lookup_role)
+	if [ "$role" != "eval" ] && [ "$role" != "standalone" ] && [ "$role" != "import" ]; then
+		return 1
+	fi
+	return 0
 }

 fail() {
-  msg=$1
-  echo "ERROR: $msg"
-  echo "Exiting."
-  exit 1
+	msg=$1
+	echo "ERROR: $msg"
+	echo "Exiting."
+	exit 1
 }

 get_random_value() {
-  length=${1:-20}
-  head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
+	length=${1:-20}
+	head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
+}
+
+wait_for_apt() {
+	local progress_callback=$1
+
+	local retry_count=30
+	local retry_timeout='10s'
+	local lock_msg="Could not acquire dpkg lock, waiting $retry_timeout for lock to release."
+	if [[ -z $progress_callback ]]; then
+		if [[ -z $progress_bar_text ]]; then
+			local old_text="Installing..."
+		else
+			local old_text="$progress_bar_text"
+		fi
+	fi
+	local count=0
+	while [[ "$count" -lt "$retry_count" ]]; do
+		((count++))
+		[[ -z $progress_callback ]] && echo "Attempting to acquire dpkg lock to run apt command... (Attempt $count/$retry_count)"
+		if __check_apt_lock; then
+			if [[ -z $progress_callback ]]; then
+				echo "  $lock_msg" | tee -a "$setup_log"
+			else
+				$progress_callback "Could not acquire dpkg lock, waiting $retry_timeout ($count/$retry_count)"
+			fi
+		else
+			[[ -z $progress_callback ]] || $progress_callback "$old_text"
+			return 0
+		fi
+		sleep "$retry_timeout"
+	done
+
+	if __check_apt_lock; then
+		[[ -z $progress_callback ]] &&	echo "Could not acquire lock after $retry_count attempts, aborting."
+		return 1
+	else
+		return 0
+	fi
+}
+
+__check_apt_lock() {
+	lsof /var/lib/dpkg/lock &> /dev/null
+	local lock=$?
+
+	return $lock
 }

 wait_for_web_response() {
-  url=$1
-  expected=$2
-  maxAttempts=${3:-300}
-  logfile=/root/wait_for_web_response.log
-  attempt=0
-  while [[ $attempt -lt $maxAttempts ]]; do
-    attempt=$((attempt+1))
-    echo "Waiting for value '$expected' at '$url' ($attempt/$maxAttempts)"
-    result=$(curl -ks -L $url)
-    exitcode=$?
+	url=$1
+	expected=$2
+	maxAttempts=${3:-300}
+	logfile=/root/wait_for_web_response.log
+	attempt=0
+	while [[ $attempt -lt $maxAttempts ]]; do
+		attempt=$((attempt+1))
+		echo "Waiting for value '$expected' at '$url' ($attempt/$maxAttempts)"
+		result=$(curl -ks -L $url)
+		exitcode=$?

-    echo "--------------------------------------------------" >> $logfile
-    echo "$(date) - Checking web URL: $url ($attempt/$maxAttempts)" >> $logfile
-    echo "$result" >> $logfile
-    echo "exit code=$exitcode" >> $logfile
-    echo "" >> $logfile
+		echo "--------------------------------------------------" >> $logfile
+		echo "$(date) - Checking web URL: $url ($attempt/$maxAttempts)" >> $logfile
+		echo "$result" >> $logfile
+		echo "exit code=$exitcode" >> $logfile
+		echo "" >> $logfile

-    if [[ $exitcode -eq 0 && "$result" =~ $expected ]]; then
-      echo "Received expected response; proceeding."
-      return 0
-    fi
-    echo "Server is not ready"
-    sleep 1
-  done
-  echo "Server still not ready after $maxAttempts attempts; giving up."
-  return 1
-}
+		if [[ $exitcode -eq 0 && "$result" =~ $expected ]]; then
+			echo "Received expected response; proceeding."
+			return 0
+		fi
+		echo "Server is not ready"
+		sleep 1
+	done
+	echo "Server still not ready after $maxAttempts attempts; giving up."
+	return 1
+}
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -84,7 +84,7 @@ calculate_useable_cores() {
 	lb_procs_round=$(printf "%.0f\n" $cores_for_zeek)

 	if [ "$lb_procs_round" -lt 1 ]; then lb_procs=1; else lb_procs=$lb_procs_round; fi
-    export lb_procs
+	export lb_procs
 }

 airgap_rules() {
@@ -799,13 +799,13 @@ detect_os() {
 		echo "Installing required packages to run installer..."
 		# Install network manager so we can do interface stuff
 		if ! command -v nmcli > /dev/null 2>&1; then
+			if wait_for_apt; then apt-get install -y network-manager >> "$setup_log" 2<&1; else exit 1; fi
 			{
-				apt-get install -y network-manager;
-				systemctl enable NetworkManager;
-				systemctl start NetworkManager;
+				systemctl enable NetworkManager
+				systemctl start NetworkManager
 			} >> "$setup_log" 2<&1
 		fi
-		apt-get install -y bc curl >> "$setup_log" 2>&1
+		if wait_for_apt; then apt-get install -y bc curl >> "$setup_log" 2>&1; else exit 1; fi

 	else
 		echo "We were unable to determine if you are using a supported OS."
@@ -882,21 +882,28 @@ docker_install() {
 	else
 		case "$install_type" in
 			'MANAGER' | 'EVAL' | 'STANDALONE' | 'MANAGERSEARCH' | 'IMPORT')
-				apt-get update >> "$setup_log" 2>&1
+				if wait_for_apt 'whiptail_prog_new_message'; then apt-get update >> "$setup_log" 2>&1; else kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1; fi
 				;;
 			*)
-				{
-					apt-key add "$temp_install_dir"/gpg/docker.pub;
-					add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable";
-					apt-get update;
-				} >> "$setup_log" 2>&1
+				if wait_for_apt 'whiptail_prog_new_message'; then
+					{
+						apt-key add "$temp_install_dir"/gpg/docker.pub;
+						add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable";
+						apt-get update;
+					} >> "$setup_log" 2>&1
+				else
+					kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
+				fi
 				;;
 		esac 
-		
-		if [ $OSVER != "xenial" ]; then
-			apt-get -y install docker-ce python3-docker >> "$setup_log" 2>&1
+		if wait_for_apt 'whiptail_prog_new_message'; then
+			if [ $OSVER != "xenial" ]; then
+				apt-get -y install docker-ce python3-docker >> "$setup_log" 2>&1
+			else
+				apt-get -y install docker-ce python-docker >> "$setup_log" 2>&1
+			fi
 		else
-			apt-get -y install docker-ce python-docker >> "$setup_log" 2>&1
+			kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
 		fi
 	fi
 	docker_registry
@@ -939,7 +946,7 @@ docker_seed_update() {
 	if [ "$install_type" == 'HELIXSENSOR' ]; then 
 		percent_delta=6
 	fi
-	((docker_seed_update_percent=docker_seed_update_percent+percent_delta))
+	((docker_seed_update_percent+=percent_delta))

 	set_progress_str "$docker_seed_update_percent" "Downloading $name"
 }
@@ -1572,7 +1579,11 @@ remove_package() {
 		fi
 	else
 		if dpkg -l | grep -q "$package_name"; then
-			apt purge -y "$package_name"
+			if wait_for_apt 'whiptail_prog_new_message'; then
+				apt purge -y "$package_name"
+			else
+				exit 1
+			fi
 		fi
 	fi
 }
@@ -1644,36 +1655,43 @@ saltify() {
 			  yum -y install epel-release
 			  yum -y install salt-minion-3002.2\
 			   	python3\
-				python36-docker\
-				python36-dateutil\
-				python36-m2crypto\
-				python36-mysql\
-				yum-utils\
-				device-mapper-persistent-data\
-				lvm2\
-				openssl\
-				jq;
-			  yum -y update --exclude=salt*;
+					python36-docker\
+					python36-dateutil\
+					python36-m2crypto\
+					python36-mysql\
+					yum-utils\
+					device-mapper-persistent-data\
+					lvm2\
+					openssl\
+					jq;
+					yum -y update --exclude=salt*;
 			fi
 			systemctl enable salt-minion;
 		} >> "$setup_log" 2>&1
 		yum versionlock salt*
 	else
-		DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade >> "$setup_log" 2>&1
+		if wait_for_apt 'whiptail_prog_new_message'; then
+			DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade >> "$setup_log" 2>&1
+		else
+			kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
+		fi

 		if [ $OSVER != "xenial" ]; then
 			# Switch to Python 3 as default if this is not xenial
 			update-alternatives --install /usr/bin/python python /usr/bin/python3.6 10 >> "$setup_log" 2>&1
 		fi
-		# Add the pre-requisites for installing docker-ce
-		apt-get -y install ca-certificates\
-			curl\
-			software-properties-common\
-			apt-transport-https\
-			openssl\
-			netcat\
-			jq >> "$setup_log" 2>&1
-
+		if wait_for_apt 'whiptail_prog_new_message'; then
+			# Add the pre-requisites for installing docker-ce
+			apt-get -y install ca-certificates\
+				curl\
+				software-properties-common\
+				apt-transport-https\
+				openssl\
+				netcat\
+				jq >> "$setup_log" 2>&1
+		else
+			kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
+		fi
 		# Grab the version from the os-release file
 		local ubuntu_version
 		ubuntu_version=$(grep VERSION_ID /etc/os-release | awk -F '[ "]' '{print $2}')
@@ -1681,7 +1699,11 @@ saltify() {

 		case "$install_type" in
 			'FLEET')
-				if [ "$OSVER" != 'xenial' ]; then apt-get -y install python3-mysqldb >> "$setup_log" 2>&1; else apt-get -y install python-mysqldb >> "$setup_log" 2>&1; fi
+				if wait_for_apt 'whiptail_prog_new_message'; then
+					if [ "$OSVER" != 'xenial' ]; then apt-get -y install python3-mysqldb >> "$setup_log" 2>&1; else apt-get -y install python-mysqldb >> "$setup_log" 2>&1; fi
+				else
+					kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
+				fi
 				;;
 			'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORT' | 'HELIXSENSOR')

@@ -1703,13 +1725,18 @@ saltify() {
 				curl -s https://packages.wazuh.com/key/GPG-KEY-WAZUH | apt-key add - >> "$setup_log" 2>&1
 				# Add repo
 				echo "deb https://packages.wazuh.com/3.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list 2>> "$setup_log"
-				# Initialize the new repos
-				apt-get update >> "$setup_log" 2>&1
-				set_progress_str 6 'Installing various dependencies'
-				apt-get -y install sqlite3 argon2 libssl-dev >> "$setup_log" 2>&1
-				set_progress_str 7 'Installing salt-master'
-				apt-get -y install salt-master=3002.2+ds-1 >> "$setup_log" 2>&1
-				apt-mark hold salt-master >> "$setup_log" 2>&1
+				
+				if wait_for_apt 'whiptail_prog_new_message'; then
+					# Initialize the new repos
+					apt-get update >> "$setup_log" 2>&1
+					set_progress_str 6 'Installing various dependencies'
+					apt-get -y install sqlite3 argon2 libssl-dev >> "$setup_log" 2>&1
+					set_progress_str 7 'Installing salt-master'
+					apt-get -y install salt-master=3002.2+ds-1 >> "$setup_log" 2>&1
+					apt-mark hold salt-master >> "$setup_log" 2>&1
+				else
+					kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
+				fi
 				;;
 			*)
 				# Copy down the gpg keys and install them from the manager
@@ -1723,18 +1750,21 @@ saltify() {
 				echo "deb https://packages.wazuh.com/3.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list 2>> "$setup_log"
 				;;
 		esac
-		apt-get update >> "$setup_log" 2>&1
-		set_progress_str 8 'Installing salt-minion & python modules'
-		apt-get -y install salt-minion=3002.2+ds-1\
-					salt-common=3002.2+ds-1 >> "$setup_log" 2>&1
-		apt-mark hold salt-minion salt-common >> "$setup_log" 2>&1
-		if [ "$OSVER" != 'xenial' ]; then 
-			apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb >> "$setup_log" 2>&1
-		else 
-			apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb  >> "$setup_log" 2>&1
+		if wait_for_apt 'whiptail_prog_new_message'; then
+			apt-get update >> "$setup_log" 2>&1
+			set_progress_str 8 'Installing salt-minion & python modules'
+			apt-get -y install salt-minion=3002.2+ds-1\
+						salt-common=3002.2+ds-1 >> "$setup_log" 2>&1
+			apt-mark hold salt-minion salt-common >> "$setup_log" 2>&1
+			if [ "$OSVER" != 'xenial' ]; then 
+				apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb >> "$setup_log" 2>&1
+			else 
+				apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb  >> "$setup_log" 2>&1
+			fi
+		else
+			kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
 		fi
 	fi
-
 }

 salt_checkin() {
@@ -1897,7 +1927,8 @@ setup_salt_master_dirs() {

 set_progress_str() {
 	local percentage_input=$1
-	local progress_bar_text=$2
+	progress_bar_text=$2
+	export progress_bar_text

 	if (( "$percentage_input" >= "$percentage" )); then
 		percentage="$percentage_input"
@@ -2154,8 +2185,12 @@ update_packages() {
 	if [ "$OS" = 'centos' ]; then
 		yum -y update >> "$setup_log"
 	else
-		apt-get -y update >> "$setup_log"
-		apt-get -y upgrade >> "$setup_log"
+		if wait_for_apt 'whiptail_prog_new_message'; then
+			apt-get -y update >> "$setup_log"
+			apt-get -y upgrade >> "$setup_log"
+		else
+			kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
+		fi
 	fi
 }

@@ -2226,6 +2261,12 @@ es_heapsize() {
 	fi
 }

+
+whiptail_prog_new_message() {
+	local message=$1
+	set_progress_str "$percentage" "$message"
+}
+
 # Enable Zeek Logs
 zeek_logs_enabled() {
 	echo "Enabling Zeek Logs" >> "$setup_log" 2>&1