Merge pull request #2478 from Security-Onion-Solutions/feature/wait-for-apt

Feature/wait for apt

salt/common/tools/sbin/so-common

@@ -141,6 +141,51 @@ get_random_value() {
 	head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
 }
 
+wait_for_apt() {
+	local progress_callback=$1
+
+	local retry_count=30
+	local retry_timeout='10s'
+	local lock_msg="Could not acquire dpkg lock, waiting $retry_timeout for lock to release."
+	if [[ -z $progress_callback ]]; then
+		if [[ -z $progress_bar_text ]]; then
+			local old_text="Installing..."
+		else
+			local old_text="$progress_bar_text"
+		fi
+	fi
+	local count=0
+	while [[ "$count" -lt "$retry_count" ]]; do
+		((count++))
+		[[ -z $progress_callback ]] && echo "Attempting to acquire dpkg lock to run apt command... (Attempt $count/$retry_count)"
+		if __check_apt_lock; then
+			if [[ -z $progress_callback ]]; then
+				echo "  $lock_msg" | tee -a "$setup_log"
+			else
+				$progress_callback "Could not acquire dpkg lock, waiting $retry_timeout ($count/$retry_count)"
+			fi
+		else
+			[[ -z $progress_callback ]] || $progress_callback "$old_text"
+			return 0
+		fi
+		sleep "$retry_timeout"
+	done
+
+	if __check_apt_lock; then
+		[[ -z $progress_callback ]] &&	echo "Could not acquire lock after $retry_count attempts, aborting."
+		return 1
+	else
+		return 0
+	fi
+}
+
+__check_apt_lock() {
+	lsof /var/lib/dpkg/lock &> /dev/null
+	local lock=$?
+
+	return $lock
+}
+
 wait_for_web_response() {
 	url=$1
 	expected=$2

setup/so-functions

@@ -799,13 +799,13 @@ detect_os() {
 		echo "Installing required packages to run installer..."
 		# Install network manager so we can do interface stuff
 		if ! command -v nmcli > /dev/null 2>&1; then
+			if wait_for_apt; then apt-get install -y network-manager >> "$setup_log" 2<&1; else exit 1; fi
 			{
-				apt-get install -y network-manager;
-				systemctl enable NetworkManager;
-				systemctl start NetworkManager;
+				systemctl enable NetworkManager
+				systemctl start NetworkManager
 			} >> "$setup_log" 2<&1
 		fi
-		apt-get install -y bc curl >> "$setup_log" 2>&1
+		if wait_for_apt; then apt-get install -y bc curl >> "$setup_log" 2>&1; else exit 1; fi
 
 	else
 		echo "We were unable to determine if you are using a supported OS."
@@ -882,22 +882,29 @@ docker_install() {
 	else
 		case "$install_type" in
 			'MANAGER' | 'EVAL' | 'STANDALONE' | 'MANAGERSEARCH' | 'IMPORT')
-				apt-get update >> "$setup_log" 2>&1
+				if wait_for_apt 'whiptail_prog_new_message'; then apt-get update >> "$setup_log" 2>&1; else kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1; fi
 				;;
 			*)
+				if wait_for_apt 'whiptail_prog_new_message'; then
 					{
 						apt-key add "$temp_install_dir"/gpg/docker.pub;
 						add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable";
 						apt-get update;
 					} >> "$setup_log" 2>&1
+				else
+					kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
+				fi
 				;;
 		esac 
-		
+		if wait_for_apt 'whiptail_prog_new_message'; then
 			if [ $OSVER != "xenial" ]; then
 				apt-get -y install docker-ce python3-docker >> "$setup_log" 2>&1
 			else
 				apt-get -y install docker-ce python-docker >> "$setup_log" 2>&1
 			fi
+		else
+			kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
+		fi
 	fi
 	docker_registry
 	{
@@ -939,7 +946,7 @@ docker_seed_update() {
 	if [ "$install_type" == 'HELIXSENSOR' ]; then 
 		percent_delta=6
 	fi
-	((docker_seed_update_percent=docker_seed_update_percent+percent_delta))
+	((docker_seed_update_percent+=percent_delta))
 
 	set_progress_str "$docker_seed_update_percent" "Downloading $name"
 }
@@ -1572,7 +1579,11 @@ remove_package() {
 		fi
 	else
 		if dpkg -l | grep -q "$package_name"; then
+			if wait_for_apt 'whiptail_prog_new_message'; then
 				apt purge -y "$package_name"
+			else
+				exit 1
+			fi
 		fi
 	fi
 }
@@ -1659,12 +1670,17 @@ saltify() {
 		} >> "$setup_log" 2>&1
 		yum versionlock salt*
 	else
+		if wait_for_apt 'whiptail_prog_new_message'; then
 			DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade >> "$setup_log" 2>&1
+		else
+			kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
+		fi
 
 		if [ $OSVER != "xenial" ]; then
 			# Switch to Python 3 as default if this is not xenial
 			update-alternatives --install /usr/bin/python python /usr/bin/python3.6 10 >> "$setup_log" 2>&1
 		fi
+		if wait_for_apt 'whiptail_prog_new_message'; then
 			# Add the pre-requisites for installing docker-ce
 			apt-get -y install ca-certificates\
 				curl\
@@ -1673,7 +1689,9 @@ saltify() {
 				openssl\
 				netcat\
 				jq >> "$setup_log" 2>&1
-
+		else
+			kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
+		fi
 		# Grab the version from the os-release file
 		local ubuntu_version
 		ubuntu_version=$(grep VERSION_ID /etc/os-release | awk -F '[ "]' '{print $2}')
@@ -1681,7 +1699,11 @@ saltify() {
 
 		case "$install_type" in
 			'FLEET')
+				if wait_for_apt 'whiptail_prog_new_message'; then
 					if [ "$OSVER" != 'xenial' ]; then apt-get -y install python3-mysqldb >> "$setup_log" 2>&1; else apt-get -y install python-mysqldb >> "$setup_log" 2>&1; fi
+				else
+					kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
+				fi
 				;;
 			'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORT' | 'HELIXSENSOR')
 
@@ -1703,6 +1725,8 @@ saltify() {
 				curl -s https://packages.wazuh.com/key/GPG-KEY-WAZUH | apt-key add - >> "$setup_log" 2>&1
 				# Add repo
 				echo "deb https://packages.wazuh.com/3.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list 2>> "$setup_log"
+				
+				if wait_for_apt 'whiptail_prog_new_message'; then
 					# Initialize the new repos
 					apt-get update >> "$setup_log" 2>&1
 					set_progress_str 6 'Installing various dependencies'
@@ -1710,6 +1734,9 @@ saltify() {
 					set_progress_str 7 'Installing salt-master'
 					apt-get -y install salt-master=3002.2+ds-1 >> "$setup_log" 2>&1
 					apt-mark hold salt-master >> "$setup_log" 2>&1
+				else
+					kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
+				fi
 				;;
 			*)
 				# Copy down the gpg keys and install them from the manager
@@ -1723,6 +1750,7 @@ saltify() {
 				echo "deb https://packages.wazuh.com/3.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list 2>> "$setup_log"
 				;;
 		esac
+		if wait_for_apt 'whiptail_prog_new_message'; then
 			apt-get update >> "$setup_log" 2>&1
 			set_progress_str 8 'Installing salt-minion & python modules'
 			apt-get -y install salt-minion=3002.2+ds-1\
@@ -1733,8 +1761,10 @@ saltify() {
 			else 
 				apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb  >> "$setup_log" 2>&1
 			fi
+		else
+			kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
+		fi
 	fi
-
 }
 
 salt_checkin() {
@@ -1897,7 +1927,8 @@ setup_salt_master_dirs() {
 
 set_progress_str() {
 	local percentage_input=$1
-	local progress_bar_text=$2
+	progress_bar_text=$2
+	export progress_bar_text
 
 	if (( "$percentage_input" >= "$percentage" )); then
 		percentage="$percentage_input"
@@ -2154,8 +2185,12 @@ update_packages() {
 	if [ "$OS" = 'centos' ]; then
 		yum -y update >> "$setup_log"
 	else
+		if wait_for_apt 'whiptail_prog_new_message'; then
 			apt-get -y update >> "$setup_log"
 			apt-get -y upgrade >> "$setup_log"
+		else
+			kill -SIGUSR1 "$(ps --pid $$ -oppid=)"; exit 1
+		fi
 	fi
 }
 
@@ -2226,6 +2261,12 @@ es_heapsize() {
 	fi
 }
 
+
+whiptail_prog_new_message() {
+	local message=$1
+	set_progress_str "$percentage" "$message"
+}
+
 # Enable Zeek Logs
 zeek_logs_enabled() {
 	echo "Enabling Zeek Logs" >> "$setup_log" 2>&1