From 200736a118062c211e2acf2e0ef38b3333bbe8c1 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Wed, 29 Dec 2021 18:15:16 -0500
Subject: [PATCH] add option to look for failed outout in retry function in
 so-common. look for Err: when running soapt-get update in setup

---
 salt/common/tools/sbin/so-common | 62 ++++++++++++++++++--------------
 setup/so-functions               | 10 +++---
 2 files changed, 41 insertions(+), 31 deletions(-)

diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common
index 86f883a43..3409b9744 100755
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -294,32 +294,42 @@ require_manager() {
 }
 
 retry() {
-	maxAttempts=$1
-	sleepDelay=$2
-	cmd=$3
-	expectedOutput=$4
-	attempt=0
-	local exitcode=0
-	while [[ $attempt -lt $maxAttempts ]]; do			
-		attempt=$((attempt+1))
-		echo "Executing command with retry support: $cmd"
-		output=$(eval "$cmd")
-		exitcode=$?
-		echo "Results: $output ($exitcode)"
-		if [ -n "$expectedOutput" ]; then
-			if [[ "$output" =~ "$expectedOutput" ]]; then
-				return $exitCode
-			else
-				echo "Expected '$expectedOutput' but got '$output'"
-			fi
-		elif [[ $exitcode -eq 0 ]]; then
-			return $exitCode
-		fi
-		echo "Command failed with exit code $exitcode; will retry in $sleepDelay seconds ($attempt / $maxAttempts)..."
-		sleep $sleepDelay
-	done
-	echo "Command continues to fail; giving up."
-	return $exitcode
+        maxAttempts=$1
+        sleepDelay=$2
+        cmd=$3
+        expectedOutput=$4
+        failedOutput=$5
+        attempt=0
+        local exitcode=0
+        while [[ $attempt -lt $maxAttempts ]]; do
+                attempt=$((attempt+1))
+                echo "Executing command with retry support: $cmd"
+                output=$(eval "$cmd")
+                exitcode=$?
+                echo "Results: $output ($exitcode)"
+                if [ -n "$expectedOutput" ]; then
+                        if [[ "$output" =~ "$expectedOutput" ]]; then
+                                return $exitCode
+                        else
+                                echo "Expected '$expectedOutput' but got '$output'"
+                        fi
+                elif [ -n "$failedOutput" ]; then
+                        if [[ "$output" =~ "$failedOutput" ]]; then
+                                echo "Found failed output '$failedOutput' in '$output'"
+                                if [[ $exitcode -eq 0 ]]; then
+                                        exitcode="${exitcode} (This is 0, but we found '$failedOutput' in the output.)"
+                                fi
+                        else
+                                return $exitCode
+                        fi
+                elif [[ $exitcode -eq 0 ]]; then
+                        return $exitCode
+                fi
+                echo "Command failed with exit code $exitcode; will retry in $sleepDelay seconds ($attempt / $maxAttempts)..."
+                sleep $sleepDelay
+        done
+        echo "Command continues to fail; giving up."
+        return $exitcode
 }
 
 run_check_net_err() {
diff --git a/setup/so-functions b/setup/so-functions
index 5a3781f39..12a252c46 100755
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -1147,7 +1147,7 @@ installer_prereq_packages() {
 		logCmd "systemctl start NetworkManager"
 	elif [ "$OS" == ubuntu ]; then
 		# Print message to stdout so the user knows setup is doing something
-		retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1
+		retry 50 10 "apt-get update" "" "Err:" >> "$setup_log" 2>&1 || exit 1
 		# Install network manager so we can do interface stuff
 		if ! command -v nmcli > /dev/null 2>&1; then
 			retry 50 10 "apt-get -y install network-manager" >> "$setup_log" 2>&1 || exit 1
@@ -1204,12 +1204,12 @@ docker_install() {
 	else
 		case "$install_type" in
 			'MANAGER' | 'EVAL' | 'STANDALONE' | 'MANAGERSEARCH' | 'IMPORT')
-				retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1
+				retry 50 10 "apt-get update" "" "Err:" >> "$setup_log" 2>&1 || exit 1
 				;;
 			*)
 				retry 50 10 "apt-key add $temp_install_dir/gpg/docker.pub" >> "$setup_log" 2>&1 || exit 1
 				add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" >> "$setup_log" 2>&1
-				retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1
+				retry 50 10 "apt-get update" "" "Err:" >> "$setup_log" 2>&1 || exit 1
 				;;
 		esac
 		if [ $OSVER == "bionic" ]; then
@@ -2259,7 +2259,7 @@ saltify() {
 				# Add repo
 				echo "deb https://packages.wazuh.com/3.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list 2>> "$setup_log"
 				
-				retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1
+				retry 50 10 "apt-get update" "" "Err:" >> "$setup_log" 2>&1 || exit 1
 				set_progress_str 6 'Installing various dependencies'
 				retry 50 10 "apt-get -y install sqlite3 libssl-dev" >> "$setup_log" 2>&1 || exit 1
 				set_progress_str 7 'Installing salt-master'
@@ -2279,7 +2279,7 @@ saltify() {
 				;;
 		esac
 
-		retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1
+		retry 50 10 "apt-get update" "" "Err:" >> "$setup_log" 2>&1 || exit 1
 		set_progress_str 8 'Installing salt-minion & python modules'
 		retry 50 10 "apt-get -y install salt-minion=3003+ds-1 salt-common=3003+ds-1" >> "$setup_log" 2>&1 || exit 1
 		retry 50 10 "apt-mark hold salt-minion salt-common" >> "$setup_log" 2>&1 || exit 1