Pull in upstream changes

setup/so-functions

@@ -200,7 +200,7 @@ check_service_status() {
 }
 
 check_web_pass() {
-	info Making sure web credential passwords match
+	info "Making sure web credential passwords match"
 	check_pass_match "$WEBPASSWD1" "$WEBPASSWD2" "WPMATCH"
 }
 
@@ -383,16 +383,11 @@ collect_mngr_hostname() {
 	fi
 }
 
-collect_net_method() {
+collect_net_method() { 
 	whiptail_net_method
-
-	if [[ "$network_traffic" == *"_MANAGER" ]]; then
-		whiptail_manager_updates_warning
-		MANAGERUPDATES=1
-	fi
-
 	if [[ "$network_traffic" == "PROXY"* ]]; then
 		collect_proxy no_ask
+		needs_proxy=true
 	fi
 }
 
@@ -909,10 +904,9 @@ detect_os() {
 
 	elif [ -f /etc/os-release ]; then
 		OS=ubuntu
-		if grep -q "UBUNTU_CODENAME=bionic" /etc/os-release; then
-			OSVER=bionic
-		elif grep -q "UBUNTU_CODENAME=focal" /etc/os-release; then
+		if grep -q "UBUNTU_CODENAME=focal" /etc/os-release; then
 			OSVER=focal
+			is_ubuntu=true
 		else
 			info "We do not support your current version of Ubuntu."
 			exit 1
@@ -1307,15 +1301,7 @@ idh_pillar() {
 logstash_pillar() {
 	# Create the logstash advanced pillar
 	touch $adv_logstash_pillar_file
-	title "Create the logstash pillar"
-	printf '%s\n'\
-		"logstash_settings:"\
-		"  ls_host: '$HOSTNAME'"\
-		"  ls_pipeline_batch_size: 125"\
-		"  ls_input_threads: 1"\
-		"  lsheap: $NODE_LS_HEAP_SIZE"\
-		"  ls_pipeline_workers: $num_cpu_cores"\
-		"" > "$logstash_pillar_file"
+	touch $logstash_pillar_file
 }
 
 # Set Logstash heap size based on total memory
@@ -1339,10 +1325,6 @@ ls_heapsize() {
 	esac
 	export LS_HEAP_SIZE
 
-	if [[ "$install_type" =~ ^(EVAL|MANAGERSEARCH|STANDALONE)$ ]]; then
-		NODE_LS_HEAP_SIZE=$LS_HEAP_SIZE
-		export NODE_LS_HEAP_SIZE
-	fi
 }
 
 idstools_pillar() {
@@ -1887,6 +1869,9 @@ drop_install_options() {
 	NODETYPE=${install_type^^}
 	echo "NODETYPE=$NODETYPE" >> /opt/so/install.txt
 	echo "CORECOUNT=$lb_procs" >> /opt/so/install.txt
+	echo "LSHOSTNAME=$HOSTNAME" >> /opt/so/install.txt
+	echo "LSHEAP=$LS_HEAP_SIZE" >> /opt/so/install.txt
+	echo "CPUCORES=$num_cpu_cores" >> /opt/so/install.txt
 }
 
 remove_package() {
@@ -1992,6 +1977,43 @@ repo_sync_local() {
 }
 
 saltify() {
+	if [[ $is_ubuntu ]]; then
+
+		DEBIAN_FRONTEND=noninteractive retry 150 20 "apt-get -y -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade" >> "$setup_log" 2>&1 || exit 1
+		update-alternatives --install /usr/bin/python python /usr/bin/python3.8 10 
+		local pkg_arr=(
+			'apache2-utils'
+			'ca-certificates'
+			'curl'
+			'software-properties-common'
+			'apt-transport-https'
+			'openssl'
+			'netcat'
+			'jq'
+		)
+		retry 150 20 "apt-get -y install ${pkg_arr[*]}" || exit 1
+
+		logCmd "mkdir -vp /opt/so/gpg"
+		logCmd "wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt/SALTSTACK-GPG-KEY.pub"
+		logCmd "wget -q --inet4-only -O /opt/so/gpg/docker.pub https://download.docker.com/linux/ubuntu/gpg"
+		logCmd "apt-key add /opt/so/gpg/SALTSTACK-GPG-KEY.pub"
+		logCmd "apt-key add /opt/so/gpg/docker.pub"
+
+		# Add SO Saltstack Repo
+		echo "deb https://repo.securityonion.net/file/securityonion-repo/ubuntu/20.04/amd64/salt3004.2/ focal main" > /etc/apt/sources.list.d/saltstack.list
+
+		# Add Docker Repo
+		add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" 
+		
+		# Ain't nothing but a GPG 
+
+		retry 150 20 "apt-get update" "" "Err:" || exit 1
+		retry 150 20 "apt-get -y install salt-minion=3004.2+ds-1 salt-common=3004.2+ds-1" || exit 1
+		retry 150 20 "apt-mark hold salt-minion salt-common" || exit 1
+		retry 150 20 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging python3-influxdb python3-lxml" || exit 1
+
+	fi
+		
 	if [[ $is_centos ]]; then
 		RUSALTY=$(rpm -qa | grep salt-minion | wc -l)
 		if [[ "$RUSALTY" -gt 0 ]]; then
@@ -2305,6 +2327,22 @@ so_add_user() {
 	fi
 }
 
+ubuntu_check() {
+	if [[ $OS == "ubuntu" ]]; then
+		if [[ $waitforstate ]]; then
+			whiptail_ubuntu_notsupported
+			exit 1
+		else
+		    if [[ $UBUNTUINSTALL == "needtoupgrade" ]]; then
+				whiptail_ubuntu_warning
+			else
+				whiptail_ubuntu_notsupported
+				exit 1
+			fi
+		fi
+	fi
+}
+
 update_sudoers_for_testing() {
 	if [ -n "$TESTING" ]; then
 		info "Ensuring $INSTALLUSERNAME has password-less sudo access for automated testing purposes."
@@ -2383,12 +2421,13 @@ wait_for_salt_minion() {
 
 verify_setup() {
 	info "Verifying setup"
-	output=$(./so-verify "$setup_type" 2>&1)
+	set -o pipefail
+	./so-verify "$setup_type" 2>&1 | tee -a $setup_log
 	result=$?
-	echo "$output" >> "$setup_log"
+	set +o pipefail
 	if [[ $result -eq 0 ]]; then
 		whiptail_setup_complete
 	else
 		whiptail_setup_failed
 	fi
-}
+}

setup/so-setup

@@ -142,6 +142,7 @@ if [[ -f /root/accept_changes ]]; then
 	mv "$setup_log" "$setup_log.bak"
 	[ -f "$error_log" ] && mv "$error_log" "$error_log.bak"
 	reinstall_init
+	reset_proxy
 fi
 
 title "Parsing Username for Install"
@@ -314,10 +315,12 @@ if ! [[ -f $install_opt_file ]]; then
 	# If you are a manager ask ALL the manager things here. I know there is code re-use but this makes it easier to add new roles.
 	if [[ $is_eval ]]; then
 		waitforstate=true
+		ubuntu_check
 		monints=true
 		check_elastic_license
 		check_requirements "manager"
 		networking_needful
+		collect_net_method
 		collect_dockernet
 		if [[ $is_iso ]]; then
 			whiptail_airgap
@@ -336,10 +339,12 @@ if ! [[ -f $install_opt_file ]]; then
 		whiptail_end_settings
 	elif [[ $is_standalone ]]; then
 		waitforstate=true
+		ubuntu_check
 		monints=true
 		check_elastic_license
 		check_requirements "manager"
 		networking_needful
+		collect_net_method
 		collect_dockernet
 		if [[ $is_iso ]]; then
 			whiptail_airgap
@@ -359,13 +364,16 @@ if ! [[ -f $install_opt_file ]]; then
 	elif [[ $is_manager ]]; then
 		check_elastic_license
 		waitforstate=true
+		ubuntu_check
 		check_requirements "manager"
 		networking_needful
+		collect_net_method
 		collect_dockernet
 		if [[ $is_iso ]]; then
 			whiptail_airgap
 		fi
 		detect_cloud
+		set_minion_info
 		set_default_log_size >> $setup_log 2>&1
 		info "Verifying all network devices are managed by Network Manager that should be"
 		check_network_manager_conf
@@ -378,13 +386,16 @@ if ! [[ -f $install_opt_file ]]; then
 	elif [[ $is_managersearch ]]; then
 		check_elastic_license
 		waitforstate=true
+		ubuntu_check
 		check_requirements "manager"
 		networking_needful
+		collect_net_method
 		collect_dockernet
 		if [[ $is_iso ]]; then
 			whiptail_airgap
 		fi
 		detect_cloud
+		set_minion_info
 		set_default_log_size >> $setup_log 2>&1
 		info "Verifying all network devices are managed by Network Manager that should be"
 		check_network_manager_conf
@@ -395,6 +406,7 @@ if ! [[ -f $install_opt_file ]]; then
 		collect_so_allow
 		whiptail_end_settings
 	elif [[ $is_sensor ]]; then
+		ubuntu_check
 		monints=true
 		check_requirements "sensor"
 		calculate_useable_cores
@@ -422,6 +434,7 @@ if ! [[ -f $install_opt_file ]]; then
 		whiptail_end_settings
 
 	elif [[ $is_searchnode ]]; then
+		ubuntu_check
 		check_requirements "elasticsearch"
 		networking_needful
 		check_network_manager_conf
@@ -434,6 +447,7 @@ if ! [[ -f $install_opt_file ]]; then
 		whiptail_end_settings
 
 	elif [[ $is_heavynode ]]; then
+		ubuntu_check
 		monints=true
 		check_requirements "heavynode"
 		calculate_useable_cores
@@ -441,17 +455,21 @@ if ! [[ -f $install_opt_file ]]; then
 		collect_mngr_hostname
 		add_mngr_ip_to_hosts
 		check_manager_connection
+		set_minion_info
 		whiptail_end_settings
 
 	elif [[ $is_idh ]]; then
+		ubuntu_check
 		check_requirements "idh"
 		networking_needful
 		collect_mngr_hostname
 		add_mngr_ip_to_hosts
 		check_manager_connection
+		set_minion_info
 		whiptail_end_settings
 
 	elif [[ $is_import ]]; then
+		ubuntu_check
 		waitforstate=true
 		monints=true
 		check_elastic_license
@@ -473,11 +491,13 @@ if ! [[ -f $install_opt_file ]]; then
 		whiptail_end_settings
 
 	elif [[ $is_receiver ]]; then
+		ubuntu_check
 		check_requirements "receiver"
 		networking_needful
 		collect_mngr_hostname
 		add_mngr_ip_to_hosts
 		check_manager_connection
+		set_minion_info
 		whiptail_end_settings
 	fi
 
@@ -487,6 +507,9 @@ if ! [[ -f $install_opt_file ]]; then
 		percentage=0
 		es_heapsize
 		ls_heapsize
+		if [[ $needs_proxy ]]; then
+			set_proxy
+		fi
 		set_redirect
 		# Generate Interface Vars
 		generate_interface_vars
@@ -535,6 +558,9 @@ if ! [[ -f $install_opt_file ]]; then
 		export PATCHSCHEDULENAME=$PATCHSCHEDULENAME
 		export INTERFACE="bond0"
 		export CORECOUNT=$lb_procs
+		export LSHOSTNAME=$HOSTNAME
+		export LSHEAP=$LS_HEAP_SIZE
+		export CPUCORES=$num_cpu_cores
 		logCmd "so-minion -o=setup" 
 		title "Creating Global SLS"
 
@@ -589,9 +615,9 @@ if ! [[ -f $install_opt_file ]]; then
 		add_web_user
 		info "Restarting SOC to pick up initial user"
 		logCmd "so-soc-restart"
+		title "Setting up Elastic Fleet"
+                logCmd "so-elastic-fleet-setup"
 		if [[ ! $is_import ]]; then
-			title "Setting up Elastic Fleet"
-			logCmd "so-elastic-fleet-setup"
 			title "Setting up Playbook"
 			logCmd "so-playbook-reset"
 		fi
@@ -620,6 +646,7 @@ if ! [[ -f $install_opt_file ]]; then
 		configure_minion "$minion_type"
 		drop_install_options
 		checkin_at_boot
+		logCmd "salt-call state.apply setup.highstate_cron --local --file-root=../salt/"
 		verify_setup
 	fi
 

setup/so-verify

@@ -17,7 +17,7 @@ using_iso() {
     if [ "$setup_type" == "iso" ]; then
         return 0
     fi
-    return 0
+    return 1
 }
 
 # Check entire setup log for errors or unexpected salt states
@@ -36,6 +36,7 @@ log_has_errors() {
         grep -vE "The Salt Master has cached the public key for this node" | \
         grep -vE "Minion failed to authenticate with the master" | \
         grep -vE "Failed to connect to ::1" | \
+	grep -vE "Failed to set locale" | \
         grep -vE "perl-Error-" | \
         grep -vE "Failed:\s*?[0-9]+" | \
         grep -vE "Status .* was not found" | \
@@ -54,7 +55,8 @@ log_has_errors() {
 # For ISO installs, we know nothing else can be running on this server, so there should be
 # nothing in any mail spool dir.
 cron_error_in_mail_spool() {
-    if find /var/spool/mail/ -type f -size +0 &> /dev/null; then
+    count=$(find /var/spool/mail/ -type f -size +0 | wc -l)
+    if [[ $count -ne 0 ]]; then
         return 0
     fi
     return 1

setup/so-whiptail

@@ -816,34 +816,15 @@ whiptail_net_method() {
 	)
 	local proxy_desc="proxy the traffic for git, docker client, wget, curl, ${pkg_mngr}, and various other SO components through a separate server in your environment."
 
-	if [[ $is_minion ]]; then
-		read -r -d '' options_msg <<- EOM
+	read -r -d '' options_msg <<- EOM
 		${options_msg}
-
-		"Direct + Manager" - all traffic passes to the Internet normally, but ${pkg_mngr} updates will instead be pulled from ${mngr_article} manager. 
-
 		"Proxy" - ${proxy_desc}
+	EOM
+	options+=(
+		" Proxy " ""
+	)
 
-		"Proxy + Manager" - proxy all traffic from the "Proxy" option except ${pkg_mngr} updates, which will instead pull from the manager.
-		EOM
-		
-		options+=(
-			" Direct + Manager " ""
-			" Proxy " ""
-			" Proxy + Manager " ""
-		)
-		local height=25
-	else
-		read -r -d '' options_msg <<- EOM
-			${options_msg}
-
-			"Proxy" - ${proxy_desc}
-		EOM
-		options+=(
-			" Proxy " ""
-		)
-		local height=17
-	fi
+	local height=17
 
 	local msg
 	read -r -d '' msg <<- EOM
@@ -1327,6 +1308,28 @@ whiptail_storage_requirements() {
 	whiptail_check_exitstatus $exitstatus
 }
 
+whiptail_ubuntu_notsupported() {
+	[ -n "$TESTING" ] && return
+	
+	read -r -d '' message <<- EOM
+		Ubuntu is not supported for this node type.
+
+		Please use a supported OS or install via ISO.
+	EOM
+	whiptail --title "$whiptail_title"  --msgbox "$message" 14 75
+}
+
+whiptail_ubuntu_warning() {
+	[ -n "$TESTING" ] && return
+	
+	read -r -d '' message <<- EOM
+		Ubuntu support for this node type is limited.
+
+		Please consider using a fully supported OS or install via ISO.
+	EOM
+	whiptail --title "$whiptail_title"  --msgbox "$message" 14 75
+
+}
 whiptail_uppercase_warning() {
 
 	[ -n "$TESTING" ] && return