merge with dev and fix merge conflict in so-functions https://github.com/Security-Onion-Solutions/securityonion/issues/3264

2025-12-06 17:22:49 +01:00 · 2021-04-20 10:03:15 -04:00
parent 5cda35db0a 3b203b9a31
commit 115764ae38
91 changed files with 1860 additions and 672 deletions
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -44,6 +44,21 @@ logCmd() {
 }
 ### End Logging Section ###

+airgap_repo() {
+	# Remove all the repo files
+	rm -rf /etc/yum.repos.d/*
+	echo "[airgap_repo]" > /etc/yum.repos.d/airgap_repo.repo
+	if $is_manager; then
+	    echo "baseurl=https://$HOSTNAME/repo" >> /etc/yum.repos.d/airgap_repo.repo
+	else
+        echo "baseurl=https://$MSRV/repo" >> /etc/yum.repos.d/airgap_repo.repo
+	fi
+    echo "gpgcheck=1" >> /etc/yum.repos.d/airgap_repo.repo
+    echo "sslverify=0" >> /etc/yum.repos.d/airgap_repo.repo
+    echo "name=Airgap Repo" >> /etc/yum.repos.d/airgap_repo.repo
+    echo "enabled=1" >> /etc/yum.repos.d/airgap_repo.repo
+}
+
 airgap_rules() {
 	# Copy the rules for suricata if using Airgap
 	mkdir -p /nsm/repo/rules
@@ -147,6 +162,25 @@ check_hive_init() {
 	docker rm so-thehive
 }

+check_manager_state() {
+	echo "Checking state of manager services. This may take a moment..."
+	retry 2 15 "__check_so_status" >> $setup_log 2>&1 && retry 2 15 "__check_salt_master" >> $setup_log 2>&1 && return 0 || return 1
+}
+
+__check_so_status() {
+	local so_status_output
+	so_status_output=$($sshcmd -i /root/.ssh/so.key soremote@"$MSRV" cat /opt/so/log/sostatus/status.log)
+	[[ -z $so_status_output ]] && so_status_output=1
+	return $so_status_output
+}
+
+__check_salt_master() {
+	local salt_master_status
+	salt_master_status=$($sshcmd -i /root/.ssh/so.key soremote@"$MSRV" systemctl is-active --quiet salt-master)
+	[[ -z $salt_master_status ]] && salt_master_status=1
+	return $salt_master_status
+}
+
 check_network_manager_conf() {
 	local gmdconf="/usr/lib/NetworkManager/conf.d/10-globally-managed-devices.conf"
 	local nmconf="/etc/NetworkManager/NetworkManager.conf"
@@ -159,11 +193,6 @@ check_network_manager_conf() {
 			systemctl restart NetworkManager
 		} >> "$setup_log" 2>&1
 	fi
-	
-	#if test -f "$nmconf"; then
-#		sed -i 's/managed=false/managed=true/g' "$nmconf" >> "$setup_log" 2>&1
-#		systemctl restart NetworkManager >> "$setup_log" 2>&1
-#	fi

 	if [[ ! -d "$preupdir" ]]; then
 		mkdir "$preupdir" >> "$setup_log" 2>&1
@@ -400,7 +429,7 @@ collect_hostname() {

 	if [[ $HOSTNAME == 'securityonion' ]]; then # Will only check HOSTNAME=securityonion once
 		if ! (whiptail_avoid_default_hostname); then
-			whiptail_set_hostname
+			whiptail_set_hostname "$HOSTNAME"
 		fi
 	fi

@@ -486,6 +515,22 @@ collect_node_ls_pipeline_worker_count() {
 	done
 }

+collect_ntp_servers() {
+	if whiptail_ntp_ask; then
+		[[ $is_airgap ]] && ntp_string=""
+		whiptail_ntp_servers "$ntp_string"
+
+		while ! valid_ntp_list "$ntp_string"; do
+			whiptail_invalid_input
+			whiptail_ntp_servers "$ntp_string"
+		done
+
+		IFS="," read -r -a ntp_servers <<< "$ntp_string" # Split string on commas into array
+	else
+		ntp_servers=()
+	fi
+}
+
 collect_oinkcode() {
 	whiptail_oinkcode

@@ -537,7 +582,7 @@ collect_patch_schedule_name_import() {

 collect_proxy() {
 	[[ -n $TESTING ]] && return
-	collect_proxy_details
+	collect_proxy_details || return
 	while ! proxy_validate; do
 		if whiptail_invalid_proxy; then
 			collect_proxy_details no_ask
@@ -581,7 +626,9 @@ collect_proxy_details() {
 		else
 			so_proxy="$proxy_addr"
 		fi
-		export proxy
+		export so_proxy
+	else
+		return 1
 	fi
 }

@@ -702,6 +749,42 @@ configure_minion() {
 	} >> "$setup_log" 2>&1
 }

+configure_ntp() {
+	local chrony_conf=/etc/chrony.conf
+
+	# Install chrony if it isn't already installed
+	if ! command -v chronyc &> /dev/null; then
+		yum -y install chrony
+	fi
+
+	[[ -f $chrony_conf ]] && mv $chrony_conf "$chrony_conf.bak"
+
+	printf '%s\n' "# NTP server list" > $chrony_conf
+
+	# Build list of servers
+	for addr in "${ntp_servers[@]}"; do
+		echo "server $addr iburst" >> $chrony_conf
+	done
+
+	printf '\n%s\n' "# Config options" >> $chrony_conf
+
+	printf '%s\n' \
+		'driftfile /var/lib/chrony/drift' \
+		'makestep 1.0 3' \
+		'rtcsync' \
+		'logdir /var/log/chrony' >> $chrony_conf
+
+	systemctl enable chronyd
+	systemctl restart chronyd
+
+	# Tell the chrony daemon to sync time & update the system time
+	# Since these commands only make a call to chronyd, wait after each command to make sure the changes are made
+	printf "Syncing chrony time to server: "
+	chronyc -a 'burst 4/4' && sleep 30
+	printf "Forcing chrony to update the time: "
+	chronyc -a makestep && sleep 30
+}
+
 checkin_at_boot() {
 	local minion_config=/etc/salt/minion

@@ -950,7 +1033,7 @@ create_repo() {

 detect_cloud() {
  echo "Testing if setup is running on a cloud instance..." | tee -a "$setup_log"
-  if ( curl --fail -s -m 5 http://169.254.169.254/latest/meta-data/instance-id > /dev/null ) || ( dmidecode -s bios-vendor | grep -q Google > /dev/null); then export is_cloud="true"; fi
+  if ( curl --fail -s -m 5 http://169.254.169.254/latest/meta-data/instance-id > /dev/null ) || ( dmidecode -s bios-vendor | grep -q Google > /dev/null) || [ -f /var/log/waagent.log ]; then export is_cloud="true"; fi
 }

 detect_os() {
@@ -1054,40 +1137,11 @@ disable_ipv6() {
 	} >> /etc/sysctl.conf
 }

-#disable_misc_network_features() {
-#	filter_unused_nics
-#	if [ ${#filtered_nics[@]} -ne 0 ]; then
-#		for unused_nic in "${filtered_nics[@]}"; do
-#			if [ -n "$unused_nic" ]; then
-#				echo "Disabling unused NIC: $unused_nic" >> "$setup_log" 2>&1
-#
-#				# Disable DHCPv4/v6 and autoconnect
-#				nmcli con mod "$unused_nic" \
-#					ipv4.method disabled \
-#					ipv6.method ignore \
-#					connection.autoconnect "no" >> "$setup_log" 2>&1
-#
-#				# Flush any existing IPs
-#				ip addr flush "$unused_nic" >> "$setup_log" 2>&1
-#			fi
-#		done
-#	fi
-#	# Disable IPv6
-#	{ 
-#		echo "net.ipv6.conf.all.disable_ipv6 = 1"
-#		echo "net.ipv6.conf.default.disable_ipv6 = 1"
-#		echo "net.ipv6.conf.lo.disable_ipv6 = 1" 
-#	} >> /etc/sysctl.conf
-#}
-
 docker_install() {

 	if [ $OS = 'centos' ]; then
 		{
 			yum clean expire-cache;
-			if [[ ! $is_airgap ]]; then
-			  yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo;
-			fi
 			if [[ ! $is_iso ]]; then
 			  yum -y install docker-ce-20.10.5-3.el7 containerd.io-1.4.4-3.1.el7;
 			fi
@@ -1436,8 +1490,6 @@ install_cleanup() {
 		info "Removing so-setup permission entry from sudoers file"
 		sed -i '/so-setup/d' /etc/sudoers
 	fi
-
-	so-ssh-harden -q
 }

 import_registry_docker() {
@@ -1539,8 +1591,7 @@ manager_pillar() {
 	printf '%s\n'\
 		"  kratoskey: '$KRATOSKEY'"\
 		"" >> "$pillar_file"
-
-  }
+}

 manager_global() {
 	local global_pillar="$local_salt_dir/pillar/global.sls"
@@ -1564,7 +1615,6 @@ manager_global() {
 		"global:"\
 		"  soversion: '$SOVERSION'"\
 		"  hnmanager: '$HNMANAGER'"\
-		"  ntpserver: '$NTPSERVER'"\
 		"  dockernet: '$DOCKERNET'"\
 		"  mdengine: '$ZEEKVERSION'"\
 		"  ids: '$NIDS'"\
@@ -1718,7 +1768,6 @@ manager_global() {
 		"  bip: '$DOCKERBIP'"\
 		"redis_settings:"\
 		"  redis_maxmemory: 812" >> "$global_pillar"
-
 		
 	printf '%s\n'  '----' >> "$setup_log" 2>&1
 }
@@ -1781,6 +1830,19 @@ network_setup() {
 	} >> "$setup_log" 2>&1
 }

+ntp_pillar() {
+	local pillar_file="$temp_install_dir"/pillar/minions/"$MINION_ID".sls
+
+	if [[ ${#ntp_servers[@]} -gt 0 ]]; then
+		printf '%s\n'\
+			"ntp:"\
+			"  servers:" >> "$pillar_file"
+			for addr in "${ntp_servers[@]}"; do
+				printf '%s\n' "    - '$addr'" >> "$pillar_file"
+			done
+	fi
+}
+
 parse_install_username() {
 	# parse out the install username so things copy correctly
 	INSTALLUSERNAME=${SUDO_USER:-${USER}}
@@ -1829,12 +1891,13 @@ print_salt_state_apply() {
 }

 proxy_validate() {
+	echo "Testing proxy..."
 	local test_url="https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS"
-	proxy_test_err=$(curl -sS "$test_url" --proxy "$so_proxy" 2>&1)
+	proxy_test_err=$(curl -sS "$test_url" --proxy "$so_proxy" --connect-timeout 5 2>&1) # set short connection timeout so user doesn't sit waiting for proxy test to timeout
 	local ret=$?

 	if [[ $ret != 0 ]]; then
-		error "Could not reach $test_url using proxy $so_proxy"
+		error "Could not reach $test_url using proxy provided"
 		error "Received error: $proxy_test_err"
 		if [[ -n $TESTING ]]; then
 			error "Exiting setup"
@@ -1988,11 +2051,6 @@ saltify() {

 	# Install updates and Salt
 	if [ $OS = 'centos' ]; then
-		set_progress_str 5 'Installing Salt repo'
-		{
-			sudo rpm --import https://repo.saltstack.com/py3/redhat/7/x86_64/archive/3002.5/SALTSTACK-GPG-KEY.pub;
-			cp ./yum_repos/saltstack.repo /etc/yum.repos.d/saltstack.repo;
-		} >> "$setup_log" 2>&1
 		set_progress_str 6 'Installing various dependencies'
 		if [[ ! $is_iso ]]; then
 		  logCmd "yum -y install wget nmap-ncat"
@@ -2001,49 +2059,31 @@ saltify() {
 			'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'FLEET' | 'HELIXSENSOR' | 'STANDALONE'| 'IMPORT')
 				reserve_group_ids >> "$setup_log" 2>&1
 				if [[ ! $is_iso ]]; then
-				  logCmd "yum -y install epel-release"
-				  logCmd "yum -y install sqlite argon2 curl mariadb-devel python3-pip"
-				  retry 50 10 "pip3 install --user influxdb" >> "$setup_log" 2>&1 || exit 1
+				  logCmd "yum -y install sqlite argon2 curl mariadb-devel"
 				fi 
 				# Download Ubuntu Keys in case manager updates = 1
 				mkdir -p /opt/so/gpg >> "$setup_log" 2>&1
 				if [[ ! $is_airgap ]]; then
-				  logCmd "wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com/py3/ubuntu/18.04/amd64/archive/3002.5/SALTSTACK-GPG-KEY.pub"
+				  logCmd "wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com/py3/ubuntu/18.04/amd64/archive/3003/SALTSTACK-GPG-KEY.pub"
 				  logCmd "wget -q --inet4-only -O /opt/so/gpg/docker.pub https://download.docker.com/linux/ubuntu/gpg"
 				  logCmd "wget -q --inet4-only -O /opt/so/gpg/GPG-KEY-WAZUH https://packages.wazuh.com/key/GPG-KEY-WAZUH"
-				  logCmd "cp ./yum_repos/wazuh.repo /etc/yum.repos.d/wazuh.repo"
 				fi
 				set_progress_str 7 'Installing salt-master'
 				if [[ ! $is_iso ]]; then
-				  logCmd "yum -y install salt-master-3002.5"
+				  logCmd "yum -y install salt-master-3003"
 				fi
 				systemctl enable salt-master >> "$setup_log" 2>&1
 				;;
 			*)
-				if [ "$MANAGERUPDATES" = '1' ]; then
-					{
-						if [[ ! $is_airgap ]]; then
-						  # Create the GPG Public Key for the Salt Repo
-						  cp ./public_keys/salt.pem /etc/pki/rpm-gpg/saltstack-signing-key;
-
-						  # Copy repo files over
-						  cp ./yum_repos/saltstack.repo /etc/yum.repos.d/saltstack.repo;
-						else
-						  info "This is airgap"
-						fi
-					} >> "$setup_log" 2>&1
-				fi
 				;;
 		esac
 		if [[ ! $is_airgap ]]; then
-		  cp ./yum_repos/wazuh.repo /etc/yum.repos.d/wazuh.repo >> "$setup_log" 2>&1
 		  yum clean expire-cache >> "$setup_log" 2>&1
 		fi
 		set_progress_str 8 'Installing salt-minion & python modules'
 		{
 			if [[ ! $is_iso ]]; then
-			  yum -y install epel-release
-			  yum -y install salt-minion-3002.5\
+			  yum -y install salt-minion-3003\
 			   	python3\
 					python36-docker\
 					python36-dateutil\
@@ -2095,8 +2135,8 @@ saltify() {
 			'MANAGER' | 'EVAL' | 'MANAGERSEARCH' | 'STANDALONE' | 'IMPORT' | 'HELIXSENSOR')

 				# Add saltstack repo(s)
-				wget -q --inet4-only -O - https://repo.saltstack.com"$py_ver_url_path"/ubuntu/"$ubuntu_version"/amd64/archive/3002.5/SALTSTACK-GPG-KEY.pub | apt-key add - >> "$setup_log" 2>&1
-				echo "deb http://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3002.5 $OSVER main" > /etc/apt/sources.list.d/saltstack.list 2>> "$setup_log"
+				wget -q --inet4-only -O - https://repo.saltstack.com"$py_ver_url_path"/ubuntu/"$ubuntu_version"/amd64/archive/3003/SALTSTACK-GPG-KEY.pub | apt-key add - >> "$setup_log" 2>&1
+				echo "deb http://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3003 $OSVER main" > /etc/apt/sources.list.d/saltstack.list 2>> "$setup_log"

 				# Add Docker repo
 				curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - >> "$setup_log" 2>&1
@@ -2104,7 +2144,7 @@ saltify() {

 				# Get gpg keys
 				mkdir -p /opt/so/gpg >> "$setup_log" 2>&1
-				wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com$py_ver_url_path/ubuntu/"$ubuntu_version"/amd64/archive/3002.5/SALTSTACK-GPG-KEY.pub >> "$setup_log" 2>&1
+				wget -q --inet4-only -O /opt/so/gpg/SALTSTACK-GPG-KEY.pub https://repo.saltstack.com$py_ver_url_path/ubuntu/"$ubuntu_version"/amd64/archive/3003/SALTSTACK-GPG-KEY.pub >> "$setup_log" 2>&1
 				wget -q --inet4-only -O /opt/so/gpg/docker.pub https://download.docker.com/linux/ubuntu/gpg >> "$setup_log" 2>&1
 				wget -q --inet4-only -O /opt/so/gpg/GPG-KEY-WAZUH https://packages.wazuh.com/key/GPG-KEY-WAZUH >> "$setup_log" 2>&1

@@ -2117,7 +2157,7 @@ saltify() {
 				set_progress_str 6 'Installing various dependencies'
 				retry 50 10 "apt-get -y install sqlite3 argon2 libssl-dev" >> "$setup_log" 2>&1 || exit 1
 				set_progress_str 7 'Installing salt-master'
-				retry 50 10 "apt-get -y install salt-master=3002.5+ds-1" >> "$setup_log" 2>&1 || exit 1
+				retry 50 10 "apt-get -y install salt-master=3003+ds-1" >> "$setup_log" 2>&1 || exit 1
 				retry 50 10 "apt-mark hold salt-master" >> "$setup_log" 2>&1 || exit 1
 				;;
 			*)
@@ -2128,14 +2168,14 @@ saltify() {
 				echo "Using apt-key add to add SALTSTACK-GPG-KEY.pub and GPG-KEY-WAZUH" >> "$setup_log" 2>&1
 				apt-key add "$temp_install_dir"/gpg/SALTSTACK-GPG-KEY.pub >> "$setup_log" 2>&1
 				apt-key add "$temp_install_dir"/gpg/GPG-KEY-WAZUH >> "$setup_log" 2>&1
-				echo "deb http://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3002.5/ $OSVER main" > /etc/apt/sources.list.d/saltstack.list 2>> "$setup_log"
+				echo "deb http://repo.saltstack.com$py_ver_url_path/ubuntu/$ubuntu_version/amd64/archive/3003/ $OSVER main" > /etc/apt/sources.list.d/saltstack.list 2>> "$setup_log"
 				echo "deb https://packages.wazuh.com/3.x/apt/ stable main" > /etc/apt/sources.list.d/wazuh.list 2>> "$setup_log"
 				;;
 		esac

 		retry 50 10 "apt-get update" >> "$setup_log" 2>&1 || exit 1
 		set_progress_str 8 'Installing salt-minion & python modules'
-		retry 50 10 "apt-get -y install salt-minion=3002.5+ds-1 salt-common=3002.5+ds-1" >> "$setup_log" 2>&1 || exit 1
+		retry 50 10 "apt-get -y install salt-minion=3003+ds-1 salt-common=3003+ds-1" >> "$setup_log" 2>&1 || exit 1
 		retry 50 10 "apt-mark hold salt-minion salt-common" >> "$setup_log" 2>&1 || exit 1
 		if [[ $OSVER != 'xenial' ]]; then
 			retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging python3-influxdb" >> "$setup_log" 2>&1 || exit 1
@@ -2265,6 +2305,30 @@ secrets_pillar(){
  fi
 }

+securityonion_repo() {
+	# Remove all the current repos
+	if [[ "$OS" == "centos" ]]; then
+	    if [[ "$INTERWEBS" == "AIRGAP" ]]; then
+		    echo "This is airgap I don't need to add this repo"
+		else
+	        mkdir -p /root/oldrepos
+	        mv -v /etc/yum.repos.d/* /root/oldrepos/
+			ls -la /etc/yum.repos.d/
+		    rm -rf /etc/yum.repos.d
+			yum clean all
+			yum repolist all
+			mkdir -p /etc/yum.repos.d
+		    if [[ ! $is_manager && "$MANAGERUPDATES" == "1" ]]; then
+		        cp -f ../salt/repo/client/files/centos/securityonioncache.repo /etc/yum.repos.d/
+		    else
+		        cp -f ../salt/repo/client/files/centos/securityonion.repo /etc/yum.repos.d/
+		    fi
+		fi
+	else
+        echo "This is Ubuntu"
+	fi
+}
+
 set_base_heapsizes() {
 	es_heapsize
 	ls_heapsize
@@ -2672,7 +2736,8 @@ update_sudoers() {

 update_packages() {
 	if [ "$OS" = 'centos' ]; then
-		yum -y update >> "$setup_log"
+	    yum repolist >> /dev/null
+        yum -y update --exclude=salt* >> "$setup_log"
 	else
 		retry 50 10 "apt-get -y update" >> "$setup_log" 2>&1 || exit 1
 		retry 50 10 "apt-get -y upgrade" >> "$setup_log" 2>&1 || exit 1