Merge branch '2.4/dev' into 2.4/taglogs

salt/common/tools/sbin/so-common

@@ -238,7 +238,7 @@ gpg_rpm_import() {
 	        local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/rocky/keys"
 	    fi
 	
-	    RPMKEYS=('RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub')
+	    RPMKEYS=('RPM-GPG-KEY-rockyofficial' 'RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub')
 
 	    for RPMKEY in "${RPMKEYS[@]}"; do
     	        rpm --import $RPMKEYSLOC/$RPMKEY

salt/elasticfleet/install_agent_grid.sls

@@ -10,6 +10,7 @@
 run_installer:
   cmd.script:
     - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux
+    - cwd: /opt/so
     - args: -token={{ GRIDNODETOKEN }}
 
 {% endif %}

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -33,6 +33,7 @@ printf "\n### Stripping out unused components"
 find /nsm/elastic-agent-workspace/elastic-agent-*/data/elastic-agent-*/components -regex '.*fleet.*\|.*packet.*\|.*apm*.*\|.*audit.*\|.*heart.*\|.*cloud.*' -delete 
 
 printf "\n### Tarring everything up again"
+
 for OS in "${OSARCH[@]}"
 do 
   printf "\nCreating tarball for $OS..."
@@ -44,6 +45,7 @@ done
 GOTARGETOS=( "linux" "windows" "darwin" "darwin/arm64" )
 GOARCH="amd64"
 printf "\n### Generating OS packages using the cleaned up tarballs"for GOOS in "${GOTARGETOS[@]}"
+
 do
     if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi
     printf "\n\n### Generating $GOOS/$GOARCH Installer...\n"

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -12,7 +12,7 @@ printf "\n### Create ES Token ###\n"
 ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' |  jq -r .value)
 
 ### Create Outputs & Fleet URLs ###
-printf "\nAdd Manager Elasticsearch Ouput...\n"
+printf "\nAdd Manager Elasticsearch Output...\n"
 ESCACRT=$(openssl x509 -in  /etc/pki/tls/certs/intca.crt)
 JSON_STRING=$( jq -n \
                   --arg ESCACRT "$ESCACRT" \

salt/repo/client/files/rocky/keys/RPM-GPG-KEY-rockyofficial

@@ -1,29 +1,31 @@
 -----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: resf.keykeeper.v1
+Comment: Keykeeper
 
-mQINBGAofzYBEAC6yS1azw6f3wmaVd//3aSy6O2c9+jeetulRQvg2LvhRRS1eNqp
+xsFNBGJ5RksBEADF/Lzssm7uryV6+VHAgL36klyCVcHwvx9Bk853LBOuHVEZWsme
-/x9tbBhfohu/tlDkGpYHV7diePgMml9SZDy1sKlI3tDhx6GZ3xwF0fd1vWBZpmNk
+kbJF3fQG7i7gfCKGuV5XW15xINToe4fBThZteGJziboSZRpkEQ2z3lYcbg34X7+d
-D9gRkUmYBeLotmcXQZ8ZpWLicosFtDpJEYpLUhuIgTKwt4gxJrHvkWsGQiBkJxKD
+co833lkBNgz1v6QO7PmAdY/x76Q6Hx0J9yiJWd+4j+vRi4hbWuh64vUtTd7rPwk8
-u3/RlL4IYA3Ot9iuCBflc91EyAw1Yj0gKcDzbOqjvlGtS3ASXgxPqSfU0uLC9USF
+0y3g4oK1YT0NR0Xm/QUO9vWmkSTVflQ6y82HhHIUrG+1vQnSOrWaC0O1lqUI3Nuo
-uKDnP2tcnlKKGfj0u6VkqISliSuRAzjlKho9Meond+mMIFOTT6qp4xyu+9Dj3IjZ
+b6jTARCmbaPsi+XVQnBbsnPPq6Tblwc+NYJSqj5d9nT0uEXT7Zovj4Je5oWVFXp9
-IC6rBXRU3xi8z0qYptoFZ6hx70NV5u+0XUzDMXdjQ5S859RYJKijiwmfMC7gZQAf
+P1OWkbo2z5XkKjoeobM/zKDESJR78h+YQAN9IOKFjL/u/Gzrk1oEgByCABXOX+H5
-OkdOcicNzen/TwD/slhiCDssHBNEe86Wwu5kmDoCri7GJlYOlWU42Xi0o1JkVltN
+hfucrq5U3bbcKy4e5tYgnnZxqpELv3fN/2l8iZknHEh5aYNT5WXVHpD/8u2rMmwm
-D8ZId+EBDIms7ugSwGOVSxyZs43q2IAfFYCRtyKHFlgHBRe9/KTWPUrnsfKxGJgC
+I9YTEMueEtmVy0ZV3opUzOlC+3ZUwjmvAJtdfJyeVW/VMy3Hw3Ih0Fij91rO613V
-Do3Yb63/IYTvfTJptVfhQtL1AhEAeF1I+buVoJRmBEyYKD9BdU4xQN39VrZKziO3
+7n72ggVlJiX25jYyT4AXlaGfAOMndJNVgBps0RArOBYsJRPnvfHlLi5cfjVd7vYx
-hDIGng/eK6PaPhUdq6XqvmnsZ2h+KVbyoj4cTo2gKCB2XA7O2HLQsuGduHzYKNjf
+QhGX9ODYuvyJ/rW70dMVikeSjlBDKS08tvdqOgtiYy4yhtY4ijQC9BmCE9H9gOxU
-QR9j0djjwTrsvGvzfEzchP19723vYf7GdcLvqtPqzpxSX2FNARpCGXBw9wARAQAB
+FN297iLimAxr0EVsED96fP96TbDGILWsfJuxAvoqmpkElv8J+P1/F7to2QARAQAB
-tDNSZWxlYXNlIEVuZ2luZWVyaW5nIDxpbmZyYXN0cnVjdHVyZUByb2NreWxpbnV4
+zU9Sb2NreSBFbnRlcnByaXNlIFNvZnR3YXJlIEZvdW5kYXRpb24gLSBSZWxlYXNl
-Lm9yZz6JAk4EEwEIADgWIQRwUcRwqSn0VM6+N7cVr12sbXRaYAUCYCh/NgIbDwUL
+IGtleSAyMDIyIDxyZWxlbmdAcm9ja3lsaW51eC5vcmc+wsGKBBMBCAA0BQJieUZL
-CQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRAVr12sbXRaYLFmEACSMvoO1FDdyAbu
+FiEEIcslauFvxUxuZSlJcC1CbTUNJ10CGwMCHgECGQEDCwkHAhUIAxYAAgIiAQAK
-1m6xEzDhs7FgnZeQNzLZECv2j+ggFSJXezlNVOZ5I1I8umBan2ywfKQD8M+IjmrW
+CRBwLUJtNQ0nXWQ5D/9472seOyRO6//bQ2ns3w9lE+aTLlJ5CY0GSTb4xNuyv+AD
-k9/7h9i54t8RS/RN7KNo7ECGnKXqXDPzBBTs1Gwo1WzltAoaDKUfXqQ4oJ4aCP/q
+IXpgvLSMtTR0fp9GV3vMw6QIWsehDqt7O5xKWi+3tYdaXRpb1cvnh8r/oCcvI4uL
-/XPVWEzgpJO1XEezvCq8VXisutyDiXEjjMIeBczxb1hbamQX+jLTIQ1MDJ4Zo1YP
+k8kImNgsx+Cj+drKeQo03vFxBTDi1BTQFkfEt32fA2Aw5gYcGElM717sNMAMQFEH
-zlUqrHW434XC2b1/WbSaylq8Wk9cksca5J+g3FqTlgiWozyy0uxygIRjb6iTzKXk
+P+OW5hYDH4kcLbtUypPXFbcXUbaf6jUjfiEp5lLjqquzAyDPLlkzMr5RVa9n3/rI
-V7SYxeXp3hNTuoUgiFkjh5/0yKWCwx7aQqlHar9GjpxmBDAO0kzOlgtTw//EqTwR
+R6OQp5loPVzCRZMgDLALBU2TcFXLVP+6hAW8qM77c+q/rOysP+Yd+N7GAd0fvEvA
-KnYZLig9FW0PhwvZJUigr0cvs/XXTTb77z/i/dfHkrjVTTYenNyXogPtTtSyxqca
+mfeA4Y6dP0mMRu96EEAJ1qSKFWUul6K6nuqy+JTxktpw8F/IBAz44na17Tf02MJH
-61fbPf0B/S3N43PW8URXBRS0sykpX4SxKu+PwKCqf+OJ7hMEVAapqzTt1q9T7zyB
+GCUWyM0n5vuO5kK+Ykkkwd+v43ZlqDnwG7akDkLwgj6O0QNx2TGkdgt3+C6aHN5S
-QwvCVx8s7WWvXbs2d6ZUrArklgjHoHQcdxJKdhuRmD34AuXWCLW+gH8rJWZpuNl3
+MiF0pi0qYbiN9LO0e05Ai2r3zTFC/pCaBWlG1ph2jx1pDy4yUVPfswWFNfe5I+4i
-+WsPZX4PvjKDgMw6YMcV7zhWX6c0SevKtzt7WP3XoKDuPhK1PMGJQqQ7spegGB+5
+CMHPRFsZNYxQnIA2Prtgt2YMwz3VIGI6DT/Z56Joqw4eOfaJTTQSXCANts/gD7qW
-DZvsJS48Ip0S45Qfmj82ibXaCBJHTNZE8Zs+rdTjQ9DS5qvzRA1sRA1dBb/7OLYE
+D3SZXPc7wQD63TpDEjJdqhmepaTECbxN7x/p+GwIZYWJN+AYhvrfGXfjud3eDu8/
-JmeWf4VZyebm+gc50szsg6Ut2yT8hw==
+i+YIbPKH1TAOMwiyxC106mIL705p+ORf5zATZMyB8Y0OvRIz5aKkBDFZM2QN6A==
-=AiP8
+=PzIf
 -----END PGP PUBLIC KEY BLOCK-----

setup/so-functions

@@ -61,7 +61,7 @@ add_mngr_ip_to_hosts() {
 
 add_socore_user_manager() {
 	info "Adding socore user"
-	logCmd "so_add_user 'socore' '939' '939' '/opt/so'" 
+	logCmd "so_add_user socore 939 939 /opt/so" 
 }
 
 add_web_user() {
@@ -967,15 +967,15 @@ detect_os() {
 }
 
 download_elastic_agent_artifacts() {
-   #TODO - ISO
 
-   logCmd "mkdir -p /nsm/elastic-fleet/artifacts/beats/elastic-agent/"
+	if [[ $is_iso ]]; then
-
+		logCmd "tar -xf /nsm/elastic-fleet/artifacts/beats/elastic-agent_SO-$SOVERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/"
-   logCmd "curl --retry 5 --retry-delay 60 https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$SOVERSION.tar.gz --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz"
+	else
-
+		logCmd "mkdir -p /nsm/elastic-fleet/artifacts/beats/elastic-agent/"
-   logCmd "tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/"
+		logCmd "curl --retry 5 --retry-delay 60 https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$SOVERSION.tar.gz --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz"
-
+		logCmd "tar -xf /nsm/elastic-fleet/artifacts/beats/elastic-agent_SO-$SOVERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/"
-	}
+	fi
+}
 
 installer_progress_loop() {
 	local i=0
@@ -1946,9 +1946,11 @@ securityonion_repo() {
 			logCmd "dnf repolist all"
 		fi
 		if [[ $waitforstate ]]; then
-			# Build the repo locally so we can use it
+			if [[ ! $is_airgap ]]; then
-			echo "Syncing Repo"
+				# Build the repo locally so we can use it
-			repo_sync_local
+				echo "Syncing Repo"
+				repo_sync_local
+			fi
 		fi
 	fi
 }
@@ -2194,12 +2196,12 @@ setup_salt_master_dirs() {
 		logCmd "rsync -avh --exclude 'TRANS.TBL' /home/$INSTALLUSERNAME/SecurityOnion/pillar/* $default_salt_dir/pillar/"
 		logCmd "rsync -avh --exclude 'TRANS.TBL' /home/$INSTALLUSERNAME/SecurityOnion/salt/* $default_salt_dir/salt/"
 		logCmd "mkdir -p $local_salt_dir/salt/zeek/policy/intel"
-		logCmd "cp -Rv /home/$INSTALLUSERNAME/SecurityOnion/files/intel.dat $local_salt_dir/salt/zeek/policy/intel/"
+		logCmd "touch $local_salt_dir/salt/zeek/policy/intel/intel.dat"
 	else
 		logCmd "cp -Rv ../pillar/* $default_salt_dir/pillar/"
 		logCmd "cp -Rv ../salt/* $default_salt_dir/salt/"
 		logCmd "mkdir -p $local_salt_dir/salt/zeek/policy/intel"
-		logCmd "cp -Rv files/intel.dat $local_salt_dir/salt/zeek/policy/intel/"
+		logCmd "touch $local_salt_dir/salt/zeek/policy/intel/intel.dat"
 	fi
 
 	info "Chown the salt dirs on the manager for socore"
@@ -2329,8 +2331,8 @@ so_add_user() {
 	if [ "$5" ]; then local pass=$5; fi
 
 	info "Add $username user"
-	logCmd "groupadd --gid '$gid' '$username'"
+	logCmd "groupadd --gid $gid $username"
-	logCmd "useradd -m --uid '$uid' --gid '$gid' --home-dir '$home_dir' '$username'"
+	logCmd "useradd -m --uid $uid --gid $gid --home-dir $home_dir $username"
 
 	# If a password has been passed in, set the password
 	if [ "$pass" ]; then

setup/so-setup

@@ -335,45 +335,53 @@ process_installtype
 
 # If this is not an automated install prompt
 if ! [[ -f $install_opt_file ]]; then
-
+	# If you are a manager ask ALL the manager things here. I know there is code re-use but this makes it easier to add new roles
-	# If you are a manager ask ALL the manager things here. I know there is code re-use but this makes it easier to add new roles.
 	if [[ $is_eval ]]; then
+		# waitforstate means we will run the full salt state at the end. This is for only nodes running the salt-master service
 		waitforstate=true
-		#ubuntu_check
+		# Does this role have monitoring interfaces?
 		monints=true
+		# Prompt the user to accept the elastic license
 		check_elastic_license
+		# If it is an install from ISO is this airgap?
+		[[ $is_iso ]] && whiptail_airgap
+		# Make sure minimum requirements are met
 		check_requirements "manager"
+		# Do networking things
 		networking_needful
-		collect_net_method
+		# Do we need a proxy?
+		[[ ! $is_airgap ]] && collect_net_method
+		# Do we need to change the dockernet subnet?
 		collect_dockernet
-		if [[ $is_iso ]]; then
+		# Are we in the clouds?
-			whiptail_airgap
+		[[ ! $is_airgap ]] && detect_cloud
-		fi
+		# Sets some minion info
-		detect_cloud
 		set_minion_info
 		set_default_log_size >> $setup_log 2>&1
 		info "Verifying all network devices are managed by Network Manager that should be"
 		check_network_manager_conf
 		set_network_dev_status_list
+		# What NIC for watching network traffic?
 		whiptail_sensor_nics
+		# How many cores do we have?
 		calculate_useable_cores
+		# What is the web user?
 		collect_webuser_inputs
+		# How are we accessing the UI?
 		get_redirect
+		# Does the user want to allow access to the UI?
 		collect_so_allow
 		whiptail_end_settings
 	elif [[ $is_standalone ]]; then
 		waitforstate=true
-		#ubuntu_check
 		monints=true
 		check_elastic_license
+		[[ $is_iso ]] && whiptail_airgap
 		check_requirements "manager"
 		networking_needful
-		collect_net_method
+		[[ ! $is_airgap ]] && collect_net_method
 		collect_dockernet
-		if [[ $is_iso ]]; then
+		[[ ! $is_airgap ]] && detect_cloud
-			whiptail_airgap
-		fi
-		detect_cloud
 		set_minion_info
 		set_default_log_size >> $setup_log 2>&1
 		info "Verifying all network devices are managed by Network Manager that should be"
@@ -389,14 +397,12 @@ if ! [[ -f $install_opt_file ]]; then
 		check_elastic_license
 		waitforstate=true
 		#ubuntu_check
+		[[ $is_iso ]] && whiptail_airgap
 		check_requirements "manager"
 		networking_needful
-		collect_net_method
+		[[ ! $is_airgap ]] && collect_net_method
 		collect_dockernet
-		if [[ $is_iso ]]; then
+		[[ ! $is_airgap ]] && detect_cloud
-			whiptail_airgap
-		fi
-		detect_cloud
 		set_minion_info
 		set_default_log_size >> $setup_log 2>&1
 		info "Verifying all network devices are managed by Network Manager that should be"
@@ -410,15 +416,12 @@ if ! [[ -f $install_opt_file ]]; then
 	elif [[ $is_managersearch ]]; then
 		check_elastic_license
 		waitforstate=true
-		#ubuntu_check
+		[[ $is_iso ]] && whiptail_airgap
 		check_requirements "manager"
 		networking_needful
-		collect_net_method
+		[[ ! $is_airgap ]] && collect_net_method
 		collect_dockernet
-		if [[ $is_iso ]]; then
+		[[ ! $is_airgap ]] && detect_cloud
-			whiptail_airgap
-		fi
-		detect_cloud
 		set_minion_info
 		set_default_log_size >> $setup_log 2>&1
 		info "Verifying all network devices are managed by Network Manager that should be"
@@ -430,7 +433,6 @@ if ! [[ -f $install_opt_file ]]; then
 		collect_so_allow
 		whiptail_end_settings
 	elif [[ $is_sensor ]]; then
-		#ubuntu_check
 		installer_prereq_packages
 		monints=true
 		check_requirements "sensor"
@@ -459,7 +461,6 @@ if ! [[ -f $install_opt_file ]]; then
 		whiptail_end_settings
 
 	elif [[ $is_searchnode ]]; then
-		#ubuntu_check
 		installer_prereq_packages
 		check_requirements "elasticsearch"
 		networking_needful
@@ -473,7 +474,6 @@ if ! [[ -f $install_opt_file ]]; then
 		whiptail_end_settings
 
 	elif [[ $is_heavynode ]]; then
-		#ubuntu_check
 		installer_prereq_packages
 		monints=true
 		check_requirements "heavynode"
@@ -486,29 +486,26 @@ if ! [[ -f $install_opt_file ]]; then
 		whiptail_end_settings
 
 	elif [[ $is_idh ]]; then
-		#ubuntu_check
 		installer_prereq_packages
 		check_requirements "idh"
 		networking_needful
 		collect_mngr_hostname
 		add_mngr_ip_to_hosts
 		check_manager_connection
-		#collect_idh_services (this may be added back sometime in the future)
 		collect_idh_preferences
 		set_minion_info
 		whiptail_end_settings
 
 	elif [[ $is_import ]]; then
-		#ubuntu_check
 		waitforstate=true
 		monints=true
+		[[ $is_iso ]] && whiptail_airgap
 		check_elastic_license
 		check_requirements "import"
 		networking_needful
-		if [[ $is_iso ]]; then
+		[[ ! $is_airgap ]] && detect_cloud
-			whiptail_airgap
+		collect_dockernet
-		fi
+		[[ ! $is_airgap ]] && collect_net_method
-		detect_cloud
 		set_minion_info
 		set_default_log_size >> $setup_log 2>&1
 		info "Verifying all network devices are managed by Network Manager that should be"
@@ -521,7 +518,6 @@ if ! [[ -f $install_opt_file ]]; then
 		whiptail_end_settings
 
 	elif [[ $is_receiver ]]; then
-		#ubuntu_check
 		installer_prereq_packages
 		check_requirements "receiver"
 		networking_needful