Merge branch '2.4/dev' into 2.4/taglogs

2025-12-06 09:12:45 +01:00 · 2023-05-15 12:13:44 -04:00
parent 64726af69c 9a3c997779
commit b381c51246
9 changed files with 87 additions and 84 deletions
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -238,7 +238,7 @@ gpg_rpm_import() {
 	        local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/rocky/keys"
 	    fi
 	
-	    RPMKEYS=('RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub')
+	    RPMKEYS=('RPM-GPG-KEY-rockyofficial' 'RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub')

 	    for RPMKEY in "${RPMKEYS[@]}"; do
    	        rpm --import $RPMKEYSLOC/$RPMKEY
--- a/salt/elasticfleet/install_agent_grid.sls
+++ b/salt/elasticfleet/install_agent_grid.sls
@@ -10,6 +10,7 @@
 run_installer:
  cmd.script:
    - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux
+    - cwd: /opt/so
    - args: -token={{ GRIDNODETOKEN }}

 {% endif %}
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers
@@ -33,6 +33,7 @@ printf "\n### Stripping out unused components"
 find /nsm/elastic-agent-workspace/elastic-agent-*/data/elastic-agent-*/components -regex '.*fleet.*\|.*packet.*\|.*apm*.*\|.*audit.*\|.*heart.*\|.*cloud.*' -delete 

 printf "\n### Tarring everything up again"
+
 for OS in "${OSARCH[@]}"
 do 
  printf "\nCreating tarball for $OS..."
@@ -44,6 +45,7 @@ done
 GOTARGETOS=( "linux" "windows" "darwin" "darwin/arm64" )
 GOARCH="amd64"
 printf "\n### Generating OS packages using the cleaned up tarballs"for GOOS in "${GOTARGETOS[@]}"
+
 do
    if [[ $GOOS == 'darwin/arm64' ]]; then GOOS="darwin" && GOARCH="arm64"; fi
    printf "\n\n### Generating $GOOS/$GOARCH Installer...\n"
@@ -56,4 +58,4 @@ do
 done

 printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace"
-rm -rf /nsm/elastic-agent-workspace
+rm -rf /nsm/elastic-agent-workspace
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
@@ -12,7 +12,7 @@ printf "\n### Create ES Token ###\n"
 ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' |  jq -r .value)

 ### Create Outputs & Fleet URLs ###
-printf "\nAdd Manager Elasticsearch Ouput...\n"
+printf "\nAdd Manager Elasticsearch Output...\n"
 ESCACRT=$(openssl x509 -in  /etc/pki/tls/certs/intca.crt)
 JSON_STRING=$( jq -n \
                  --arg ESCACRT "$ESCACRT" \
@@ -98,4 +98,4 @@ salt-call state.apply elasticfleet queue=True

 # Generate installers & install Elastic Agent on the node
 so-elastic-agent-gen-installers
-salt-call state.apply elasticfleet.install_agent_grid queue=True
+salt-call state.apply elasticfleet.install_agent_grid queue=True
--- a/salt/influxdb/templates/dashboard-security_onion_performance.json
+++ b/salt/influxdb/templates/dashboard-security_onion_performance.json
--- a/salt/repo/client/files/rocky/keys/RPM-GPG-KEY-rockyofficial
+++ b/salt/repo/client/files/rocky/keys/RPM-GPG-KEY-rockyofficial
@@ -1,29 +1,31 @@
 -----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: resf.keykeeper.v1
+Comment: Keykeeper

-mQINBGAofzYBEAC6yS1azw6f3wmaVd//3aSy6O2c9+jeetulRQvg2LvhRRS1eNqp
-/x9tbBhfohu/tlDkGpYHV7diePgMml9SZDy1sKlI3tDhx6GZ3xwF0fd1vWBZpmNk
-D9gRkUmYBeLotmcXQZ8ZpWLicosFtDpJEYpLUhuIgTKwt4gxJrHvkWsGQiBkJxKD
-u3/RlL4IYA3Ot9iuCBflc91EyAw1Yj0gKcDzbOqjvlGtS3ASXgxPqSfU0uLC9USF
-uKDnP2tcnlKKGfj0u6VkqISliSuRAzjlKho9Meond+mMIFOTT6qp4xyu+9Dj3IjZ
-IC6rBXRU3xi8z0qYptoFZ6hx70NV5u+0XUzDMXdjQ5S859RYJKijiwmfMC7gZQAf
-OkdOcicNzen/TwD/slhiCDssHBNEe86Wwu5kmDoCri7GJlYOlWU42Xi0o1JkVltN
-D8ZId+EBDIms7ugSwGOVSxyZs43q2IAfFYCRtyKHFlgHBRe9/KTWPUrnsfKxGJgC
-Do3Yb63/IYTvfTJptVfhQtL1AhEAeF1I+buVoJRmBEyYKD9BdU4xQN39VrZKziO3
-hDIGng/eK6PaPhUdq6XqvmnsZ2h+KVbyoj4cTo2gKCB2XA7O2HLQsuGduHzYKNjf
-QR9j0djjwTrsvGvzfEzchP19723vYf7GdcLvqtPqzpxSX2FNARpCGXBw9wARAQAB
-tDNSZWxlYXNlIEVuZ2luZWVyaW5nIDxpbmZyYXN0cnVjdHVyZUByb2NreWxpbnV4
-Lm9yZz6JAk4EEwEIADgWIQRwUcRwqSn0VM6+N7cVr12sbXRaYAUCYCh/NgIbDwUL
-CQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRAVr12sbXRaYLFmEACSMvoO1FDdyAbu
-1m6xEzDhs7FgnZeQNzLZECv2j+ggFSJXezlNVOZ5I1I8umBan2ywfKQD8M+IjmrW
-k9/7h9i54t8RS/RN7KNo7ECGnKXqXDPzBBTs1Gwo1WzltAoaDKUfXqQ4oJ4aCP/q
-/XPVWEzgpJO1XEezvCq8VXisutyDiXEjjMIeBczxb1hbamQX+jLTIQ1MDJ4Zo1YP
-zlUqrHW434XC2b1/WbSaylq8Wk9cksca5J+g3FqTlgiWozyy0uxygIRjb6iTzKXk
-V7SYxeXp3hNTuoUgiFkjh5/0yKWCwx7aQqlHar9GjpxmBDAO0kzOlgtTw//EqTwR
-KnYZLig9FW0PhwvZJUigr0cvs/XXTTb77z/i/dfHkrjVTTYenNyXogPtTtSyxqca
-61fbPf0B/S3N43PW8URXBRS0sykpX4SxKu+PwKCqf+OJ7hMEVAapqzTt1q9T7zyB
-QwvCVx8s7WWvXbs2d6ZUrArklgjHoHQcdxJKdhuRmD34AuXWCLW+gH8rJWZpuNl3
-+WsPZX4PvjKDgMw6YMcV7zhWX6c0SevKtzt7WP3XoKDuPhK1PMGJQqQ7spegGB+5
-DZvsJS48Ip0S45Qfmj82ibXaCBJHTNZE8Zs+rdTjQ9DS5qvzRA1sRA1dBb/7OLYE
-JmeWf4VZyebm+gc50szsg6Ut2yT8hw==
-=AiP8
+xsFNBGJ5RksBEADF/Lzssm7uryV6+VHAgL36klyCVcHwvx9Bk853LBOuHVEZWsme
+kbJF3fQG7i7gfCKGuV5XW15xINToe4fBThZteGJziboSZRpkEQ2z3lYcbg34X7+d
+co833lkBNgz1v6QO7PmAdY/x76Q6Hx0J9yiJWd+4j+vRi4hbWuh64vUtTd7rPwk8
+0y3g4oK1YT0NR0Xm/QUO9vWmkSTVflQ6y82HhHIUrG+1vQnSOrWaC0O1lqUI3Nuo
+b6jTARCmbaPsi+XVQnBbsnPPq6Tblwc+NYJSqj5d9nT0uEXT7Zovj4Je5oWVFXp9
+P1OWkbo2z5XkKjoeobM/zKDESJR78h+YQAN9IOKFjL/u/Gzrk1oEgByCABXOX+H5
+hfucrq5U3bbcKy4e5tYgnnZxqpELv3fN/2l8iZknHEh5aYNT5WXVHpD/8u2rMmwm
+I9YTEMueEtmVy0ZV3opUzOlC+3ZUwjmvAJtdfJyeVW/VMy3Hw3Ih0Fij91rO613V
+7n72ggVlJiX25jYyT4AXlaGfAOMndJNVgBps0RArOBYsJRPnvfHlLi5cfjVd7vYx
+QhGX9ODYuvyJ/rW70dMVikeSjlBDKS08tvdqOgtiYy4yhtY4ijQC9BmCE9H9gOxU
+FN297iLimAxr0EVsED96fP96TbDGILWsfJuxAvoqmpkElv8J+P1/F7to2QARAQAB
+zU9Sb2NreSBFbnRlcnByaXNlIFNvZnR3YXJlIEZvdW5kYXRpb24gLSBSZWxlYXNl
+IGtleSAyMDIyIDxyZWxlbmdAcm9ja3lsaW51eC5vcmc+wsGKBBMBCAA0BQJieUZL
+FiEEIcslauFvxUxuZSlJcC1CbTUNJ10CGwMCHgECGQEDCwkHAhUIAxYAAgIiAQAK
+CRBwLUJtNQ0nXWQ5D/9472seOyRO6//bQ2ns3w9lE+aTLlJ5CY0GSTb4xNuyv+AD
+IXpgvLSMtTR0fp9GV3vMw6QIWsehDqt7O5xKWi+3tYdaXRpb1cvnh8r/oCcvI4uL
+k8kImNgsx+Cj+drKeQo03vFxBTDi1BTQFkfEt32fA2Aw5gYcGElM717sNMAMQFEH
+P+OW5hYDH4kcLbtUypPXFbcXUbaf6jUjfiEp5lLjqquzAyDPLlkzMr5RVa9n3/rI
+R6OQp5loPVzCRZMgDLALBU2TcFXLVP+6hAW8qM77c+q/rOysP+Yd+N7GAd0fvEvA
+mfeA4Y6dP0mMRu96EEAJ1qSKFWUul6K6nuqy+JTxktpw8F/IBAz44na17Tf02MJH
+GCUWyM0n5vuO5kK+Ykkkwd+v43ZlqDnwG7akDkLwgj6O0QNx2TGkdgt3+C6aHN5S
+MiF0pi0qYbiN9LO0e05Ai2r3zTFC/pCaBWlG1ph2jx1pDy4yUVPfswWFNfe5I+4i
+CMHPRFsZNYxQnIA2Prtgt2YMwz3VIGI6DT/Z56Joqw4eOfaJTTQSXCANts/gD7qW
+D3SZXPc7wQD63TpDEjJdqhmepaTECbxN7x/p+GwIZYWJN+AYhvrfGXfjud3eDu8/
+i+YIbPKH1TAOMwiyxC106mIL705p+ORf5zATZMyB8Y0OvRIz5aKkBDFZM2QN6A==
+=PzIf
 -----END PGP PUBLIC KEY BLOCK-----
--- a/setup/so-functions
+++ b/setup/so-functions
@@ -61,7 +61,7 @@ add_mngr_ip_to_hosts() {

 add_socore_user_manager() {
 	info "Adding socore user"
-	logCmd "so_add_user 'socore' '939' '939' '/opt/so'" 
+	logCmd "so_add_user socore 939 939 /opt/so" 
 }

 add_web_user() {
@@ -967,15 +967,15 @@ detect_os() {
 }

 download_elastic_agent_artifacts() {
-   #TODO - ISO

-   logCmd "mkdir -p /nsm/elastic-fleet/artifacts/beats/elastic-agent/"
-
-   logCmd "curl --retry 5 --retry-delay 60 https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$SOVERSION.tar.gz --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz"
-
-   logCmd "tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/"
-
-	}
+	if [[ $is_iso ]]; then
+		logCmd "tar -xf /nsm/elastic-fleet/artifacts/beats/elastic-agent_SO-$SOVERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/"
+	else
+		logCmd "mkdir -p /nsm/elastic-fleet/artifacts/beats/elastic-agent/"
+		logCmd "curl --retry 5 --retry-delay 60 https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$SOVERSION.tar.gz --output /nsm/elastic-fleet/artifacts/elastic-agent_SO-$SOVERSION.tar.gz"
+		logCmd "tar -xf /nsm/elastic-fleet/artifacts/beats/elastic-agent_SO-$SOVERSION.tar.gz -C /nsm/elastic-fleet/artifacts/beats/elastic-agent/"
+	fi
+}

 installer_progress_loop() {
 	local i=0
@@ -1946,9 +1946,11 @@ securityonion_repo() {
 			logCmd "dnf repolist all"
 		fi
 		if [[ $waitforstate ]]; then
-			# Build the repo locally so we can use it
-			echo "Syncing Repo"
-			repo_sync_local
+			if [[ ! $is_airgap ]]; then
+				# Build the repo locally so we can use it
+				echo "Syncing Repo"
+				repo_sync_local
+			fi
 		fi
 	fi
 }
@@ -2194,12 +2196,12 @@ setup_salt_master_dirs() {
 		logCmd "rsync -avh --exclude 'TRANS.TBL' /home/$INSTALLUSERNAME/SecurityOnion/pillar/* $default_salt_dir/pillar/"
 		logCmd "rsync -avh --exclude 'TRANS.TBL' /home/$INSTALLUSERNAME/SecurityOnion/salt/* $default_salt_dir/salt/"
 		logCmd "mkdir -p $local_salt_dir/salt/zeek/policy/intel"
-		logCmd "cp -Rv /home/$INSTALLUSERNAME/SecurityOnion/files/intel.dat $local_salt_dir/salt/zeek/policy/intel/"
+		logCmd "touch $local_salt_dir/salt/zeek/policy/intel/intel.dat"
 	else
 		logCmd "cp -Rv ../pillar/* $default_salt_dir/pillar/"
 		logCmd "cp -Rv ../salt/* $default_salt_dir/salt/"
 		logCmd "mkdir -p $local_salt_dir/salt/zeek/policy/intel"
-		logCmd "cp -Rv files/intel.dat $local_salt_dir/salt/zeek/policy/intel/"
+		logCmd "touch $local_salt_dir/salt/zeek/policy/intel/intel.dat"
 	fi

 	info "Chown the salt dirs on the manager for socore"
@@ -2329,8 +2331,8 @@ so_add_user() {
 	if [ "$5" ]; then local pass=$5; fi

 	info "Add $username user"
-	logCmd "groupadd --gid '$gid' '$username'"
-	logCmd "useradd -m --uid '$uid' --gid '$gid' --home-dir '$home_dir' '$username'"
+	logCmd "groupadd --gid $gid $username"
+	logCmd "useradd -m --uid $uid --gid $gid --home-dir $home_dir $username"

 	# If a password has been passed in, set the password
 	if [ "$pass" ]; then
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -335,45 +335,53 @@ process_installtype

 # If this is not an automated install prompt
 if ! [[ -f $install_opt_file ]]; then
-
-	# If you are a manager ask ALL the manager things here. I know there is code re-use but this makes it easier to add new roles.
+	# If you are a manager ask ALL the manager things here. I know there is code re-use but this makes it easier to add new roles
 	if [[ $is_eval ]]; then
+		# waitforstate means we will run the full salt state at the end. This is for only nodes running the salt-master service
 		waitforstate=true
-		#ubuntu_check
+		# Does this role have monitoring interfaces?
 		monints=true
+		# Prompt the user to accept the elastic license
 		check_elastic_license
+		# If it is an install from ISO is this airgap?
+		[[ $is_iso ]] && whiptail_airgap
+		# Make sure minimum requirements are met
 		check_requirements "manager"
+		# Do networking things
 		networking_needful
-		collect_net_method
+		# Do we need a proxy?
+		[[ ! $is_airgap ]] && collect_net_method
+		# Do we need to change the dockernet subnet?
 		collect_dockernet
-		if [[ $is_iso ]]; then
-			whiptail_airgap
-		fi
-		detect_cloud
+		# Are we in the clouds?
+		[[ ! $is_airgap ]] && detect_cloud
+		# Sets some minion info
 		set_minion_info
 		set_default_log_size >> $setup_log 2>&1
 		info "Verifying all network devices are managed by Network Manager that should be"
 		check_network_manager_conf
 		set_network_dev_status_list
+		# What NIC for watching network traffic?
 		whiptail_sensor_nics
+		# How many cores do we have?
 		calculate_useable_cores
+		# What is the web user?
 		collect_webuser_inputs
+		# How are we accessing the UI?
 		get_redirect
+		# Does the user want to allow access to the UI?
 		collect_so_allow
 		whiptail_end_settings
 	elif [[ $is_standalone ]]; then
 		waitforstate=true
-		#ubuntu_check
 		monints=true
 		check_elastic_license
+		[[ $is_iso ]] && whiptail_airgap
 		check_requirements "manager"
 		networking_needful
-		collect_net_method
+		[[ ! $is_airgap ]] && collect_net_method
 		collect_dockernet
-		if [[ $is_iso ]]; then
-			whiptail_airgap
-		fi
-		detect_cloud
+		[[ ! $is_airgap ]] && detect_cloud
 		set_minion_info
 		set_default_log_size >> $setup_log 2>&1
 		info "Verifying all network devices are managed by Network Manager that should be"
@@ -389,14 +397,12 @@ if ! [[ -f $install_opt_file ]]; then
 		check_elastic_license
 		waitforstate=true
 		#ubuntu_check
+		[[ $is_iso ]] && whiptail_airgap
 		check_requirements "manager"
 		networking_needful
-		collect_net_method
+		[[ ! $is_airgap ]] && collect_net_method
 		collect_dockernet
-		if [[ $is_iso ]]; then
-			whiptail_airgap
-		fi
-		detect_cloud
+		[[ ! $is_airgap ]] && detect_cloud
 		set_minion_info
 		set_default_log_size >> $setup_log 2>&1
 		info "Verifying all network devices are managed by Network Manager that should be"
@@ -410,15 +416,12 @@ if ! [[ -f $install_opt_file ]]; then
 	elif [[ $is_managersearch ]]; then
 		check_elastic_license
 		waitforstate=true
-		#ubuntu_check
+		[[ $is_iso ]] && whiptail_airgap
 		check_requirements "manager"
 		networking_needful
-		collect_net_method
+		[[ ! $is_airgap ]] && collect_net_method
 		collect_dockernet
-		if [[ $is_iso ]]; then
-			whiptail_airgap
-		fi
-		detect_cloud
+		[[ ! $is_airgap ]] && detect_cloud
 		set_minion_info
 		set_default_log_size >> $setup_log 2>&1
 		info "Verifying all network devices are managed by Network Manager that should be"
@@ -430,7 +433,6 @@ if ! [[ -f $install_opt_file ]]; then
 		collect_so_allow
 		whiptail_end_settings
 	elif [[ $is_sensor ]]; then
-		#ubuntu_check
 		installer_prereq_packages
 		monints=true
 		check_requirements "sensor"
@@ -459,7 +461,6 @@ if ! [[ -f $install_opt_file ]]; then
 		whiptail_end_settings

 	elif [[ $is_searchnode ]]; then
-		#ubuntu_check
 		installer_prereq_packages
 		check_requirements "elasticsearch"
 		networking_needful
@@ -473,7 +474,6 @@ if ! [[ -f $install_opt_file ]]; then
 		whiptail_end_settings

 	elif [[ $is_heavynode ]]; then
-		#ubuntu_check
 		installer_prereq_packages
 		monints=true
 		check_requirements "heavynode"
@@ -486,29 +486,26 @@ if ! [[ -f $install_opt_file ]]; then
 		whiptail_end_settings

 	elif [[ $is_idh ]]; then
-		#ubuntu_check
 		installer_prereq_packages
 		check_requirements "idh"
 		networking_needful
 		collect_mngr_hostname
 		add_mngr_ip_to_hosts
 		check_manager_connection
-		#collect_idh_services (this may be added back sometime in the future)
 		collect_idh_preferences
 		set_minion_info
 		whiptail_end_settings

 	elif [[ $is_import ]]; then
-		#ubuntu_check
 		waitforstate=true
 		monints=true
+		[[ $is_iso ]] && whiptail_airgap
 		check_elastic_license
 		check_requirements "import"
 		networking_needful
-		if [[ $is_iso ]]; then
-			whiptail_airgap
-		fi
-		detect_cloud
+		[[ ! $is_airgap ]] && detect_cloud
+		collect_dockernet
+		[[ ! $is_airgap ]] && collect_net_method
 		set_minion_info
 		set_default_log_size >> $setup_log 2>&1
 		info "Verifying all network devices are managed by Network Manager that should be"
@@ -521,7 +518,6 @@ if ! [[ -f $install_opt_file ]]; then
 		whiptail_end_settings

 	elif [[ $is_receiver ]]; then
-		#ubuntu_check
 		installer_prereq_packages
 		check_requirements "receiver"
 		networking_needful
--- a/setup/so-verify
+++ b/setup/so-verify
@@ -31,7 +31,7 @@ log_has_errors() {

    # Ignore Failed: 0 since that is the salt state output, and we detect state failures
    # via Result: False already.
-
+    
    grep -E "FAILED|Failed|failed|ERROR|Result: False" "$setup_log" | \
        grep -vE "The Salt Master has cached the public key for this node" | \
        grep -vE "Minion failed to authenticate with the master" | \