Merge pull request #16006 from Security-Onion-Solutions/reyesj2-patch-5

remove heayvnode FleetServer_* directory creation, and skip empty dir…

salt/elasticfleet/config.sls

@@ -173,7 +173,7 @@ eaoptionalintegrationsdir:
 
 {% for minion in node_data %}
 {% set role = node_data[minion]["role"] %}
-{% if role in [ "eval","fleet","heavynode","import","manager", "managerhype", "managersearch","standalone" ] %}
+{% if role in [ "eval","fleet","import","manager", "managerhype", "managersearch","standalone" ] %}
 {% set optional_integrations = ELASTICFLEETMERGED.optional_integrations %}
 {% set integration_keys = optional_integrations.keys() %}
 fleet_server_integrations_{{ minion }}:

salt/elasticfleet/manager.sls

@@ -67,8 +67,6 @@ so-elastic-fleet-package-upgrade:
         interval: 30
     - require:
       - http: wait_for_so-kibana
-    - onchanges:
-      - file: /opt/so/state/elastic_fleet_packages.txt
 
 so-elastic-fleet-integrations:
   cmd.run:

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -9,13 +9,11 @@
 RETURN_CODE=0
 
 if [ ! -f /opt/so/state/eaintegrations.txt ]; then
-  # First, check for any package upgrades
-  /usr/sbin/so-elastic-fleet-package-upgrade
 
-  # Second, update Fleet Server policies
+  # update Fleet Server policies
   /usr/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server
 
-  # Third, configure Elastic Defend Integration seperately
+  # configure Elastic Defend Integration separately
   /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend
 
   # Each group fetches its agent policy once and dispatches create/update writes concurrently.
@@ -32,9 +30,12 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
   elastic_fleet_load_integrations_dir "so-grid-nodes_heavy" \
     /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy "Grid Nodes Policy_Heavy" || RETURN_CODE=1
 
-  # Fleet Server - Optional integrations (one agent policy per FleetServer_* directory)
+  # Fleet Server - Optional integrations (adds integration configuration to a given FleetServer_ policy)
   for FLEET_DIR in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/; do
     [ -d "$FLEET_DIR" ] || continue
+    INTEGRATIONS=("${FLEET_DIR%/}"/*.json)
+    [ -e "${INTEGRATIONS[0]}" ] || continue
+
     FLEET_POLICY=$(basename "$FLEET_DIR")
     elastic_fleet_load_integrations_dir "$FLEET_POLICY" \
       "${FLEET_DIR%/}" "Fleet Server Policy" "elasticsearch-logs" || RETURN_CODE=1

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-upgrade

@@ -12,17 +12,22 @@ PKG_LOAD_FAILURES=0
 PKG_LOAD_FAILURES_NAMES=()
 
 {%- for PACKAGE in SUPPORTED_PACKAGES %}
-echo "Upgrading {{ PACKAGE }} package..."
+if INSTALLED_VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}") && LATEST_VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
-if VERSION=$(elastic_fleet_package_latest_version_check "{{ PACKAGE }}"); then
+
-    if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
+    if [ "$INSTALLED_VERSION" == "$LATEST_VERSION" ]; then
-        PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
+        echo "{{ PACKAGE }} integration version $INSTALLED_VERSION is already at the reported latest version $LATEST_VERSION, skipping upgrade."
-        PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
+    else
+        echo "Upgrading {{ PACKAGE }} package to version $LATEST_VERSION..."
+        if ! elastic_fleet_package_install "{{ PACKAGE }}" "$LATEST_VERSION"; then
+            PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
+            PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
+        fi
     fi
 else
+    echo "ERROR: Failed to get version information for integration {{ PACKAGE }}"
     PKG_LOAD_FAILURES=$((PKG_LOAD_FAILURES + 1))
     PKG_LOAD_FAILURES_NAMES+=("{{ PACKAGE }}")
 fi
-echo
 {%- endfor %}
 
 if [ $PKG_LOAD_FAILURES -gt 0 ]; then
@@ -35,6 +40,3 @@ if [ $PKG_LOAD_FAILURES -gt 0 ]; then
 else
     echo "Successfully upgraded all packages."
 fi
-
-echo
-/usr/sbin/so-elasticsearch-templates-load

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -181,6 +181,9 @@ if ! elastic_fleet_policy_create "so-grid-nodes_heavy" "SO Grid Nodes - Heavy No
     exit 1
 fi
 
+# Check for package upgrades
+so-elastic-fleet-package-upgrade
+
 # Load Integrations for default policies
 so-elastic-fleet-integration-policy-load
 

salt/manager/files/mirror-kernel.txt

@@ -1,2 +0,0 @@
-https://repo.securityonion.net/file/so-repo/prod/3/oracle/9-uek8
-https://repo-alt.securityonion.net/prod/3/oracle/9-uek8

salt/manager/files/repodownload.conf

@@ -10,9 +10,4 @@ keepcache=0
 name=Security Onion Repo repo
 mirrorlist=file:///opt/so/conf/reposync/mirror.txt
 enabled=1
-gpgcheck=1
+gpgcheck=1
-[securityonionkernel]
-name=Security Onion Repo repo
-mirrorlist=file:///opt/so/conf/reposync/mirror-kernel.txt
-enabled=1
-gpgcheck=1

salt/manager/init.sls

@@ -86,28 +86,6 @@ repo_dir:
       - group
     - show_changes: False
 
-kernelrepo_dir:
-  file.directory:
-    - name: /nsm/kernelrepo
-    - user: socore
-    - group: socore
-    - recurse:
-      - user
-      - group
-    - show_changes: False
-
-# Ensure /nsm/kernelrepo is always a valid (if empty) repo before it is ever assigned to
-# a client. Without repodata/repomd.xml an enabled file:///nsm/kernelrepo repo makes every
-# dnf operation fail; so-repo-sync only populates it after the highstate, so seed an empty
-# repo here. Only runs when repodata is missing, so it won't clobber a synced repo.
-kernelrepo_init_empty:
-  cmd.run:
-    - name: createrepo /nsm/kernelrepo
-    - unless: 'test -e /nsm/kernelrepo/repodata/repomd.xml'
-    - require:
-      - file: kernelrepo_dir
-      - pkg: install_createrepo
-
 manager_sbin:
   file.recurse:
     - name: /usr/sbin
@@ -144,13 +122,6 @@ so-repo-mirrorlist:
     - user: socore
     - group: socore
 
-so-repo-kernel-mirrorlist:
-  file.managed:
-    - name: /opt/so/conf/reposync/mirror-kernel.txt
-    - source: salt://manager/files/mirror-kernel.txt
-    - user: socore
-    - group: socore
-
 so-repo-sync:
   {%     if MANAGERMERGED.reposync.enabled %}
   cron.present:

salt/manager/tools/sbin/so-repo-sync

@@ -10,16 +10,5 @@ NOROOT=1
 set -e
 
 curl --retry 5 --retry-delay 60 -A "reposync/$(sync_options)" https://sigs.securityonion.net/checkup --output /tmp/checkup
-
 dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/
 createrepo /nsm/repo
-
-# The kernel repo section is deployed to repodownload.conf by the manager highstate, which
-# runs AFTER this script during soup. On the first upgrade to a kernel-aware version the
-# on-disk config still predates the section, so guard on its presence to avoid dnf's
-# "Unknown repo: 'securityonionkernel'" aborting the sync (set -e). The next sync after the
-# highstate deploys the section will pick it up.
-if grep -q '^\[securityonionkernel\]' /opt/so/conf/reposync/repodownload.conf; then
-  dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/
-  createrepo /nsm/kernelrepo
-fi

salt/nginx/enabled.sls

@@ -59,7 +59,6 @@ so-nginx:
       - /opt/so/conf/navigator/layers/:/opt/socore/html/navigator/assets/so:ro
       - /opt/so/conf/navigator/config.json:/opt/socore/html/navigator/assets/config.json:ro
       - /nsm/repo:/opt/socore/html/repo:ro
-      - /nsm/kernelrepo:/opt/socore/html/kernelrepo:ro
       - /nsm/rules:/nsm/rules:ro
       {%   if NGINXMERGED.external_suricata %}
       - /opt/so/rules/nids/suri:/surirules:ro

salt/repo/client/oracle.sls

@@ -57,26 +57,6 @@ so_repo:
     - enabled: 1
     - gpgcheck: 1
 
-so_kernel_repo:
-  pkgrepo.managed:
-    - name: securityonionkernel
-    - humanname: Security Onion Kernel Repo
-  {% if GLOBALS.is_manager %}
-    - baseurl: file:///nsm/kernelrepo/
-  {% else %}
-    - baseurl: https://{{ GLOBALS.repo_host }}/kernelrepo
-  {% endif %}
-    - enabled: 1
-    - gpgcheck: 1
-    # Supplementary kernel repo: tolerate it being empty/unreachable (e.g. before the
-    # manager has populated /nsm/kernelrepo) so a missing repomd.xml can't make every
-    # dnf/pkg operation on the grid fail.
-    - skip_if_unavailable: 1
-    # Only assign the kernel repo once physical NIC names are pinned by MAC, so the
-    # UEK8 kernel update can't renumber interfaces SO binds by name (see pin_nic_names
-    # in salt/common/init.sls, which drops this marker via /usr/sbin/so-nic-pin).
-    - onlyif: 'test -e /opt/so/state/nic_names_pinned'
-
 {% endif %}
   
 # TODO: Add a pillar entry for custom repos

setup/so-functions

@@ -886,7 +886,6 @@ create_repo() {
 	title "Create the repo directory"
 	logCmd "dnf -y install yum-utils createrepo_c"
 	logCmd "createrepo /nsm/repo"
-	logCmd "createrepo /nsm/kernelrepo"
 }
 
 
@@ -1813,16 +1812,6 @@ securityonion_repo() {
 			echo "mirrorlist=file:///etc/yum/mirror.txt" >> /etc/yum.repos.d/securityonion.repo
 			echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo
 			echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo
-			echo "https://repo.securityonion.net/file/so-repo/prod/3/oracle/9-uek8" > /etc/yum/mirror-kernel.txt
-			echo "https://so-repo-east.s3.us-east-005.backblazeb2.com/prod/3/oracle/9-uek8" >> /etc/yum/mirror-kernel.txt
-			echo "[securityonionkernel]" > /etc/yum.repos.d/securityonionkernel.repo
-			echo "name=Security Onion Kernel Repo repo" >> /etc/yum.repos.d/securityonionkernel.repo
-			echo "mirrorlist=file:///etc/yum/mirror-kernel.txt" >> /etc/yum.repos.d/securityonionkernel.repo
-			echo "enabled=1" >> /etc/yum.repos.d/securityonionkernel.repo
-			echo "gpgcheck=1" >> /etc/yum.repos.d/securityonionkernel.repo
-			# Supplementary kernel repo: tolerate it being empty/unreachable so a missing
-			# repomd.xml can't make every dnf operation fail before the repo is populated.
-			echo "skip_if_unavailable=1" >> /etc/yum.repos.d/securityonionkernel.repo
 			logCmd "dnf repolist"
 		else
 			echo "[securityonion]" > /etc/yum.repos.d/securityonion.repo
@@ -1831,13 +1820,6 @@ securityonion_repo() {
 			echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo
 			echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo
 			echo "sslverify=0" >> /etc/yum.repos.d/securityonion.repo
-			echo "[securityonionkernel]" > /etc/yum.repos.d/securityonionkernel.repo
-			echo "name=Security Onion Kernel Repo" >> /etc/yum.repos.d/securityonionkernel.repo
-			echo "baseurl=https://$MSRV/kernelrepo" >> /etc/yum.repos.d/securityonionkernel.repo
-			echo "enabled=1" >> /etc/yum.repos.d/securityonionkernel.repo
-			echo "gpgcheck=1" >> /etc/yum.repos.d/securityonionkernel.repo
-			echo "sslverify=0" >> /etc/yum.repos.d/securityonionkernel.repo
-			echo "skip_if_unavailable=1" >> /etc/yum.repos.d/securityonionkernel.repo
 			logCmd "dnf repolist"
 		fi
 	elif [[ ! $waitforstate ]]; then
@@ -1847,25 +1829,12 @@ securityonion_repo() {
 		echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo
 		echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo
 		echo "sslverify=0" >> /etc/yum.repos.d/securityonion.repo
-		echo "[securityonionkernel]" > /etc/yum.repos.d/securityonionkernel.repo
-		echo "name=Security Onion Kernel Repo" >> /etc/yum.repos.d/securityonionkernel.repo
-		echo "baseurl=https://$MSRV/kernelrepo" >> /etc/yum.repos.d/securityonionkernel.repo
-		echo "enabled=1" >> /etc/yum.repos.d/securityonionkernel.repo
-		echo "gpgcheck=1" >> /etc/yum.repos.d/securityonionkernel.repo
-		echo "sslverify=0" >> /etc/yum.repos.d/securityonionkernel.repo
-		echo "skip_if_unavailable=1" >> /etc/yum.repos.d/securityonionkernel.repo
 	elif [[ $waitforstate ]]; then
 		echo "[securityonion]" > /etc/yum.repos.d/securityonion.repo
 		echo "name=Security Onion Repo" >> /etc/yum.repos.d/securityonion.repo
 		echo "baseurl=file:///nsm/repo/" >> /etc/yum.repos.d/securityonion.repo
 		echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo
 		echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo
-		echo "[securityonionkernel]" > /etc/yum.repos.d/securityonionkernel.repo
-		echo "name=Security Onion Kernel Repo" >> /etc/yum.repos.d/securityonionkernel.repo
-		echo "baseurl=file:///nsm/kernelrepo/" >> /etc/yum.repos.d/securityonionkernel.repo
-		echo "enabled=1" >> /etc/yum.repos.d/securityonionkernel.repo
-		echo "gpgcheck=1" >> /etc/yum.repos.d/securityonionkernel.repo
-		echo "skip_if_unavailable=1" >> /etc/yum.repos.d/securityonionkernel.repo
 	fi
 	logCmd "dnf repolist all"
 	if [[ $waitforstate ]]; then
@@ -1881,12 +1850,9 @@ repo_sync_local() {
 	# Sync the repo from the SO repo locally.
 	info "Adding Repo Download Configuration"
 	mkdir -p /nsm/repo
-	mkdir -p /nsm/kernelrepo
 	mkdir -p /opt/so/conf/reposync/cache
 	echo "https://repo.securityonion.net/file/so-repo/prod/3/oracle/9" > /opt/so/conf/reposync/mirror.txt
 	echo "https://repo-alt.securityonion.net/prod/3/oracle/9" >> /opt/so/conf/reposync/mirror.txt
-	echo "https://repo.securityonion.net/file/so-repo/prod/3/oracle/9-uek8" > /opt/so/conf/reposync/mirror-kernel.txt
-	echo "https://repo-alt.securityonion.net/prod/3/oracle/9-uek8" >> /opt/so/conf/reposync/mirror-kernel.txt
 	echo "[main]" > /opt/so/conf/reposync/repodownload.conf
 	echo "gpgcheck=1" >> /opt/so/conf/reposync/repodownload.conf
 	echo "installonly_limit=3" >> /opt/so/conf/reposync/repodownload.conf
@@ -1900,18 +1866,12 @@ repo_sync_local() {
 	echo "mirrorlist=file:///opt/so/conf/reposync/mirror.txt" >> /opt/so/conf/reposync/repodownload.conf
 	echo "enabled=1" >> /opt/so/conf/reposync/repodownload.conf
 	echo "gpgcheck=1" >> /opt/so/conf/reposync/repodownload.conf
-	echo "[securityonionkernel]" >> /opt/so/conf/reposync/repodownload.conf
-	echo "name=Security Onion Kernel Repo repo" >> /opt/so/conf/reposync/repodownload.conf
-	echo "mirrorlist=file:///opt/so/conf/reposync/mirror-kernel.txt" >> /opt/so/conf/reposync/repodownload.conf
-	echo "enabled=1" >> /opt/so/conf/reposync/repodownload.conf
-	echo "gpgcheck=1" >> /opt/so/conf/reposync/repodownload.conf
 
 	logCmd "dnf repolist"
 
 	if [[ ! $is_airgap ]]; then
 		curl --retry 5 --retry-delay 60 -A "netinstall/$SOVERSION/$OS/$(uname -r)/1" https://sigs.securityonion.net/checkup --output /tmp/install
 		retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" >> "$setup_log" 2>&1 || fail_setup
-		retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/" >> "$setup_log" 2>&1 || fail_setup
 		# After the download is complete run createrepo
 		create_repo
 	fi
@@ -2268,13 +2228,6 @@ update_sudoers_for_testing() {
 }
 
 update_packages() {
-	# Pin physical NIC names by MAC BEFORE pulling packages, so the UEK8 kernel that
-	# the update below installs can't renumber the interfaces SO binds by name. Doing
-	# it here (instead of waiting for the common highstate) also drops the
-	# /opt/so/state/nic_names_pinned marker that gates the kernel repo, so the kernel
-	# repo is assigned on the very first highstate and the kernel isn't downgraded and
-	# then re-upgraded. Run-once: so-nic-pin no-ops if the marker already exists.
-	logCmd "bash ../salt/common/tools/sbin/so-nic-pin"
 	logCmd "dnf repolist"
 	logCmd "dnf -y update --allowerasing --exclude=salt*,docker*,containerd*"
 	RMREPOFILES=("oracle-linux-ol9.repo" "uek-ol9.repo" "virt-ol9.repo")