Serve /kernelrepo through nginx so minions can reach the kernel repo

The /nsm/kernelrepo bind mount exposed the files, but without a matching
location block external requests to /kernelrepo/ fell through to the SOC
app and returned HTML, so minions hit 'repomd.xml parser error'. Add a
/kernelrepo/ location mirroring /repo/.

salt/common/init.sls

@@ -130,6 +130,17 @@ common_sbin:
       - so-pcap-import
 {% endif %}
 
+# Pin physical NIC names by MAC (run-once) so a kernel upgrade can't renumber the
+# interfaces SO binds by name. The marker keeps it a one-time setup; an admin can
+# pre-create the marker to opt out.
+pin_nic_names:
+  cmd.run:
+    - name: /usr/sbin/so-nic-pin
+    - unless: 'test -e /opt/so/state/nic_names_pinned'
+    - require:
+      - file: common_sbin
+      - file: statedir
+
 common_sbin_jinja:
   file.recurse:
     - name: /usr/sbin

salt/common/tools/sbin/so-kernel-upgrade

@@ -0,0 +1,57 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+#
+# so-kernel-upgrade — switch the boot default to the installed UEK8 (6.x) kernel.
+#
+# Security Onion is moving off the EL9 stock kernel / UEK7 (5.x) onto UEK8 (6.x).
+# Installing the kernel-uek-core package adds a UEK8 boot entry but does NOT make it the
+# default: kernel-install/grubby only auto-promote a new kernel within the running
+# kernel's flavor lineage, and we're crossing from a 5.x kernel to the new 6.x UEK flavor.
+# So even with UPDATEDEFAULT=yes and DEFAULTKERNEL=kernel-uek-core the box keeps booting
+# the old kernel. This tool finds the newest installed 6.x UEK kernel and makes it the
+# GRUB default via grubby so the next boot comes up on UEK8.
+#
+# Idempotent: if the UEK8 kernel is already the default it does nothing. It only sets the
+# boot default; it does NOT reboot — the admin reboots the node on their own schedule.
+
+log() { echo "[so-kernel-upgrade] $*"; }
+
+[ "$(id -u)" -eq 0 ] || { log "must run as root"; exit 1; }
+command -v grubby >/dev/null 2>&1 || { log "grubby not found"; exit 1; }
+
+# Newest installed UEK8 (6.x) kernel known to the bootloader. UEK8 vmlinuz paths look like
+# /boot/vmlinuz-6.12.0-203.76.7.5.el9uek.x86_64; the 5.x UEK7 and 5.14 RHCK won't match.
+target="$(grubby --info=ALL 2>/dev/null \
+    | sed -n 's/^kernel="\(.*\)"$/\1/p' \
+    | grep -E '/vmlinuz-6\.[0-9]+.*uek' \
+    | sort -V | tail -1)"
+
+if [ -z "$target" ]; then
+    log "no installed 6.x UEK (UEK8) kernel found — confirm the kernel repo is assigned and"
+    log "'dnf update' has installed kernel-uek-core. Nothing to do."
+    exit 0
+fi
+
+current="$(grubby --default-kernel 2>/dev/null)"
+if [ "$current" = "$target" ]; then
+    log "UEK8 kernel is already the boot default: $target"
+    exit 0
+fi
+
+log "current default kernel: ${current:-unknown}"
+log "switching boot default to UEK8 kernel: $target"
+grubby --set-default="$target" || { log "ERROR: grubby --set-default failed for $target"; exit 1; }
+
+# Verify the change actually took before claiming success.
+now="$(grubby --default-kernel 2>/dev/null)"
+if [ "$now" != "$target" ]; then
+    log "ERROR: default kernel is still '${now:-unknown}' after set-default"
+    exit 1
+fi
+
+log "boot default is now $target"
+log "REBOOT REQUIRED to start using the UEK8 kernel (currently running $(uname -r))."

salt/common/tools/sbin/so-nic-pin

@@ -0,0 +1,76 @@
+#!/bin/bash
+#
+# so-nic-pin — pin physical NIC names by permanent MAC via classic by-MAC udev
+#              rules, so a kernel upgrade can't renumber them.
+#
+# Security Onion binds its management and monitor interfaces BY NAME in pillar
+# (host:mainint, sensor:mainint, and bond0 is built on a specific physical NIC).
+# A kernel upgrade can change the kernel/systemd-udevd predictable-naming output
+# and renumber those NICs (e.g. enp1s0 -> enp2s0), which breaks the grid: the
+# pillar references a name that no longer exists and bond/bridge bring-up fails.
+#
+# This writes /etc/udev/rules.d/70-persistent-net.rules pinning each PHYSICAL NIC
+# to its CURRENT name by its PERMANENT MAC, freezing the names across future kernel
+# changes. It only writes the rules file; it does NOT live-trigger a rename (the
+# rules apply on the next boot/kernel, and a live rename would be disruptive).
+#
+# Run-once: gated by the drop file /opt/so/state/nic_names_pinned. If the marker is
+# present the script does nothing, so an admin can pre-create it to opt out. Invoked
+# from the common state on every highstate; the marker keeps it a one-time setup.
+
+NET_RULES_FILE="/etc/udev/rules.d/70-persistent-net.rules"
+MARKER="/opt/so/state/nic_names_pinned"
+
+log() { echo -e "[so-nic-pin] $*"; }
+
+# Echo "<name> <permanent-mac>" for every PHYSICAL NIC. A physical NIC is backed by a
+# real device (has device/driver), which excludes bond0/sobridge/docker0/veth*/lo whose
+# MACs are dynamic and must never be pinned. The PERMANENT MAC is used (ethtool -P, with
+# fallbacks), not the current one: an enslaved bond member's current MAC is rewritten to
+# the bond's, so matching on it would be wrong/ambiguous.
+physical_nics() {
+    local path n mac
+    for path in /sys/class/net/*; do
+        n="${path##*/}"
+        [ "$n" = "lo" ] && continue
+        [ -e "${path}/device/driver" ] || continue          # real device only
+        mac="$(ethtool -P "$n" 2>/dev/null | awk '/Permanent address/{print $NF}')"
+        case "$mac" in ""|00:00:00:00:00:00) mac="$(cat "${path}/bonding_slave/perm_hwaddr" 2>/dev/null)" ;; esac
+        case "$mac" in ""|00:00:00:00:00:00) mac="$(cat "${path}/address" 2>/dev/null)" ;; esac
+        case "$mac" in ""|00:00:00:00:00:00) continue ;; esac
+        echo "$n $mac"
+    done
+}
+
+# Turn "<name> <mac>" lines on stdin into classic by-MAC persistent-net udev rules.
+render_net_rules() {
+    echo "# Generated by so-nic-pin: pin NIC names by MAC so kernel upgrades can't renumber them."
+    echo "# Security Onion binds its management/monitor interfaces by name; do not hand-edit."
+    local n mac
+    while read -r n mac; do
+        [ -n "$n" ] || continue
+        printf 'SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="%s", NAME="%s"\n' \
+            "$mac" "$n"
+    done
+}
+
+[ "$(id -u)" -eq 0 ] || exit 0                   # salt runs us as root; bail quietly otherwise
+[ -e "${MARKER}" ] && exit 0                      # run-once guard (mirrors the state's unless)
+
+nics="$(physical_nics)"
+if [ -z "${nics}" ]; then
+    log "no physical NICs detected — nothing to pin (will retry on next highstate)"
+    exit 0                                         # do NOT drop the marker; let it retry later
+fi
+
+log "pinning physical NICs by permanent MAC:"
+echo "${nics}" | sed 's/^/    /'
+
+[ -f "${NET_RULES_FILE}" ] && cp -f "${NET_RULES_FILE}" "${NET_RULES_FILE}.bak"
+echo "${nics}" | render_net_rules > "${NET_RULES_FILE}" || {
+    log "ERROR: failed to write ${NET_RULES_FILE}"
+    exit 1
+}
+
+mkdir -p "$(dirname "${MARKER}")" && touch "${MARKER}"
+log "wrote ${NET_RULES_FILE} ($(grep -c '^SUBSYSTEM' "${NET_RULES_FILE}") NIC(s) pinned); dropped ${MARKER}"

salt/manager/files/mirror-kernel.txt

@@ -0,0 +1,2 @@
+https://repo.securityonion.net/file/so-repo/prod/3/oracle/9-uek8
+https://repo-alt.securityonion.net/prod/3/oracle/9-uek8

salt/manager/files/repodownload.conf

@@ -10,4 +10,9 @@ keepcache=0
 name=Security Onion Repo repo
 mirrorlist=file:///opt/so/conf/reposync/mirror.txt
 enabled=1
-gpgcheck=1
+gpgcheck=1
+[securityonionkernel]
+name=Security Onion Repo repo
+mirrorlist=file:///opt/so/conf/reposync/mirror-kernel.txt
+enabled=1
+gpgcheck=1

salt/manager/init.sls

@@ -86,6 +86,28 @@ repo_dir:
       - group
     - show_changes: False
 
+kernelrepo_dir:
+  file.directory:
+    - name: /nsm/kernelrepo
+    - user: socore
+    - group: socore
+    - recurse:
+      - user
+      - group
+    - show_changes: False
+
+# Ensure /nsm/kernelrepo is always a valid (if empty) repo before it is ever assigned to
+# a client. Without repodata/repomd.xml an enabled file:///nsm/kernelrepo repo makes every
+# dnf operation fail; so-repo-sync only populates it after the highstate, so seed an empty
+# repo here. Only runs when repodata is missing, so it won't clobber a synced repo.
+kernelrepo_init_empty:
+  cmd.run:
+    - name: createrepo /nsm/kernelrepo
+    - unless: 'test -e /nsm/kernelrepo/repodata/repomd.xml'
+    - require:
+      - file: kernelrepo_dir
+      - pkg: install_createrepo
+
 manager_sbin:
   file.recurse:
     - name: /usr/sbin
@@ -122,6 +144,13 @@ so-repo-mirrorlist:
     - user: socore
     - group: socore
 
+so-repo-kernel-mirrorlist:
+  file.managed:
+    - name: /opt/so/conf/reposync/mirror-kernel.txt
+    - source: salt://manager/files/mirror-kernel.txt
+    - user: socore
+    - group: socore
+
 so-repo-sync:
   {%     if MANAGERMERGED.reposync.enabled %}
   cron.present:

salt/manager/tools/sbin/so-repo-sync

@@ -10,5 +10,16 @@ NOROOT=1
 set -e
 
 curl --retry 5 --retry-delay 60 -A "reposync/$(sync_options)" https://sigs.securityonion.net/checkup --output /tmp/checkup
+
 dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/
 createrepo /nsm/repo
+
+# The kernel repo section is deployed to repodownload.conf by the manager highstate, which
+# runs AFTER this script during soup. On the first upgrade to a kernel-aware version the
+# on-disk config still predates the section, so guard on its presence to avoid dnf's
+# "Unknown repo: 'securityonionkernel'" aborting the sync (set -e). The next sync after the
+# highstate deploys the section will pick it up.
+if grep -q '^\[securityonionkernel\]' /opt/so/conf/reposync/repodownload.conf; then
+  dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/
+  createrepo /nsm/kernelrepo
+fi

salt/manager/tools/sbin/soup

@@ -131,6 +131,8 @@ check_err() {
 # Collect bash error context before passing off to check_err()
 on_err() {
   local exit_code=$?
+  # Ignore failures in blocks that explicitly disabled errexit with `set +e`.
+  [[ $- == *e* ]] || return $exit_code
   # turn off xtrace to prevent added noise in debug log
   set +x 2>/dev/null || true
 

salt/nginx/enabled.sls

@@ -59,6 +59,7 @@ so-nginx:
       - /opt/so/conf/navigator/layers/:/opt/socore/html/navigator/assets/so:ro
       - /opt/so/conf/navigator/config.json:/opt/socore/html/navigator/assets/config.json:ro
       - /nsm/repo:/opt/socore/html/repo:ro
+      - /nsm/kernelrepo:/opt/socore/html/kernelrepo:ro
       - /nsm/rules:/nsm/rules:ro
       {%   if NGINXMERGED.external_suricata %}
       - /opt/so/rules/nids/suri:/surirules:ro

salt/nginx/etc/nginx.conf

@@ -323,6 +323,16 @@ http {
 			autoindex_localtime on;
 		}
 
+		location /kernelrepo/ {
+			allow all;
+			sendfile on;
+			sendfile_max_chunk 1m;
+			autoindex on;
+			autoindex_exact_size off;
+			autoindex_format html;
+			autoindex_localtime on;
+		}
+
 		location /influxdb/ {
 			auth_request          /auth/sessions/whoami;
 			rewrite               /influxdb/api/(.*) /api/$1 break;

salt/registry/enabled.sls

@@ -16,7 +16,7 @@ include:
 # Install the registry container
 so-dockerregistry:
   docker_container.running:
-    - image: ghcr.io/security-onion-solutions/registry:3.0.0
+    - image: ghcr.io/security-onion-solutions/registry:3.1.1
     - hostname: so-registry
     - networks:
       - sobridge:

salt/repo/client/oracle.sls

@@ -6,6 +6,10 @@
 {% from 'repo/client/map.jinja' import REPOPATH with context %}
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
+{% import_yaml 'salt/minion.defaults.yaml' as saltversion %}
+{% set saltversion = saltversion.salt.minion.version %}
+{% set INSTALLEDSALTVERSION = grains.saltversion %}
+
 {% set role = grains.id.split('_') | last %}
 {% set MANAGER = salt['grains.get']('master') %}
 {% if grains['os'] == 'OEL' %}
@@ -57,6 +61,32 @@ so_repo:
     - enabled: 1
     - gpgcheck: 1
 
+# Only assign the kernel repo once this node's running salt matches the version this
+# SO release ships. During a soup the grid is mid-salt-upgrade; gating here keeps the
+# UEK8 kernel repo (and the kernel update it enables) from activating until the node is
+# fully on the target salt, the same way other states defer across the upgrade window.
+{% if saltversion | string == INSTALLEDSALTVERSION | string %}
+so_kernel_repo:
+  pkgrepo.managed:
+    - name: securityonionkernel
+    - humanname: Security Onion Kernel Repo
+  {% if GLOBALS.is_manager %}
+    - baseurl: file:///nsm/kernelrepo/
+  {% else %}
+    - baseurl: https://{{ GLOBALS.repo_host }}/kernelrepo
+  {% endif %}
+    - enabled: 1
+    - gpgcheck: 1
+    # Supplementary kernel repo: tolerate it being empty/unreachable (e.g. before the
+    # manager has populated /nsm/kernelrepo) so a missing repomd.xml can't make every
+    # dnf/pkg operation on the grid fail.
+    - skip_if_unavailable: 1
+    # Only assign the kernel repo once physical NIC names are pinned by MAC, so the
+    # UEK8 kernel update can't renumber interfaces SO binds by name (see pin_nic_names
+    # in salt/common/init.sls, which drops this marker via /usr/sbin/so-nic-pin).
+    - onlyif: 'test -e /opt/so/state/nic_names_pinned'
+{% endif %}
+
 {% endif %}
   
 # TODO: Add a pillar entry for custom repos

setup/so-functions

@@ -886,6 +886,7 @@ create_repo() {
 	title "Create the repo directory"
 	logCmd "dnf -y install yum-utils createrepo_c"
 	logCmd "createrepo /nsm/repo"
+	logCmd "createrepo /nsm/kernelrepo"
 }
 
 
@@ -1812,6 +1813,16 @@ securityonion_repo() {
 			echo "mirrorlist=file:///etc/yum/mirror.txt" >> /etc/yum.repos.d/securityonion.repo
 			echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo
 			echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo
+			echo "https://repo.securityonion.net/file/so-repo/prod/3/oracle/9-uek8" > /etc/yum/mirror-kernel.txt
+			echo "https://so-repo-east.s3.us-east-005.backblazeb2.com/prod/3/oracle/9-uek8" >> /etc/yum/mirror-kernel.txt
+			echo "[securityonionkernel]" > /etc/yum.repos.d/securityonionkernel.repo
+			echo "name=Security Onion Kernel Repo repo" >> /etc/yum.repos.d/securityonionkernel.repo
+			echo "mirrorlist=file:///etc/yum/mirror-kernel.txt" >> /etc/yum.repos.d/securityonionkernel.repo
+			echo "enabled=1" >> /etc/yum.repos.d/securityonionkernel.repo
+			echo "gpgcheck=1" >> /etc/yum.repos.d/securityonionkernel.repo
+			# Supplementary kernel repo: tolerate it being empty/unreachable so a missing
+			# repomd.xml can't make every dnf operation fail before the repo is populated.
+			echo "skip_if_unavailable=1" >> /etc/yum.repos.d/securityonionkernel.repo
 			logCmd "dnf repolist"
 		else
 			echo "[securityonion]" > /etc/yum.repos.d/securityonion.repo
@@ -1820,6 +1831,13 @@ securityonion_repo() {
 			echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo
 			echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo
 			echo "sslverify=0" >> /etc/yum.repos.d/securityonion.repo
+			echo "[securityonionkernel]" > /etc/yum.repos.d/securityonionkernel.repo
+			echo "name=Security Onion Kernel Repo" >> /etc/yum.repos.d/securityonionkernel.repo
+			echo "baseurl=https://$MSRV/kernelrepo" >> /etc/yum.repos.d/securityonionkernel.repo
+			echo "enabled=1" >> /etc/yum.repos.d/securityonionkernel.repo
+			echo "gpgcheck=1" >> /etc/yum.repos.d/securityonionkernel.repo
+			echo "sslverify=0" >> /etc/yum.repos.d/securityonionkernel.repo
+			echo "skip_if_unavailable=1" >> /etc/yum.repos.d/securityonionkernel.repo
 			logCmd "dnf repolist"
 		fi
 	elif [[ ! $waitforstate ]]; then
@@ -1829,12 +1847,25 @@ securityonion_repo() {
 		echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo
 		echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo
 		echo "sslverify=0" >> /etc/yum.repos.d/securityonion.repo
+		echo "[securityonionkernel]" > /etc/yum.repos.d/securityonionkernel.repo
+		echo "name=Security Onion Kernel Repo" >> /etc/yum.repos.d/securityonionkernel.repo
+		echo "baseurl=https://$MSRV/kernelrepo" >> /etc/yum.repos.d/securityonionkernel.repo
+		echo "enabled=1" >> /etc/yum.repos.d/securityonionkernel.repo
+		echo "gpgcheck=1" >> /etc/yum.repos.d/securityonionkernel.repo
+		echo "sslverify=0" >> /etc/yum.repos.d/securityonionkernel.repo
+		echo "skip_if_unavailable=1" >> /etc/yum.repos.d/securityonionkernel.repo
 	elif [[ $waitforstate ]]; then
 		echo "[securityonion]" > /etc/yum.repos.d/securityonion.repo
 		echo "name=Security Onion Repo" >> /etc/yum.repos.d/securityonion.repo
 		echo "baseurl=file:///nsm/repo/" >> /etc/yum.repos.d/securityonion.repo
 		echo "enabled=1" >> /etc/yum.repos.d/securityonion.repo
 		echo "gpgcheck=1" >> /etc/yum.repos.d/securityonion.repo
+		echo "[securityonionkernel]" > /etc/yum.repos.d/securityonionkernel.repo
+		echo "name=Security Onion Kernel Repo" >> /etc/yum.repos.d/securityonionkernel.repo
+		echo "baseurl=file:///nsm/kernelrepo/" >> /etc/yum.repos.d/securityonionkernel.repo
+		echo "enabled=1" >> /etc/yum.repos.d/securityonionkernel.repo
+		echo "gpgcheck=1" >> /etc/yum.repos.d/securityonionkernel.repo
+		echo "skip_if_unavailable=1" >> /etc/yum.repos.d/securityonionkernel.repo
 	fi
 	logCmd "dnf repolist all"
 	if [[ $waitforstate ]]; then
@@ -1850,9 +1881,12 @@ repo_sync_local() {
 	# Sync the repo from the SO repo locally.
 	info "Adding Repo Download Configuration"
 	mkdir -p /nsm/repo
+	mkdir -p /nsm/kernelrepo
 	mkdir -p /opt/so/conf/reposync/cache
 	echo "https://repo.securityonion.net/file/so-repo/prod/3/oracle/9" > /opt/so/conf/reposync/mirror.txt
 	echo "https://repo-alt.securityonion.net/prod/3/oracle/9" >> /opt/so/conf/reposync/mirror.txt
+	echo "https://repo.securityonion.net/file/so-repo/prod/3/oracle/9-uek8" > /opt/so/conf/reposync/mirror-kernel.txt
+	echo "https://repo-alt.securityonion.net/prod/3/oracle/9-uek8" >> /opt/so/conf/reposync/mirror-kernel.txt
 	echo "[main]" > /opt/so/conf/reposync/repodownload.conf
 	echo "gpgcheck=1" >> /opt/so/conf/reposync/repodownload.conf
 	echo "installonly_limit=3" >> /opt/so/conf/reposync/repodownload.conf
@@ -1866,12 +1900,18 @@ repo_sync_local() {
 	echo "mirrorlist=file:///opt/so/conf/reposync/mirror.txt" >> /opt/so/conf/reposync/repodownload.conf
 	echo "enabled=1" >> /opt/so/conf/reposync/repodownload.conf
 	echo "gpgcheck=1" >> /opt/so/conf/reposync/repodownload.conf
+	echo "[securityonionkernel]" >> /opt/so/conf/reposync/repodownload.conf
+	echo "name=Security Onion Kernel Repo repo" >> /opt/so/conf/reposync/repodownload.conf
+	echo "mirrorlist=file:///opt/so/conf/reposync/mirror-kernel.txt" >> /opt/so/conf/reposync/repodownload.conf
+	echo "enabled=1" >> /opt/so/conf/reposync/repodownload.conf
+	echo "gpgcheck=1" >> /opt/so/conf/reposync/repodownload.conf
 
 	logCmd "dnf repolist"
 
 	if [[ ! $is_airgap ]]; then
 		curl --retry 5 --retry-delay 60 -A "netinstall/$SOVERSION/$OS/$(uname -r)/1" https://sigs.securityonion.net/checkup --output /tmp/install
 		retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/" >> "$setup_log" 2>&1 || fail_setup
+		retry 5 60 "dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionkernel --download-metadata -p /nsm/kernelrepo/" >> "$setup_log" 2>&1 || fail_setup
 		# After the download is complete run createrepo
 		create_repo
 	fi
@@ -2228,6 +2268,13 @@ update_sudoers_for_testing() {
 }
 
 update_packages() {
+	# Pin physical NIC names by MAC BEFORE pulling packages, so the UEK8 kernel that
+	# the update below installs can't renumber the interfaces SO binds by name. Doing
+	# it here (instead of waiting for the common highstate) also drops the
+	# /opt/so/state/nic_names_pinned marker that gates the kernel repo, so the kernel
+	# repo is assigned on the very first highstate and the kernel isn't downgraded and
+	# then re-upgraded. Run-once: so-nic-pin no-ops if the marker already exists.
+	logCmd "bash ../salt/common/tools/sbin/so-nic-pin"
 	logCmd "dnf repolist"
 	logCmd "dnf -y update --allowerasing --exclude=salt*,docker*,containerd*"
 	RMREPOFILES=("oracle-linux-ol9.repo" "uek-ol9.repo" "virt-ol9.repo")