From c86399327bbab0f34ec05f9a1fda26725ee3142e Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 28 Apr 2026 14:27:59 -0400 Subject: [PATCH 1/3] fix so-docker-refresh push for multi-arch source images docker pull of a multi-arch tag on Docker 29.x leaves the local tag pointing at the image index rather than the platform-specific manifest. The subsequent docker push then tries to push every sub-manifest the index references and fails on layers we never fetched. Resolve the local-platform manifest digest from the upstream index via docker buildx imagetools inspect, pull by that digest, and re-tag locally to the canonical tag. The signing flow and the existing tag/push to the embedded registry are unchanged. --- salt/common/tools/sbin/so-image-common | 44 +++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index e8f604681..b91bb07e1 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -104,6 +104,16 @@ update_docker_containers() { set_version set_os + # Resolve the local Docker daemon's platform once. Used below to pick the + # matching sub-manifest from a multi-arch image index — Docker 29.x will not + # push a tag that points at an index unless every referenced sub-manifest's + # content is present locally. + local PLATFORM_OS PLATFORM_ARCH + PLATFORM_OS=$(docker version --format '{{.Server.Os}}' 2>/dev/null) + PLATFORM_ARCH=$(docker version --format '{{.Server.Arch}}' 2>/dev/null) + PLATFORM_OS=${PLATFORM_OS:-linux} + PLATFORM_ARCH=${PLATFORM_ARCH:-amd64} + if [ -z "$TRUSTED_CONTAINERS" ]; then container_list fi @@ -161,10 +171,36 @@ update_docker_containers() { local image=$i:$VERSION$IMAGE_TAG_SUFFIX local sig_url=https://sigs.securityonion.net/$VERSION/$image.sig fi - # Pull down the trusted docker image - run_check_net_err \ - "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" \ - "Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 + # Pull down the trusted docker image. If the upstream tag is a multi-arch + # image index, resolve the local-platform manifest digest and pull by + # digest so the local tag points at a single-arch image (not an index). + # That keeps Docker 29.x's index-aware push from trying to push sub-manifests + # whose content we never fetched. + local FULL_IMAGE="$CONTAINER_REGISTRY/$IMAGEREPO/$image" + local PLATFORM_DIGEST + PLATFORM_DIGEST=$(docker buildx imagetools inspect --raw "$FULL_IMAGE" 2>/dev/null \ + | jq -r --arg os "$PLATFORM_OS" --arg arch "$PLATFORM_ARCH" ' + .manifests[]? + | select(.platform.os == $os + and .platform.architecture == $arch + and ((.platform.variant // "") == "") + and ((.annotations["vnd.docker.reference.type"] // "") != "attestation-manifest")) + | .digest' 2>/dev/null \ + | head -n 1) + + if [ -n "$PLATFORM_DIGEST" ] && [ "$PLATFORM_DIGEST" != "null" ]; then + run_check_net_err \ + "docker pull $FULL_IMAGE@$PLATFORM_DIGEST" \ + "Could not pull $image ($PLATFORM_OS/$PLATFORM_ARCH), please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 + docker tag "$FULL_IMAGE@$PLATFORM_DIGEST" "$FULL_IMAGE" >> "$LOG_FILE" 2>&1 || { + echo "Unable to retag $image to canonical tag" >> "$LOG_FILE" 2>&1 + exit 1 + } + else + run_check_net_err \ + "docker pull $FULL_IMAGE" \ + "Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 + fi # Get signature run_check_net_err \ From 288a823edf0776911c638c3fbf990762dfba9fd7 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 28 Apr 2026 14:49:02 -0400 Subject: [PATCH 2/3] push images via buildx imagetools create Replaces `docker push` with a registry-to-registry copy. On Docker 29.x with the containerd image store, `docker push` of a freshly-pulled image hits a path that wraps single-platform manifests in a synthetic index and then can't push the layers it claims to reference, producing `NotFound: content digest ...` even when the image is fully present. Keep the local `docker tag` so so-image-pull's `docker images | grep :5000` existence check continues to work. --- salt/common/tools/sbin/so-image-common | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index b91bb07e1..049bb03f9 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -225,11 +225,20 @@ update_docker_containers() { HOSTNAME=$(hostname) fi docker tag $CONTAINER_REGISTRY/$IMAGEREPO/$image $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 || { - echo "Unable to tag $image" >> "$LOG_FILE" 2>&1 + echo "Unable to tag $image" >> "$LOG_FILE" 2>&1 exit 1 } - docker push $HOSTNAME:5000/$IMAGEREPO/$image >> "$LOG_FILE" 2>&1 || { - echo "Unable to push $image" >> "$LOG_FILE" 2>&1 + # Push to the embedded registry via a registry-to-registry copy. Avoids + # `docker push`, which on Docker 29.x with the containerd image store + # represents freshly-pulled images as an index whose layer content + # isn't reachable through the push path. The local `docker tag` above + # is preserved so so-image-pull's `:5000` existence check still works. + local PUSH_SRC="$CONTAINER_REGISTRY/$IMAGEREPO/$image" + if [ -n "$PLATFORM_DIGEST" ] && [ "$PLATFORM_DIGEST" != "null" ]; then + PUSH_SRC="$CONTAINER_REGISTRY/$IMAGEREPO/$image@$PLATFORM_DIGEST" + fi + docker buildx imagetools create --tag $HOSTNAME:5000/$IMAGEREPO/$image "$PUSH_SRC" >> "$LOG_FILE" 2>&1 || { + echo "Unable to copy $image to embedded registry" >> "$LOG_FILE" 2>&1 exit 1 } fi From 82dac82d155e80afd1445903eee314a0a5afe718 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 28 Apr 2026 14:54:25 -0400 Subject: [PATCH 3/3] drop platform/digest pull resolution The digest-pull logic was added to make `docker push` work for multi-arch upstream tags. Now that the push step is `docker buildx imagetools create` pinned to the gpg-verified RepoDigest, the registry-to-registry copy handles single- and multi-arch sources without help. Reverts the pull back to the original line and removes the unused PLATFORM_OS/_ARCH detection. --- salt/common/tools/sbin/so-image-common | 56 ++++++-------------------- 1 file changed, 12 insertions(+), 44 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 049bb03f9..833b9a7d8 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -104,16 +104,6 @@ update_docker_containers() { set_version set_os - # Resolve the local Docker daemon's platform once. Used below to pick the - # matching sub-manifest from a multi-arch image index — Docker 29.x will not - # push a tag that points at an index unless every referenced sub-manifest's - # content is present locally. - local PLATFORM_OS PLATFORM_ARCH - PLATFORM_OS=$(docker version --format '{{.Server.Os}}' 2>/dev/null) - PLATFORM_ARCH=$(docker version --format '{{.Server.Arch}}' 2>/dev/null) - PLATFORM_OS=${PLATFORM_OS:-linux} - PLATFORM_ARCH=${PLATFORM_ARCH:-amd64} - if [ -z "$TRUSTED_CONTAINERS" ]; then container_list fi @@ -171,37 +161,11 @@ update_docker_containers() { local image=$i:$VERSION$IMAGE_TAG_SUFFIX local sig_url=https://sigs.securityonion.net/$VERSION/$image.sig fi - # Pull down the trusted docker image. If the upstream tag is a multi-arch - # image index, resolve the local-platform manifest digest and pull by - # digest so the local tag points at a single-arch image (not an index). - # That keeps Docker 29.x's index-aware push from trying to push sub-manifests - # whose content we never fetched. - local FULL_IMAGE="$CONTAINER_REGISTRY/$IMAGEREPO/$image" - local PLATFORM_DIGEST - PLATFORM_DIGEST=$(docker buildx imagetools inspect --raw "$FULL_IMAGE" 2>/dev/null \ - | jq -r --arg os "$PLATFORM_OS" --arg arch "$PLATFORM_ARCH" ' - .manifests[]? - | select(.platform.os == $os - and .platform.architecture == $arch - and ((.platform.variant // "") == "") - and ((.annotations["vnd.docker.reference.type"] // "") != "attestation-manifest")) - | .digest' 2>/dev/null \ - | head -n 1) + # Pull down the trusted docker image + run_check_net_err \ + "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" \ + "Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 - if [ -n "$PLATFORM_DIGEST" ] && [ "$PLATFORM_DIGEST" != "null" ]; then - run_check_net_err \ - "docker pull $FULL_IMAGE@$PLATFORM_DIGEST" \ - "Could not pull $image ($PLATFORM_OS/$PLATFORM_ARCH), please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 - docker tag "$FULL_IMAGE@$PLATFORM_DIGEST" "$FULL_IMAGE" >> "$LOG_FILE" 2>&1 || { - echo "Unable to retag $image to canonical tag" >> "$LOG_FILE" 2>&1 - exit 1 - } - else - run_check_net_err \ - "docker pull $FULL_IMAGE" \ - "Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 - fi - # Get signature run_check_net_err \ "curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' $sig_url --output $SIGNPATH/$image.sig" \ @@ -233,11 +197,15 @@ update_docker_containers() { # represents freshly-pulled images as an index whose layer content # isn't reachable through the push path. The local `docker tag` above # is preserved so so-image-pull's `:5000` existence check still works. - local PUSH_SRC="$CONTAINER_REGISTRY/$IMAGEREPO/$image" - if [ -n "$PLATFORM_DIGEST" ] && [ "$PLATFORM_DIGEST" != "null" ]; then - PUSH_SRC="$CONTAINER_REGISTRY/$IMAGEREPO/$image@$PLATFORM_DIGEST" + # Pin to the digest already gpg-verified above so we copy exactly the + # bytes we approved. + local VERIFIED_REF + VERIFIED_REF=$(echo "$DOCKERINSPECT" | jq -r ".[0].RepoDigests[] | select(. | contains(\"$CONTAINER_REGISTRY\"))" | head -n 1) + if [ -z "$VERIFIED_REF" ] || [ "$VERIFIED_REF" = "null" ]; then + echo "Unable to determine verified digest for $image" >> "$LOG_FILE" 2>&1 + exit 1 fi - docker buildx imagetools create --tag $HOSTNAME:5000/$IMAGEREPO/$image "$PUSH_SRC" >> "$LOG_FILE" 2>&1 || { + docker buildx imagetools create --tag $HOSTNAME:5000/$IMAGEREPO/$image "$VERIFIED_REF" >> "$LOG_FILE" 2>&1 || { echo "Unable to copy $image to embedded registry" >> "$LOG_FILE" 2>&1 exit 1 }