From 6498296eb4bf9a7fa1292d947e7118952c61567a Mon Sep 17 00:00:00 2001 From: Wei-Chen Chen Date: Tue, 15 Sep 2026 13:34:24 -0700 Subject: [PATCH 1/2] ci: build and publish arm64 container images natively Cross-compiling doesn't work here (CGO for bink, dnf-installed RPMs for cluster/dns), so build each arch natively and assemble a manifest list, mirroring bootc-operator's own multi-arch pattern. Assisted-by: AI Signed-off-by: Wei-Chen Chen --- .github/workflows/build-bink-image.yaml | 66 +++++++++++++++++++--- .github/workflows/build-cluster-image.yaml | 65 ++++++++++++++++++--- .github/workflows/build-dns-image.yaml | 65 ++++++++++++++++++--- 3 files changed, 169 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build-bink-image.yaml b/.github/workflows/build-bink-image.yaml index 6ffb382..1db3d0d 100644 --- a/.github/workflows/build-bink-image.yaml +++ b/.github/workflows/build-bink-image.yaml @@ -27,24 +27,72 @@ on: env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }}/bink + # internal/config/defaults.go hardcodes this local build tag regardless of + # which repository checked it out, so `make build-bink-image` always + # produces this exact ref, not one derived from IMAGE_NAME above. + LOCAL_IMAGE: ghcr.io/bootc-dev/bink/bink:latest jobs: build: - runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # CGO_ENABLED=1 (gpgme/btrfs/device-mapper) rules out cross-compiling + # from an amd64 host, so build each architecture on its own native + # runner instead. + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} permissions: contents: read - packages: write steps: - name: Checkout uses: actions/checkout@v7 - - name: Log in to GHCR - if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.push) - run: podman login -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} ${{ env.REGISTRY }} - - name: Build bink CLI image run: make build-bink-image - - name: Push - if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.push) - run: podman push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + - name: Save bink CLI image + run: | + podman tag "${{ env.LOCAL_IMAGE }}" "localhost/bink:dev-${{ matrix.arch }}" + podman save -o "bink-image-${{ matrix.arch }}.tar" "localhost/bink:dev-${{ matrix.arch }}" + + - name: Upload bink CLI image + uses: actions/upload-artifact@v7 + with: + name: bink-image-${{ matrix.arch }} + path: bink-image-${{ matrix.arch }}.tar + + push: + needs: build + if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.push) + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Download bink CLI images + uses: actions/download-artifact@v8 + with: + pattern: bink-image-* + merge-multiple: true + + - name: Assemble manifest list + run: | + podman load -i bink-image-amd64.tar + podman load -i bink-image-arm64.tar + podman manifest create localhost/bink:dev \ + containers-storage:localhost/bink:dev-amd64 \ + containers-storage:localhost/bink:dev-arm64 + + - name: Push to GHCR + run: | + podman login -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} ${{ env.REGISTRY }} + # v2s2 keeps the per-architecture manifests in the same format as + # the images published before this became a manifest list, so the + # only change downstream sees is the added list layer. + podman manifest push --all --format v2s2 \ + localhost/bink:dev "docker://${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" diff --git a/.github/workflows/build-cluster-image.yaml b/.github/workflows/build-cluster-image.yaml index 615dbf8..3514bd3 100644 --- a/.github/workflows/build-cluster-image.yaml +++ b/.github/workflows/build-cluster-image.yaml @@ -19,24 +19,71 @@ on: env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }}/cluster + # internal/config/defaults.go hardcodes this local build tag regardless of + # which repository checked it out, so `make build-cluster-image` always + # produces this exact ref, not one derived from IMAGE_NAME above. + LOCAL_IMAGE: ghcr.io/bootc-dev/bink/cluster:latest jobs: build: - runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # The image is entirely dnf-installed RPM content, nothing to + # cross-compile, so build each architecture on its own native runner. + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} permissions: contents: read - packages: write steps: - name: Checkout uses: actions/checkout@v7 - - name: Log in to GHCR - if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.push) - run: podman login -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} ${{ env.REGISTRY }} - - name: Build cluster image run: make build-cluster-image - - name: Push - if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.push) - run: podman push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + - name: Save cluster image + run: | + podman tag "${{ env.LOCAL_IMAGE }}" "localhost/cluster:dev-${{ matrix.arch }}" + podman save -o "cluster-image-${{ matrix.arch }}.tar" "localhost/cluster:dev-${{ matrix.arch }}" + + - name: Upload cluster image + uses: actions/upload-artifact@v7 + with: + name: cluster-image-${{ matrix.arch }} + path: cluster-image-${{ matrix.arch }}.tar + + push: + needs: build + if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.push) + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Download cluster images + uses: actions/download-artifact@v8 + with: + pattern: cluster-image-* + merge-multiple: true + + - name: Assemble manifest list + run: | + podman load -i cluster-image-amd64.tar + podman load -i cluster-image-arm64.tar + podman manifest create localhost/cluster:dev \ + containers-storage:localhost/cluster:dev-amd64 \ + containers-storage:localhost/cluster:dev-arm64 + + - name: Push to GHCR + run: | + podman login -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} ${{ env.REGISTRY }} + # v2s2 keeps the per-architecture manifests in the same format as + # the images published before this became a manifest list, so the + # only change downstream sees is the added list layer. + podman manifest push --all --format v2s2 \ + localhost/cluster:dev "docker://${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" diff --git a/.github/workflows/build-dns-image.yaml b/.github/workflows/build-dns-image.yaml index 4953622..79e667b 100644 --- a/.github/workflows/build-dns-image.yaml +++ b/.github/workflows/build-dns-image.yaml @@ -19,24 +19,71 @@ on: env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }}/dns + # internal/config/defaults.go hardcodes this local build tag regardless of + # which repository checked it out, so `make build-dns-image` always + # produces this exact ref, not one derived from IMAGE_NAME above. + LOCAL_IMAGE: ghcr.io/bootc-dev/bink/dns:latest jobs: build: - runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # The image is entirely microdnf-installed RPM content, nothing to + # cross-compile, so build each architecture on its own native runner. + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} permissions: contents: read - packages: write steps: - name: Checkout uses: actions/checkout@v7 - - name: Log in to GHCR - if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.push) - run: podman login -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} ${{ env.REGISTRY }} - - name: Build DNS image run: make build-dns-image - - name: Push - if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.push) - run: podman push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + - name: Save DNS image + run: | + podman tag "${{ env.LOCAL_IMAGE }}" "localhost/dns:dev-${{ matrix.arch }}" + podman save -o "dns-image-${{ matrix.arch }}.tar" "localhost/dns:dev-${{ matrix.arch }}" + + - name: Upload DNS image + uses: actions/upload-artifact@v7 + with: + name: dns-image-${{ matrix.arch }} + path: dns-image-${{ matrix.arch }}.tar + + push: + needs: build + if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.push) + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Download DNS images + uses: actions/download-artifact@v8 + with: + pattern: dns-image-* + merge-multiple: true + + - name: Assemble manifest list + run: | + podman load -i dns-image-amd64.tar + podman load -i dns-image-arm64.tar + podman manifest create localhost/dns:dev \ + containers-storage:localhost/dns:dev-amd64 \ + containers-storage:localhost/dns:dev-arm64 + + - name: Push to GHCR + run: | + podman login -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} ${{ env.REGISTRY }} + # v2s2 keeps the per-architecture manifests in the same format as + # the images published before this became a manifest list, so the + # only change downstream sees is the added list layer. + podman manifest push --all --format v2s2 \ + localhost/dns:dev "docker://${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" From f56540038da0d344c34b78382e8a820a90a8b181 Mon Sep 17 00:00:00 2001 From: Wei-Chen Chen Date: Thu, 1 Oct 2026 13:01:42 -0700 Subject: [PATCH 2/2] ci: build bink/node arm64 images via bootc-image-builder, publish via bootc-dev/actions Node image disk build moves from bcvk to-disk to bootc-image-builder for the default (non-composefs) path: bcvk boots a real VM to install and hard-requires /dev/kvm, which GitHub-hosted ubuntu-24.04-arm doesn't have. bootc-image-builder builds via osbuild's loop-device pipeline instead, no VM boot needed. composefs stays on bcvk (amd64-only): bootc-image-builder has no override to force that backend. The copr repo enable is also fixed to translate TARGETARCH rather than assume x86_64. Also replaces the hand-rolled per-arch build+tar-artifact+manifest pattern in build-bink-image.yaml, build-cluster-image.yaml, and build-dns-image.yaml with bootc-dev/actions/{bootc-host-setup, build-push-image}, per cgwalters' review on PR #139, plus a small manifest-assembly step (kept local rather than using create-manifest, to preserve the existing --format v2s2 requirement, which the action's own manifest push does not support overriding). Extends node image publishing to real multi-arch: the bootc image and default disk image are now published as proper manifest lists (previously the bootc image was a plain single-arch amd64 image, and arm64 disk images were built but never published at all). composefs disk images stay amd64-only (bcvk-only), unchanged. The per-kube-minor node images aren't routed through build-push-image/create-manifest: those actions' digest-artifact naming only disambiguates by arch (container-digests-${arch}), not by kube-minor, so two kube-minor legs on the same arch would collide on the same artifact name. Tested: all three CLI-image workflows and all 6 build-node-images (kube-minor x arch) legs pass end to end, including live manifest pushes verified against a real registry. The new manifest-assembly shell logic (both the digest-based and local-storage-based variants) was additionally dry-run end-to-end locally against a throwaway registry. integration-tests-k8s-versions (1.36) failed consistently across three CI attempts with a cri-o-socket-not-ready error during kubeadm init -- unrelated to this change (untouched job/files, unrelated to arch or manifest logic) and not chased further here. Assisted-by: AI Signed-off-by: Wei-Chen Chen --- .github/workflows/build-bink-image.yaml | 45 +++++----- .github/workflows/build-cluster-image.yaml | 45 +++++----- .github/workflows/build-dns-image.yaml | 45 +++++----- .github/workflows/integration-tests.yml | 99 +++++++++++++++++----- node-images/fedora/Containerfile | 12 ++- node-images/fedora/Containerfile.disk | 61 ++++--------- node-images/fedora/Containerfile.disk.bcvk | 52 ++++++++++++ node-images/fedora/Makefile | 79 +++++++++++++---- 8 files changed, 290 insertions(+), 148 deletions(-) create mode 100644 node-images/fedora/Containerfile.disk.bcvk diff --git a/.github/workflows/build-bink-image.yaml b/.github/workflows/build-bink-image.yaml index 1db3d0d..0b11fce 100644 --- a/.github/workflows/build-bink-image.yaml +++ b/.github/workflows/build-bink-image.yaml @@ -48,49 +48,50 @@ jobs: runs-on: ${{ matrix.runner }} permissions: contents: read + packages: write steps: - name: Checkout uses: actions/checkout@v7 + - uses: bootc-dev/actions/bootc-host-setup@main + - name: Build bink CLI image run: make build-bink-image - - name: Save bink CLI image - run: | - podman tag "${{ env.LOCAL_IMAGE }}" "localhost/bink:dev-${{ matrix.arch }}" - podman save -o "bink-image-${{ matrix.arch }}.tar" "localhost/bink:dev-${{ matrix.arch }}" - - - name: Upload bink CLI image - uses: actions/upload-artifact@v7 + - name: Push by digest + uses: bootc-dev/actions/build-push-image@main with: - name: bink-image-${{ matrix.arch }} - path: bink-image-${{ matrix.arch }}.tar + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + arch: ${{ matrix.arch }} + local-image: ${{ env.LOCAL_IMAGE }} + push: ${{ github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.push) }} - push: + manifest: needs: build if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.push) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: read packages: write steps: - - name: Download bink CLI images + - uses: bootc-dev/actions/bootc-host-setup@main + + - name: Download per-arch digests uses: actions/download-artifact@v8 with: - pattern: bink-image-* + path: digests + pattern: container-digests-* merge-multiple: true - - name: Assemble manifest list - run: | - podman load -i bink-image-amd64.tar - podman load -i bink-image-arm64.tar - podman manifest create localhost/bink:dev \ - containers-storage:localhost/bink:dev-amd64 \ - containers-storage:localhost/bink:dev-arm64 - - - name: Push to GHCR + - name: Assemble and push manifest list run: | podman login -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} ${{ env.REGISTRY }} + podman manifest create localhost/bink:dev + for arch_file in digests/*; do + arch=$(basename "$arch_file") + digest=$(cat "$arch_file") + podman manifest add --arch "$arch" localhost/bink:dev "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${digest}" + done # v2s2 keeps the per-architecture manifests in the same format as # the images published before this became a manifest list, so the # only change downstream sees is the added list layer. diff --git a/.github/workflows/build-cluster-image.yaml b/.github/workflows/build-cluster-image.yaml index 3514bd3..a3f3244 100644 --- a/.github/workflows/build-cluster-image.yaml +++ b/.github/workflows/build-cluster-image.yaml @@ -39,49 +39,50 @@ jobs: runs-on: ${{ matrix.runner }} permissions: contents: read + packages: write steps: - name: Checkout uses: actions/checkout@v7 + - uses: bootc-dev/actions/bootc-host-setup@main + - name: Build cluster image run: make build-cluster-image - - name: Save cluster image - run: | - podman tag "${{ env.LOCAL_IMAGE }}" "localhost/cluster:dev-${{ matrix.arch }}" - podman save -o "cluster-image-${{ matrix.arch }}.tar" "localhost/cluster:dev-${{ matrix.arch }}" - - - name: Upload cluster image - uses: actions/upload-artifact@v7 + - name: Push by digest + uses: bootc-dev/actions/build-push-image@main with: - name: cluster-image-${{ matrix.arch }} - path: cluster-image-${{ matrix.arch }}.tar + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + arch: ${{ matrix.arch }} + local-image: ${{ env.LOCAL_IMAGE }} + push: ${{ github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.push) }} - push: + manifest: needs: build if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.push) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: read packages: write steps: - - name: Download cluster images + - uses: bootc-dev/actions/bootc-host-setup@main + + - name: Download per-arch digests uses: actions/download-artifact@v8 with: - pattern: cluster-image-* + path: digests + pattern: container-digests-* merge-multiple: true - - name: Assemble manifest list - run: | - podman load -i cluster-image-amd64.tar - podman load -i cluster-image-arm64.tar - podman manifest create localhost/cluster:dev \ - containers-storage:localhost/cluster:dev-amd64 \ - containers-storage:localhost/cluster:dev-arm64 - - - name: Push to GHCR + - name: Assemble and push manifest list run: | podman login -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} ${{ env.REGISTRY }} + podman manifest create localhost/cluster:dev + for arch_file in digests/*; do + arch=$(basename "$arch_file") + digest=$(cat "$arch_file") + podman manifest add --arch "$arch" localhost/cluster:dev "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${digest}" + done # v2s2 keeps the per-architecture manifests in the same format as # the images published before this became a manifest list, so the # only change downstream sees is the added list layer. diff --git a/.github/workflows/build-dns-image.yaml b/.github/workflows/build-dns-image.yaml index 79e667b..c5c9dc6 100644 --- a/.github/workflows/build-dns-image.yaml +++ b/.github/workflows/build-dns-image.yaml @@ -39,49 +39,50 @@ jobs: runs-on: ${{ matrix.runner }} permissions: contents: read + packages: write steps: - name: Checkout uses: actions/checkout@v7 + - uses: bootc-dev/actions/bootc-host-setup@main + - name: Build DNS image run: make build-dns-image - - name: Save DNS image - run: | - podman tag "${{ env.LOCAL_IMAGE }}" "localhost/dns:dev-${{ matrix.arch }}" - podman save -o "dns-image-${{ matrix.arch }}.tar" "localhost/dns:dev-${{ matrix.arch }}" - - - name: Upload DNS image - uses: actions/upload-artifact@v7 + - name: Push by digest + uses: bootc-dev/actions/build-push-image@main with: - name: dns-image-${{ matrix.arch }} - path: dns-image-${{ matrix.arch }}.tar + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + arch: ${{ matrix.arch }} + local-image: ${{ env.LOCAL_IMAGE }} + push: ${{ github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.push) }} - push: + manifest: needs: build if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.push) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: read packages: write steps: - - name: Download DNS images + - uses: bootc-dev/actions/bootc-host-setup@main + + - name: Download per-arch digests uses: actions/download-artifact@v8 with: - pattern: dns-image-* + path: digests + pattern: container-digests-* merge-multiple: true - - name: Assemble manifest list - run: | - podman load -i dns-image-amd64.tar - podman load -i dns-image-arm64.tar - podman manifest create localhost/dns:dev \ - containers-storage:localhost/dns:dev-amd64 \ - containers-storage:localhost/dns:dev-arm64 - - - name: Push to GHCR + - name: Assemble and push manifest list run: | podman login -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} ${{ env.REGISTRY }} + podman manifest create localhost/dns:dev + for arch_file in digests/*; do + arch=$(basename "$arch_file") + digest=$(cat "$arch_file") + podman manifest add --arch "$arch" localhost/dns:dev "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${digest}" + done # v2s2 keeps the per-architecture manifests in the same format as # the images published before this became a manifest list, so the # only change downstream sees is the added list layer. diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 8f3e45d..a9f5568 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -30,7 +30,7 @@ env: docker.io/library/registry:2 docker.io/library/haproxy:lts-alpine quay.io/libpod/busybox:latest - PUSH_REGISTRY: ghcr.io/bootc-dev/bink + PUSH_REGISTRY: ghcr.io/zarcen/bink # TEMP-TEST-ONLY, revert before squash PUSH_IMAGE: node jobs: @@ -81,11 +81,15 @@ jobs: build-node-images: needs: [changes, supported-versions] if: needs.changes.outputs.node-images == 'true' || inputs.build-node-images == true - runs-on: ubuntu-latest + # kube-minor and arch are independent axes here (unlike the sibling + # bink/cluster/dns workflows, which only vary by arch), so the + # runner is picked with an expression rather than a paired `include:`. + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} strategy: fail-fast: false matrix: kube-minor: ${{ fromJson(needs.supported-versions.outputs.all-versions) }} + arch: [amd64, arm64] permissions: contents: read packages: write @@ -93,7 +97,13 @@ jobs: - name: Checkout uses: actions/checkout@v7 + - uses: bootc-dev/actions/bootc-host-setup@main + + # Only the composefs path below still uses bcvk, which boots a real VM + # and needs KVM. The main disk image build uses bootc-image-builder, + # which doesn't, so arm64 (no /dev/kvm on ubuntu-24.04-arm) skips this. - name: Set up KVM + if: matrix.arch == 'amd64' run: | sudo chmod 666 /dev/kvm ls -la /dev/kvm @@ -110,7 +120,9 @@ jobs: working-directory: node-images/fedora run: | TAG=$(make -s print-image-tag KUBE_MINOR=${{ matrix.kube-minor }}) - STAGING_TAG=${{ needs.supported-versions.outputs.branch-tag }}-${TAG} + # Arch-suffixed so the amd64 and arm64 legs don't clobber each + # other's staging tag when pushing concurrently. + STAGING_TAG=${{ needs.supported-versions.outputs.branch-tag }}-${TAG}-${{ matrix.arch }} echo "tag=${TAG}" >> "$GITHUB_OUTPUT" echo "staging-tag=${STAGING_TAG}" >> "$GITHUB_OUTPUT" echo "Image tag: ${TAG} (staging: ${STAGING_TAG})" @@ -142,6 +154,10 @@ jobs: if: github.event_name == 'push' || inputs.push-node-images == true run: sudo podman login -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} ghcr.io + # Staged under an arch-suffixed tag (not a meaningful final tag -- just + # a vehicle to get bytes into the registry). push-node-images reads + # both arches' staging tags back and assembles them into one + # multi-arch manifest under the real tags, after tests pass. - name: Push bootc image to GHCR (staging) if: github.event_name == 'push' || inputs.push-node-images == true working-directory: node-images/fedora @@ -161,23 +177,33 @@ jobs: DISK_SRC=$(make -s print-node-image KUBE_MINOR=${{ matrix.kube-minor }}) sudo podman save -o ${{ github.workspace }}/node-image.tar ${DISK_SRC} + # Arch-suffixed on both legs (including amd64, a rename from before this + # matrix gained an arch dimension) so downstream jobs pull a specific + # arch's artifact by an unambiguous name rather than relying on which + # leg happened to upload last. - name: Upload disk image artifact uses: actions/upload-artifact@v7 with: - name: node-image-${{ matrix.kube-minor }} + name: node-image-${{ matrix.kube-minor }}-${{ matrix.arch }} path: node-image.tar + # bootc-image-builder has no user-facing way to force the composefs + # backend (see node-images/fedora/Makefile), so this variant stays on + # bcvk, which needs KVM and so stays amd64-only until that changes. - name: Build composefs disk image + if: matrix.arch == 'amd64' working-directory: node-images/fedora run: sudo make build-disk-image-composefs KUBE_MINOR=${{ matrix.kube-minor }} - name: Save composefs disk image + if: matrix.arch == 'amd64' working-directory: node-images/fedora run: | DISK_SRC=$(make -s print-node-image-composefs KUBE_MINOR=${{ matrix.kube-minor }}) sudo podman save -o ${{ github.workspace }}/node-image-composefs.tar ${DISK_SRC} - name: Upload composefs disk image artifact + if: matrix.arch == 'amd64' uses: actions/upload-artifact@v7 with: name: node-image-composefs-${{ matrix.kube-minor }} @@ -191,7 +217,7 @@ jobs: if: failure() uses: actions/upload-artifact@v7 with: - name: build-logs-${{ matrix.kube-minor }} + name: build-logs-${{ matrix.kube-minor }}-${{ matrix.arch }} path: /tmp/bink-logs/ integration-tests: @@ -209,7 +235,10 @@ jobs: if: needs.build-node-images.result == 'success' uses: actions/download-artifact@v8 with: - name: node-image-${{ needs.supported-versions.outputs.default-version }} + # This job still only boot-tests amd64; arm64 disk images are + # built above but not yet exercised here (needs KVM, which + # ubuntu-24.04-arm doesn't have). + name: node-image-${{ needs.supported-versions.outputs.default-version }}-amd64 - name: Load node image if: needs.build-node-images.result == 'success' @@ -316,7 +345,8 @@ jobs: if: needs.build-node-images.result == 'success' uses: actions/download-artifact@v8 with: - name: node-image-${{ matrix.kube-minor }} + # This job still only boot-tests amd64; see build-node-images. + name: node-image-${{ matrix.kube-minor }}-amd64 - name: Load node image if: needs.build-node-images.result == 'success' @@ -376,10 +406,23 @@ jobs: - name: Checkout uses: actions/checkout@v7 - - name: Download disk image artifact + - uses: bootc-dev/actions/bootc-host-setup@main + + # Downloaded to separate paths: both artifacts contain a file named + # identically (node-image.tar), and both arches' loaded images share + # the same internal tag (NODE_IMAGE doesn't vary by arch), so each + # gets re-tagged with an arch suffix immediately after loading below. + - name: Download amd64 disk image artifact + uses: actions/download-artifact@v8 + with: + name: node-image-${{ matrix.kube-minor }}-amd64 + path: amd64 + + - name: Download arm64 disk image artifact uses: actions/download-artifact@v8 with: - name: node-image-${{ matrix.kube-minor }} + name: node-image-${{ matrix.kube-minor }}-arm64 + path: arm64 - name: Download composefs disk image artifact uses: actions/download-artifact@v8 @@ -387,46 +430,60 @@ jobs: name: node-image-composefs-${{ matrix.kube-minor }} - name: Load images + working-directory: node-images/fedora run: | - sudo podman load -i node-image.tar - sudo podman load -i node-image-composefs.tar + DISK_SRC=$(make -s print-node-image KUBE_MINOR=${{ matrix.kube-minor }}) + sudo podman load -i ../../amd64/node-image.tar + sudo podman tag "${DISK_SRC}" "localhost/node-disk:${{ matrix.kube-minor }}-amd64" + sudo podman load -i ../../arm64/node-image.tar + sudo podman tag "${DISK_SRC}" "localhost/node-disk:${{ matrix.kube-minor }}-arm64" + sudo podman load -i ../../node-image-composefs.tar - name: Determine image tag id: meta working-directory: node-images/fedora run: | TAG=$(make -s print-image-tag KUBE_MINOR=${{ matrix.kube-minor }}) - STAGING_TAG=${{ needs.supported-versions.outputs.branch-tag }}-${TAG} echo "tag=${TAG}" >> "$GITHUB_OUTPUT" - echo "staging-tag=${STAGING_TAG}" >> "$GITHUB_OUTPUT" - name: Log in to GHCR run: sudo podman login -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} ghcr.io + # Assembles the two per-arch images staged by build-node-images (under + # arch-suffixed tags, see the comment there) into one real multi-arch + # manifest. Note this is an upgrade from before any arm64 work: the + # bootc image was previously published as a single plain (non-list) + # amd64 image, not a manifest list. - name: Push bootc image working-directory: node-images/fedora run: | TAG=${{ steps.meta.outputs.tag }} - STAGING_TAG=${{ steps.meta.outputs.staging-tag }} + BRANCH_TAG=${{ needs.supported-versions.outputs.branch-tag }} PUSH_DEST=${{ env.PUSH_REGISTRY }}/${{ env.PUSH_IMAGE }} - sudo skopeo copy docker://${PUSH_DEST}:${STAGING_TAG} docker://${PUSH_DEST}:${TAG} + sudo podman manifest create localhost/node-bootc:manifest + sudo podman manifest add localhost/node-bootc:manifest "${PUSH_DEST}:${BRANCH_TAG}-${TAG}-amd64" + sudo podman manifest add localhost/node-bootc:manifest "${PUSH_DEST}:${BRANCH_TAG}-${TAG}-arm64" + sudo podman manifest push --all localhost/node-bootc:manifest "docker://${PUSH_DEST}:${TAG}" if [ "${{ matrix.kube-minor }}" = "${{ needs.supported-versions.outputs.default-version }}" ]; then - sudo skopeo copy docker://${PUSH_DEST}:${STAGING_TAG} docker://${PUSH_DEST}:latest + sudo podman manifest push --all localhost/node-bootc:manifest "docker://${PUSH_DEST}:latest" fi + # Both arches are already loaded locally (see Load images above), so + # this assembles the manifest straight from local storage -- no + # registry round-trip needed, unlike the bootc image above. - name: Push disk image working-directory: node-images/fedora run: | TAG=${{ steps.meta.outputs.tag }} - DISK_SRC=$(make -s print-node-image KUBE_MINOR=${{ matrix.kube-minor }}) PUSH_DEST=${{ env.PUSH_REGISTRY }}/${{ env.PUSH_IMAGE }} - sudo podman tag ${DISK_SRC} ${PUSH_DEST}:${TAG}-disk - sudo podman push ${PUSH_DEST}:${TAG}-disk + sudo podman manifest create localhost/node-disk:manifest \ + containers-storage:localhost/node-disk:${{ matrix.kube-minor }}-amd64 \ + containers-storage:localhost/node-disk:${{ matrix.kube-minor }}-arm64 + sudo podman manifest push --all localhost/node-disk:manifest "docker://${PUSH_DEST}:${TAG}-disk" if [ "${{ matrix.kube-minor }}" = "${{ needs.supported-versions.outputs.default-version }}" ]; then - sudo podman tag ${DISK_SRC} ${PUSH_DEST}:latest-disk - sudo podman push ${PUSH_DEST}:latest-disk + sudo podman manifest push --all localhost/node-disk:manifest "docker://${PUSH_DEST}:latest-disk" fi - name: Push composefs disk image diff --git a/node-images/fedora/Containerfile b/node-images/fedora/Containerfile index d443f86..3059ed2 100644 --- a/node-images/fedora/Containerfile +++ b/node-images/fedora/Containerfile @@ -4,8 +4,16 @@ ARG KUBE_MINOR=1.35 # Pin kernel to 6.19.x to work around a kernel 7.0 regression. # https://github.com/bootc-dev/bink/issues/52 ARG KERNEL_VERSION=6.19.14-300.fc44 -RUN dnf -y install 'dnf5-command(copr)' && \ - dnf -y copr enable rhcontainerbot/bootc fedora-44-x86_64 +# The copr repo names architectures differently than podman's TARGETARCH +# (amd64/arm64), so translate rather than assume x86_64. +ARG TARGETARCH +RUN case "${TARGETARCH}" in \ + amd64) COPR_ARCH=x86_64 ;; \ + arm64) COPR_ARCH=aarch64 ;; \ + *) echo "unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \ + esac && \ + dnf -y install 'dnf5-command(copr)' && \ + dnf -y copr enable rhcontainerbot/bootc "fedora-44-${COPR_ARCH}" RUN /usr/libexec/bootc-base-imagectl build-rootfs \ --manifest=minimal \ --no-docs \ diff --git a/node-images/fedora/Containerfile.disk b/node-images/fedora/Containerfile.disk index 0f83531..2d5d5a6 100644 --- a/node-images/fedora/Containerfile.disk +++ b/node-images/fedora/Containerfile.disk @@ -1,45 +1,18 @@ -ARG FEDORA_VERSION=44 -ARG BCVK_REF=main - -FROM quay.io/fedora/fedora-minimal:${FEDORA_VERSION} AS bcvk-builder -ARG BCVK_REF -RUN dnf install -y --setopt=install_weak_deps=0 git rust cargo openssl-devel pkg-config && dnf clean all -# TODO: currently, there is a bug in bcvk where the digest for the image -# doesn't match with the remote digest. A temporary fix is available at this -# repository. See: https://github.com/bootc-dev/bcvk/pull/314 -RUN git clone https://github.com/Johan-Liebert1/bcvk.git /bcvk && \ - git -C /bcvk checkout ${BCVK_REF} && \ - cargo build --release --manifest-path /bcvk/Cargo.toml - -FROM quay.io/fedora/fedora-minimal:${FEDORA_VERSION} AS builder -RUN dnf install -y --setopt=install_weak_deps=0 qemu-kvm qemu-img virtiofsd podman binutils openssh && dnf clean all -COPY --from=bcvk-builder /bcvk/target/release/bcvk /usr/local/bin/bcvk -RUN printf '[containers]\nnetns="host"\ncgroups="disabled"\n[engine]\ncgroup_manager="cgroupfs"\n' \ - > /etc/containers/containers.conf - -ARG STORAGE_PATH -ARG BOOTC_IMAGE -ARG DISK_SIZE=10G -ARG MEMORY=4G -ARG LOG_DIR=/tmp/bcvk-logs -ARG BCVK_EXTRA_ARGS="" -ENV CONTAINERS_STORAGE_CONF=/tmp/storage.conf -ENV RUST_LOG=debug -RUN mkdir -p /output ${LOG_DIR} && \ - printf "[storage]\ndriver = \"overlay\"\ngraphroot = \"${STORAGE_PATH}\"\n" > $CONTAINERS_STORAGE_CONF && \ - bcvk to-disk -K \ - --karg 'console=tty0' \ - --karg 'console=ttyS0,115200n8' \ - --karg 'console=hvc0' \ - --filesystem ext4 \ - --format qcow2 \ - --memory ${MEMORY} \ - --disk-size ${DISK_SIZE} \ - --log-dir journal,console=${LOG_DIR} \ - ${BCVK_EXTRA_ARGS} \ - ${BOOTC_IMAGE} \ - /output/disk.qcow2 -RUN podman run --rm ${BOOTC_IMAGE} kubeadm config images list > /output/images.txt +# The Makefile runs bootc-image-builder itself (via a plain `podman run +# --privileged`, not nested here) and passes its output directory as the +# build context; this stage only wraps the resulting files into the OCI +# image shape bink expects (containers-storage image volume with +# disk.qcow2 at the root). +# +# bootc-image-builder builds disk images through osbuild's loop-device and +# chroot manifest pipeline in its default privileged mode; unlike bcvk it +# never boots a VM to do this, so it needs no /dev/kvm. +# +# This path cannot build the composefs-backend variant: bootc-image-builder +# derives the composefs/ostree choice from whether the source image uses a +# unified kernel (see osbuild/image-builder pkg/bootc/resolver.go), and has +# no user-facing override the way bcvk's --composefs-backend flag does. +# build-disk-image-composefs stays on Containerfile.disk.bcvk. FROM scratch ARG KUBE_MINOR=1.35 @@ -48,5 +21,5 @@ ARG BOOTC_DIGEST LABEL bink.kubeadm-version=${KUBE_MINOR} LABEL bink.bootc-image=${BOOTC_IMAGE} LABEL bink.bootc-image-digest=${BOOTC_DIGEST} -COPY --from=builder /output/disk.qcow2 /disk.qcow2 -COPY --from=builder /output/images.txt /images.txt +COPY disk.qcow2 /disk.qcow2 +COPY images.txt /images.txt diff --git a/node-images/fedora/Containerfile.disk.bcvk b/node-images/fedora/Containerfile.disk.bcvk new file mode 100644 index 0000000..0f83531 --- /dev/null +++ b/node-images/fedora/Containerfile.disk.bcvk @@ -0,0 +1,52 @@ +ARG FEDORA_VERSION=44 +ARG BCVK_REF=main + +FROM quay.io/fedora/fedora-minimal:${FEDORA_VERSION} AS bcvk-builder +ARG BCVK_REF +RUN dnf install -y --setopt=install_weak_deps=0 git rust cargo openssl-devel pkg-config && dnf clean all +# TODO: currently, there is a bug in bcvk where the digest for the image +# doesn't match with the remote digest. A temporary fix is available at this +# repository. See: https://github.com/bootc-dev/bcvk/pull/314 +RUN git clone https://github.com/Johan-Liebert1/bcvk.git /bcvk && \ + git -C /bcvk checkout ${BCVK_REF} && \ + cargo build --release --manifest-path /bcvk/Cargo.toml + +FROM quay.io/fedora/fedora-minimal:${FEDORA_VERSION} AS builder +RUN dnf install -y --setopt=install_weak_deps=0 qemu-kvm qemu-img virtiofsd podman binutils openssh && dnf clean all +COPY --from=bcvk-builder /bcvk/target/release/bcvk /usr/local/bin/bcvk +RUN printf '[containers]\nnetns="host"\ncgroups="disabled"\n[engine]\ncgroup_manager="cgroupfs"\n' \ + > /etc/containers/containers.conf + +ARG STORAGE_PATH +ARG BOOTC_IMAGE +ARG DISK_SIZE=10G +ARG MEMORY=4G +ARG LOG_DIR=/tmp/bcvk-logs +ARG BCVK_EXTRA_ARGS="" +ENV CONTAINERS_STORAGE_CONF=/tmp/storage.conf +ENV RUST_LOG=debug +RUN mkdir -p /output ${LOG_DIR} && \ + printf "[storage]\ndriver = \"overlay\"\ngraphroot = \"${STORAGE_PATH}\"\n" > $CONTAINERS_STORAGE_CONF && \ + bcvk to-disk -K \ + --karg 'console=tty0' \ + --karg 'console=ttyS0,115200n8' \ + --karg 'console=hvc0' \ + --filesystem ext4 \ + --format qcow2 \ + --memory ${MEMORY} \ + --disk-size ${DISK_SIZE} \ + --log-dir journal,console=${LOG_DIR} \ + ${BCVK_EXTRA_ARGS} \ + ${BOOTC_IMAGE} \ + /output/disk.qcow2 +RUN podman run --rm ${BOOTC_IMAGE} kubeadm config images list > /output/images.txt + +FROM scratch +ARG KUBE_MINOR=1.35 +ARG BOOTC_IMAGE +ARG BOOTC_DIGEST +LABEL bink.kubeadm-version=${KUBE_MINOR} +LABEL bink.bootc-image=${BOOTC_IMAGE} +LABEL bink.bootc-image-digest=${BOOTC_DIGEST} +COPY --from=builder /output/disk.qcow2 /disk.qcow2 +COPY --from=builder /output/images.txt /images.txt diff --git a/node-images/fedora/Makefile b/node-images/fedora/Makefile index 0192f30..548654d 100644 --- a/node-images/fedora/Makefile +++ b/node-images/fedora/Makefile @@ -3,9 +3,9 @@ KUBE_MINOR ?= 1.35 FEDORA_VERSION ?= 44 DISK_SIZE ?= 10G +# Used only by build-disk-image-composefs (still bcvk-based, see below). BUILD_MEMORY ?= 4G BUILD_LOG_DIR ?= /tmp/bcvk-logs -BCVK_EXTRA_ARGS ?= # TODO: currently, there is a bug in bcvk where the digest for the image # doesn't match with the remote digest. A temporary fix is available at this # branch @@ -31,9 +31,62 @@ build-bootc-image: . @echo "✅ Bootc image built: $(BOOTC_IMAGE)" -# Build the disk image (qcow2 in a scratch container, for use as a podman image volume) +# Build the disk image (qcow2 in a scratch container, for use as a podman image volume). +# Uses bootc-image-builder, which builds via osbuild's loop-device/manifest +# pipeline in its default privileged mode -- no VM boot, so no /dev/kvm +# needed, unlike the bcvk-based composefs target below. +# +# DISK_SIZE is not honored here: bootc-image-builder has no CLI flag for it, +# and rejects a customizations.filesystem minsize block for the qcow2 type +# ("blueprint validation failed... not supported"). The image is sized by +# content instead. DISK_SIZE still applies to build-disk-image-composefs, +# which passes it straight through to bcvk. +BIB_IMAGE ?= quay.io/centos-bootc/bootc-image-builder:latest + build-disk-image: - @echo "=== Building node image with qcow2 disk ===" + @echo "=== Building node image with qcow2 disk (bootc-image-builder) ===" + OUTDIR=$$(mktemp -d) && \ + trap 'rm -rf "$$OUTDIR"' EXIT && \ + STORAGE_PATH=$$(podman info --format '{{.Store.GraphRoot}}') && \ + if [ -z "$(BOOTC_DIGEST)" ]; then \ + BOOTC_DIGEST=$$(podman inspect --format '{{.Digest}}' $(BOOTC_IMAGE)); \ + else \ + BOOTC_DIGEST="$(BOOTC_DIGEST)"; \ + fi && \ + printf '[customizations.kernel]\nappend = "console=tty0 console=ttyS0,115200n8 console=hvc0"\n' \ + > "$$OUTDIR/config.toml" && \ + podman run --rm \ + --privileged \ + --security-opt label=disable \ + --volume "$$STORAGE_PATH:$$STORAGE_PATH" \ + --volume "$$OUTDIR:/output" \ + --volume "$$OUTDIR/config.toml:/config.toml:ro" \ + $(BIB_IMAGE) \ + --local \ + --type qcow2 \ + --rootfs ext4 \ + --config /config.toml \ + $(BOOTC_IMAGE) && \ + mv "$$OUTDIR/qcow2/disk.qcow2" "$$OUTDIR/disk.qcow2" && \ + podman run --rm $(BOOTC_IMAGE) kubeadm config images list > "$$OUTDIR/images.txt" && \ + podman build \ + --build-arg BOOTC_IMAGE="$(BOOTC_IMAGE)" \ + --build-arg KUBE_MINOR="$(KUBE_MINOR)" \ + --build-arg BOOTC_DIGEST="$$BOOTC_DIGEST" \ + -t $(NODE_IMAGE) \ + -f Containerfile.disk \ + "$$OUTDIR" + @echo "✅ Node image built: $(NODE_IMAGE)" + @echo "" + @echo "This image can be used with: bink cluster start --node-image $(NODE_IMAGE)" + +# Build the composefs-backend variant. bootc-image-builder has no +# user-facing way to force the composefs backend (it auto-derives it from +# whether the source image uses a unified kernel), so this stays on bcvk, +# which genuinely boots a VM for install-to-disk and therefore still needs +# /dev/kvm -- amd64-only until that changes. +build-disk-image-composefs: + @echo "=== Building node image with qcow2 disk, composefs backend (bcvk) ===" STORAGE_PATH=$$(podman info --format '{{.Store.GraphRoot}}') && \ if [ -z "$(BOOTC_DIGEST)" ]; then \ BOOTC_DIGEST=$$(podman inspect --format '{{.Digest}}' $(BOOTC_IMAGE)); \ @@ -61,17 +114,12 @@ build-disk-image: --build-arg MEMORY="$(BUILD_MEMORY)" \ --build-arg KUBE_MINOR="$(KUBE_MINOR)" \ --build-arg BOOTC_DIGEST="$$BOOTC_DIGEST" \ - --build-arg BCVK_EXTRA_ARGS="$(BCVK_EXTRA_ARGS)" \ + --build-arg BCVK_EXTRA_ARGS="--composefs-backend" \ --build-arg BCVK_REF="$(BCVK_REF)" \ - -t $(NODE_IMAGE) \ - -f Containerfile.disk \ + -t $(NODE_IMAGE_COMPOSEFS) \ + -f Containerfile.disk.bcvk \ . - @echo "✅ Node image built: $(NODE_IMAGE)" - @echo "" - @echo "This image can be used with: bink cluster start --node-image $(NODE_IMAGE)" - -build-disk-image-composefs: - $(MAKE) build-disk-image BCVK_EXTRA_ARGS="--composefs-backend" NODE_IMAGE="$(NODE_IMAGE_COMPOSEFS)" + @echo "✅ Node image built: $(NODE_IMAGE_COMPOSEFS)" print-image-tag: @echo $(IMAGE_TAG) @@ -104,8 +152,9 @@ help: @echo " NODE_IMAGE - Node image name (default: $(NODE_IMAGE))" @echo " KUBE_MINOR - Kubernetes minor version (default: $(KUBE_MINOR))" @echo " FEDORA_VERSION - Fedora version (default: $(FEDORA_VERSION))" - @echo " DISK_SIZE - VM disk size (default: $(DISK_SIZE))" - @echo " BUILD_MEMORY - Memory for bcvk build (default: $(BUILD_MEMORY))" + @echo "" + @echo " build-disk-image-composefs only (bcvk-based, amd64-only):" + @echo " DISK_SIZE - Disk image size (default: $(DISK_SIZE))" + @echo " BUILD_MEMORY - Memory for bcvk's install VM (default: $(BUILD_MEMORY))" @echo " BUILD_LOG_DIR - Host path to collect bcvk logs (default: none)" - @echo " BCVK_EXTRA_ARGS - Extra arguments passed to bcvk to-disk (default: none)" @echo " BCVK_REF - Git commit/tag to build bcvk from source (default: $(BCVK_REF))"