From da6b8e8b8159cdfa43236839310755e26bf62bcb Mon Sep 17 00:00:00 2001 From: Srishti Jaiswal <96656007+Srishti-j18@users.noreply.github.com> Date: Mon, 11 May 2026 13:27:50 +0530 Subject: [PATCH] image-builder: run validation with Dind (#13212) * image-builder: run validation locally with BuildKit * upadte checksum * set BuildKit no-process-sandbox flag for CI * run BuildKit with rootless-safe daemon flags in CI * update * updating root logic * fix ci failure for can't enable NoProcessSandbox * switch to stable url * add --oci-worker-no-process-sandbox ci flag in root path * add more support * add validate-docker and validate-single docker --- test-infra/image-builder/Makefile | 24 ++- test-infra/image-builder/README.md | 3 + test-infra/image-builder/cluster.yml | 2 +- .../roles/kubevirt-images/defaults/main.yml | 6 +- .../roles/kubevirt-images/tasks/main.yml | 138 +++++++++++++++++- 5 files changed, 164 insertions(+), 9 deletions(-) diff --git a/test-infra/image-builder/Makefile b/test-infra/image-builder/Makefile index 4e02388ff..7a0d6a23f 100644 --- a/test-infra/image-builder/Makefile +++ b/test-infra/image-builder/Makefile @@ -2,7 +2,27 @@ deploy: ansible-playbook -i hosts.ini -e docker_password=$(docker_password) cluster.yml validate: - ansible-playbook -i hosts.ini -e '{"kubevirt_images_push": false}' cluster.yml + ansible-playbook -i localhost, -c local \ + -e images_dir=$(CURDIR)/.image-builder \ + -e kubevirt_buildkit_output_dir=$(CURDIR)/.image-builder/buildkit-output \ + -e '{"kubevirt_images_push": false, "kubevirt_container_builder": "buildkit", "kubevirt_images_target_host": "localhost"}' \ + cluster.yml validate-single: - ansible-playbook -i hosts.ini -e '{"kubevirt_images_push": false, "kubevirt_images_selected": ["$(image_name)"]}' cluster.yml + ansible-playbook -i localhost, -c local \ + -e images_dir=$(CURDIR)/.image-builder \ + -e kubevirt_buildkit_output_dir=$(CURDIR)/.image-builder/buildkit-output \ + -e '{"kubevirt_images_push": false, "kubevirt_container_builder": "buildkit", "kubevirt_images_target_host": "localhost", "kubevirt_images_selected": ["$(image_name)"]}' \ + cluster.yml + +validate-docker: + ansible-playbook -i localhost, -c local \ + -e images_dir=$(CURDIR)/.image-builder \ + -e '{"kubevirt_images_push": false, "kubevirt_container_builder": "docker", "kubevirt_images_target_host": "localhost"}' \ + cluster.yml + +validate-single-docker: + ansible-playbook -i localhost, -c local \ + -e images_dir=$(CURDIR)/.image-builder \ + -e '{"kubevirt_images_push": false, "kubevirt_container_builder": "docker", "kubevirt_images_target_host": "localhost", "kubevirt_images_selected": ["$(image_name)"]}' \ + cluster.yml diff --git a/test-infra/image-builder/README.md b/test-infra/image-builder/README.md index c45c992f3..dd7cd69d1 100644 --- a/test-infra/image-builder/README.md +++ b/test-infra/image-builder/README.md @@ -65,6 +65,8 @@ cd test-infra/image-builder/ make validate ``` +This validation path runs locally and uses BuildKit, so it does not depend on SSH access to the remote builder host or a Docker daemon. + ### Build only for one image ```bash @@ -76,3 +78,4 @@ make validate-single image_name=ubuntu-2404 - `kubevirt_images_push` (default: `true`): when `false`, skip docker login/push/logout. - `kubevirt_images_selected` (default: `[]`): list of image keys to build. Empty list builds all images. +- `kubevirt_container_builder` (default: `docker`): use `buildkit` for local CI validation without Docker daemon access. diff --git a/test-infra/image-builder/cluster.yml b/test-infra/image-builder/cluster.yml index 4a622ca2f..ed3e64352 100644 --- a/test-infra/image-builder/cluster.yml +++ b/test-infra/image-builder/cluster.yml @@ -1,6 +1,6 @@ --- - name: Build kubevirt images - hosts: image-builder + hosts: "{{ kubevirt_images_target_host | default('image-builder') }}" gather_facts: false roles: - kubevirt-images diff --git a/test-infra/image-builder/roles/kubevirt-images/defaults/main.yml b/test-infra/image-builder/roles/kubevirt-images/defaults/main.yml index 3231f91c9..8a241a93d 100644 --- a/test-infra/image-builder/roles/kubevirt-images/defaults/main.yml +++ b/test-infra/image-builder/roles/kubevirt-images/defaults/main.yml @@ -6,6 +6,8 @@ docker_host: quay.io registry: quay.io/kubespray kubevirt_images_push: true kubevirt_images_selected: [] +kubevirt_container_builder: docker +kubevirt_buildkit_output_dir: "{{ images_dir }}/buildkit-output" images: ubuntu-2004: @@ -24,8 +26,8 @@ images: ubuntu-2404: filename: noble-server-cloudimg-amd64.img - url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img - checksum: sha256:0cf56a2b23b430c350311dbcb9221b64823a5f7a401b5cf6ab4821f2ffdabe76 + url: https://cloud-images.ubuntu.com/noble/20260323/noble-server-cloudimg-amd64.img + checksum: sha256:6e7016f2c9f4d3c00f48789eb6b9043ba2172ccc1b6b1eaf3ed1e29dd3e52bb3 converted: false tag: "latest" diff --git a/test-infra/image-builder/roles/kubevirt-images/tasks/main.yml b/test-infra/image-builder/roles/kubevirt-images/tasks/main.yml index 22fe1da35..37f41119a 100644 --- a/test-infra/image-builder/roles/kubevirt-images/tasks/main.yml +++ b/test-infra/image-builder/roles/kubevirt-images/tasks/main.yml @@ -10,12 +10,63 @@ - kubevirt_images_selected | length == 0 or kubevirt_images_to_build | length > 0 fail_msg: "No matching images found in `images` for `kubevirt_images_selected={{ kubevirt_images_selected }}`" + - name: Validate requested container builder + assert: + that: + - kubevirt_container_builder in ['docker', 'buildkit'] + fail_msg: "Unsupported kubevirt_container_builder={{ kubevirt_container_builder }}" + + - name: Validate BuildKit push mode + assert: + that: + - not (kubevirt_container_builder == 'buildkit' and kubevirt_images_push) + fail_msg: "BuildKit validation currently requires kubevirt_images_push=false" + + - name: Check qemu-img availability + command: qemu-img --version + changed_when: false + + - name: Check Docker availability + command: docker --version + changed_when: false + when: kubevirt_container_builder == 'docker' + + - name: Detect BuildKit daemonless wrapper availability + shell: command -v buildctl-daemonless.sh + args: + executable: /bin/bash + register: kubevirt_buildctl_daemonless_available + changed_when: false + failed_when: false + when: kubevirt_container_builder == 'buildkit' + + - name: Check BuildKit availability + shell: | + set -euo pipefail + if [ "{{ kubevirt_buildctl_daemonless_available.rc | default(1) }}" -eq 0 ]; then + buildctl-daemonless.sh --version + else + buildctl --version + buildkitd --version + fi + args: + executable: /bin/bash + changed_when: false + when: kubevirt_container_builder == 'buildkit' + - name: Create image directory file: state: directory path: "{{ images_dir }}" mode: "0755" + - name: Create buildkit output directory + file: + state: directory + path: "{{ kubevirt_buildkit_output_dir }}" + mode: "0755" + when: kubevirt_container_builder == 'buildkit' + - name: Download images files get_url: url: "{{ item.value.url }}" @@ -56,16 +107,95 @@ - name: Create docker images for each OS command: docker build -t {{ registry }}/vm-{{ item.key }}:{{ item.value.tag }} --build-arg cloud_image="{{ item.key }}.qcow2" {{ images_dir }} loop: "{{ kubevirt_images_to_build }}" + when: kubevirt_container_builder == 'docker' + + - name: Create container images for each OS with BuildKit + shell: | + set -euo pipefail + IMAGE_REF="{{ registry }}/vm-{{ item.key }}:{{ item.value.tag }}" + OUTPUT_TAR="{{ kubevirt_buildkit_output_dir }}/vm-{{ item.key }}-{{ item.value.tag }}.tar" + + # Rootless BuildKit is the CI path; root mode must not use rootless-only flags. + if [ "$(id -u)" -eq 0 ]; then + BUILDKITD_FLAGS="${BUILDKITD_FLAGS:-}" + else + BUILDKITD_FLAGS="${BUILDKITD_FLAGS:---rootless --oci-worker-no-process-sandbox --oci-worker-snapshotter=native}" + fi + + run_buildkit() { + if [ "{{ kubevirt_buildctl_daemonless_available.rc | default(1) }}" -eq 0 ]; then + export BUILDKITD_FLAGS + buildctl-daemonless.sh build \ + --frontend dockerfile.v0 \ + --local context={{ images_dir }} \ + --local dockerfile={{ images_dir }} \ + --opt filename=Dockerfile \ + --opt build-arg:cloud_image={{ item.key }}.qcow2 \ + --output "{{ 'type=image,name=' ~ registry ~ '/vm-' ~ item.key ~ ':' ~ item.value.tag ~ ',push=true' if kubevirt_images_push else 'type=oci,dest=' ~ kubevirt_buildkit_output_dir ~ '/vm-' ~ item.key ~ '-' ~ item.value.tag ~ '.tar' }}" + else + BUILDKIT_ADDR="unix:///tmp/buildkitd-{{ item.key }}.sock" + buildkitd ${BUILDKITD_FLAGS} --addr "${BUILDKIT_ADDR}" >/tmp/buildkitd-{{ item.key }}.log 2>&1 & + buildkitd_pid=$! + + cleanup() { + kill "${buildkitd_pid}" >/dev/null 2>&1 || true + wait "${buildkitd_pid}" >/dev/null 2>&1 || true + } + trap cleanup EXIT + + for _ in $(seq 1 50); do + if buildctl --addr "${BUILDKIT_ADDR}" debug workers >/dev/null 2>&1; then + break + fi + sleep 0.2 + done + + buildctl --addr "${BUILDKIT_ADDR}" build \ + --frontend dockerfile.v0 \ + --local context={{ images_dir }} \ + --local dockerfile={{ images_dir }} \ + --opt filename=Dockerfile \ + --opt build-arg:cloud_image={{ item.key }}.qcow2 \ + --output "{{ 'type=image,name=' ~ registry ~ '/vm-' ~ item.key ~ ':' ~ item.value.tag ~ ',push=true' if kubevirt_images_push else 'type=oci,dest=' ~ kubevirt_buildkit_output_dir ~ '/vm-' ~ item.key ~ '-' ~ item.value.tag ~ '.tar' }}" + fi + } + + if run_buildkit; then + exit 0 + fi + + echo "BuildKit failed in this environment; attempting Docker fallback for {{ item.key }}" >&2 + if ! command -v docker >/dev/null 2>&1; then + echo "Docker fallback unavailable: docker command not found" >&2 + exit 1 + fi + + docker build -t "${IMAGE_REF}" --build-arg cloud_image="{{ item.key }}.qcow2" {{ images_dir }} + {% if kubevirt_images_push %} + docker push "${IMAGE_REF}" + {% else %} + docker save -o "${OUTPUT_TAR}" "${IMAGE_REF}" + {% endif %} + args: + executable: /bin/bash + loop: "{{ kubevirt_images_to_build }}" + when: kubevirt_container_builder == 'buildkit' - name: Docker login command: docker login -u="{{ docker_user }}" -p="{{ docker_password }}" "{{ docker_host }}" - when: kubevirt_images_push + when: + - kubevirt_container_builder == 'docker' + - kubevirt_images_push - name: Docker push image command: docker push {{ registry }}/vm-{{ item.key }}:{{ item.value.tag }} loop: "{{ kubevirt_images_to_build }}" - when: kubevirt_images_push + when: + - kubevirt_container_builder == 'docker' + - kubevirt_images_push - name: Docker logout - command: docker logout -u="{{ docker_user }}" "{{ docker_host }}" - when: kubevirt_images_push + command: docker logout "{{ docker_host }}" + when: + - kubevirt_container_builder == 'docker' + - kubevirt_images_push