image-builder: run validation with Dind (#13212)

* image-builder: run validation locally with BuildKit

* upadte checksum

* set BuildKit no-process-sandbox flag for CI

* run BuildKit with rootless-safe daemon flags in CI

* update

* updating root logic

* fix ci failure for can't enable NoProcessSandbox

* switch to stable url

* add --oci-worker-no-process-sandbox ci flag in root path

* add more support

* add validate-docker and validate-single docker

test-infra/image-builder/Makefile

@@ -2,7 +2,27 @@ deploy:
 	ansible-playbook -i hosts.ini -e docker_password=$(docker_password) cluster.yml
 
 validate:
-	ansible-playbook -i hosts.ini -e '{"kubevirt_images_push": false}' cluster.yml
+	ansible-playbook -i localhost, -c local \
+		-e images_dir=$(CURDIR)/.image-builder \
+		-e kubevirt_buildkit_output_dir=$(CURDIR)/.image-builder/buildkit-output \
+		-e '{"kubevirt_images_push": false, "kubevirt_container_builder": "buildkit", "kubevirt_images_target_host": "localhost"}' \
+		cluster.yml
 
 validate-single:
-	ansible-playbook -i hosts.ini -e '{"kubevirt_images_push": false, "kubevirt_images_selected": ["$(image_name)"]}' cluster.yml
+	ansible-playbook -i localhost, -c local \
+		-e images_dir=$(CURDIR)/.image-builder \
+		-e kubevirt_buildkit_output_dir=$(CURDIR)/.image-builder/buildkit-output \
+		-e '{"kubevirt_images_push": false, "kubevirt_container_builder": "buildkit", "kubevirt_images_target_host": "localhost", "kubevirt_images_selected": ["$(image_name)"]}' \
+		cluster.yml
+
+validate-docker:
+	ansible-playbook -i localhost, -c local \
+		-e images_dir=$(CURDIR)/.image-builder \
+		-e '{"kubevirt_images_push": false, "kubevirt_container_builder": "docker", "kubevirt_images_target_host": "localhost"}' \
+		cluster.yml
+
+validate-single-docker:
+	ansible-playbook -i localhost, -c local \
+		-e images_dir=$(CURDIR)/.image-builder \
+		-e '{"kubevirt_images_push": false, "kubevirt_container_builder": "docker", "kubevirt_images_target_host": "localhost", "kubevirt_images_selected": ["$(image_name)"]}' \
+		cluster.yml

test-infra/image-builder/README.md

@@ -65,6 +65,8 @@ cd test-infra/image-builder/
 make validate
 ```
 
+This validation path runs locally and uses BuildKit, so it does not depend on SSH access to the remote builder host or a Docker daemon.
+
 ### Build only for one image
 
 ```bash
@@ -76,3 +78,4 @@ make validate-single image_name=ubuntu-2404
 
 - `kubevirt_images_push` (default: `true`): when `false`, skip docker login/push/logout.
 - `kubevirt_images_selected` (default: `[]`): list of image keys to build. Empty list builds all images.
+- `kubevirt_container_builder` (default: `docker`): use `buildkit` for local CI validation without Docker daemon access.

test-infra/image-builder/cluster.yml

@@ -1,6 +1,6 @@
 ---
 - name: Build kubevirt images
-  hosts: image-builder
+  hosts: "{{ kubevirt_images_target_host | default('image-builder') }}"
   gather_facts: false
   roles:
     - kubevirt-images

test-infra/image-builder/roles/kubevirt-images/defaults/main.yml

@@ -6,6 +6,8 @@ docker_host: quay.io
 registry: quay.io/kubespray
 kubevirt_images_push: true
 kubevirt_images_selected: []
+kubevirt_container_builder: docker
+kubevirt_buildkit_output_dir: "{{ images_dir }}/buildkit-output"
 
 images:
   ubuntu-2004:
@@ -24,8 +26,8 @@ images:
 
   ubuntu-2404:
     filename: noble-server-cloudimg-amd64.img
-    url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
-    checksum: sha256:0cf56a2b23b430c350311dbcb9221b64823a5f7a401b5cf6ab4821f2ffdabe76
+    url: https://cloud-images.ubuntu.com/noble/20260323/noble-server-cloudimg-amd64.img
+    checksum: sha256:6e7016f2c9f4d3c00f48789eb6b9043ba2172ccc1b6b1eaf3ed1e29dd3e52bb3
     converted: false
     tag: "latest"
 

test-infra/image-builder/roles/kubevirt-images/tasks/main.yml

@@ -10,12 +10,63 @@
           - kubevirt_images_selected | length == 0 or kubevirt_images_to_build | length > 0
         fail_msg: "No matching images found in `images` for `kubevirt_images_selected={{ kubevirt_images_selected }}`"
 
+    - name: Validate requested container builder
+      assert:
+        that:
+          - kubevirt_container_builder in ['docker', 'buildkit']
+        fail_msg: "Unsupported kubevirt_container_builder={{ kubevirt_container_builder }}"
+
+    - name: Validate BuildKit push mode
+      assert:
+        that:
+          - not (kubevirt_container_builder == 'buildkit' and kubevirt_images_push)
+        fail_msg: "BuildKit validation currently requires kubevirt_images_push=false"
+
+    - name: Check qemu-img availability
+      command: qemu-img --version
+      changed_when: false
+
+    - name: Check Docker availability
+      command: docker --version
+      changed_when: false
+      when: kubevirt_container_builder == 'docker'
+
+    - name: Detect BuildKit daemonless wrapper availability
+      shell: command -v buildctl-daemonless.sh
+      args:
+        executable: /bin/bash
+      register: kubevirt_buildctl_daemonless_available
+      changed_when: false
+      failed_when: false
+      when: kubevirt_container_builder == 'buildkit'
+
+    - name: Check BuildKit availability
+      shell: |
+        set -euo pipefail
+        if [ "{{ kubevirt_buildctl_daemonless_available.rc | default(1) }}" -eq 0 ]; then
+          buildctl-daemonless.sh --version
+        else
+          buildctl --version
+          buildkitd --version
+        fi
+      args:
+        executable: /bin/bash
+      changed_when: false
+      when: kubevirt_container_builder == 'buildkit'
+
     - name: Create image directory
       file:
         state: directory
         path: "{{ images_dir }}"
         mode: "0755"
 
+    - name: Create buildkit output directory
+      file:
+        state: directory
+        path: "{{ kubevirt_buildkit_output_dir }}"
+        mode: "0755"
+      when: kubevirt_container_builder == 'buildkit'
+
     - name: Download images files
       get_url:
         url: "{{ item.value.url }}"
@@ -56,16 +107,95 @@
     - name: Create docker images for each OS
       command: docker build -t {{ registry }}/vm-{{ item.key }}:{{ item.value.tag }} --build-arg cloud_image="{{ item.key }}.qcow2" {{ images_dir }}
       loop: "{{ kubevirt_images_to_build }}"
+      when: kubevirt_container_builder == 'docker'
+
+    - name: Create container images for each OS with BuildKit
+      shell: |
+        set -euo pipefail
+        IMAGE_REF="{{ registry }}/vm-{{ item.key }}:{{ item.value.tag }}"
+        OUTPUT_TAR="{{ kubevirt_buildkit_output_dir }}/vm-{{ item.key }}-{{ item.value.tag }}.tar"
+
+        # Rootless BuildKit is the CI path; root mode must not use rootless-only flags.
+        if [ "$(id -u)" -eq 0 ]; then
+          BUILDKITD_FLAGS="${BUILDKITD_FLAGS:-}"
+        else
+          BUILDKITD_FLAGS="${BUILDKITD_FLAGS:---rootless --oci-worker-no-process-sandbox --oci-worker-snapshotter=native}"
+        fi
+
+        run_buildkit() {
+        if [ "{{ kubevirt_buildctl_daemonless_available.rc | default(1) }}" -eq 0 ]; then
+          export BUILDKITD_FLAGS
+          buildctl-daemonless.sh build \
+            --frontend dockerfile.v0 \
+            --local context={{ images_dir }} \
+            --local dockerfile={{ images_dir }} \
+            --opt filename=Dockerfile \
+            --opt build-arg:cloud_image={{ item.key }}.qcow2 \
+            --output "{{ 'type=image,name=' ~ registry ~ '/vm-' ~ item.key ~ ':' ~ item.value.tag ~ ',push=true' if kubevirt_images_push else 'type=oci,dest=' ~ kubevirt_buildkit_output_dir ~ '/vm-' ~ item.key ~ '-' ~ item.value.tag ~ '.tar' }}"
+        else
+          BUILDKIT_ADDR="unix:///tmp/buildkitd-{{ item.key }}.sock"
+          buildkitd ${BUILDKITD_FLAGS} --addr "${BUILDKIT_ADDR}" >/tmp/buildkitd-{{ item.key }}.log 2>&1 &
+          buildkitd_pid=$!
+
+          cleanup() {
+            kill "${buildkitd_pid}" >/dev/null 2>&1 || true
+            wait "${buildkitd_pid}" >/dev/null 2>&1 || true
+          }
+          trap cleanup EXIT
+
+          for _ in $(seq 1 50); do
+            if buildctl --addr "${BUILDKIT_ADDR}" debug workers >/dev/null 2>&1; then
+              break
+            fi
+            sleep 0.2
+          done
+
+          buildctl --addr "${BUILDKIT_ADDR}" build \
+            --frontend dockerfile.v0 \
+            --local context={{ images_dir }} \
+            --local dockerfile={{ images_dir }} \
+            --opt filename=Dockerfile \
+            --opt build-arg:cloud_image={{ item.key }}.qcow2 \
+            --output "{{ 'type=image,name=' ~ registry ~ '/vm-' ~ item.key ~ ':' ~ item.value.tag ~ ',push=true' if kubevirt_images_push else 'type=oci,dest=' ~ kubevirt_buildkit_output_dir ~ '/vm-' ~ item.key ~ '-' ~ item.value.tag ~ '.tar' }}"
+        fi
+        }
+
+        if run_buildkit; then
+          exit 0
+        fi
+
+        echo "BuildKit failed in this environment; attempting Docker fallback for {{ item.key }}" >&2
+        if ! command -v docker >/dev/null 2>&1; then
+          echo "Docker fallback unavailable: docker command not found" >&2
+          exit 1
+        fi
+
+        docker build -t "${IMAGE_REF}" --build-arg cloud_image="{{ item.key }}.qcow2" {{ images_dir }}
+        {% if kubevirt_images_push %}
+        docker push "${IMAGE_REF}"
+        {% else %}
+        docker save -o "${OUTPUT_TAR}" "${IMAGE_REF}"
+        {% endif %}
+      args:
+        executable: /bin/bash
+      loop: "{{ kubevirt_images_to_build }}"
+      when: kubevirt_container_builder == 'buildkit'
 
     - name: Docker login
       command: docker login -u="{{ docker_user }}" -p="{{ docker_password }}" "{{ docker_host }}"
-      when: kubevirt_images_push
+      when:
+        - kubevirt_container_builder == 'docker'
+        - kubevirt_images_push
 
     - name: Docker push image
       command: docker push {{ registry }}/vm-{{ item.key }}:{{ item.value.tag }}
       loop: "{{ kubevirt_images_to_build }}"
-      when: kubevirt_images_push
+      when:
+        - kubevirt_container_builder == 'docker'
+        - kubevirt_images_push
 
     - name: Docker logout
-      command: docker logout -u="{{ docker_user }}" "{{ docker_host }}"
-      when: kubevirt_images_push
+      command: docker logout "{{ docker_host }}"
+      when:
+        - kubevirt_container_builder == 'docker'
+        - kubevirt_images_push