CI: rework pipeline: short/extended based on labels (#11324)

* CI: reduce VM resources requests to improve scheduling

* CI: Reduce default jobs; add labels(ci-full/extended) to run more test

* CI: use jobs dependencies instead of stages

* precommit one-job

* CI: Use Kubevirt VM to run Molecule and Vagrant jobs

tests/Makefile

@@ -35,8 +35,7 @@ cleanup-packet:
 
 create-vagrant:
 	vagrant up
-	find / -name vagrant_ansible_inventory
-	cp  /builds/kargo-ci/kubernetes-sigs-kubespray/inventory/sample/vagrant_ansible_inventory $(INVENTORY)
+	cp  $(CI_PROJECT_DIR)/inventory/sample/vagrant_ansible_inventory $(INVENTORY)
 
 delete-vagrant:
 	vagrant destroy -f

tests/cloud_playbooks/roles/packet-ci/defaults/main.yml

@@ -4,14 +4,13 @@
 vm_cpu_cores: 2
 vm_cpu_sockets: 1
 vm_cpu_threads: 2
-vm_memory: 2048Mi
+vm_memory: 2048
 
 # Replace invalid characters so that we can use the branch name in kubernetes labels
 branch_name_sane: "{{ branch | regex_replace('/', '-') }}"
 
 # Request/Limit allocation settings
-
-cpu_allocation_ratio: 0.5
+cpu_allocation_ratio: 0.25
 memory_allocation_ratio: 1
 
 # Default path for inventory

tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2

@@ -4,6 +4,8 @@ kind: VirtualMachine
 metadata:
   name: "instance-{{ vm_id }}"
   namespace: "{{ test_name }}"
+  annotations:
+    kubespray.com/ci.template-path: "tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2"
   labels:
     kubevirt.io/os: {{ cloud_image }}
 spec:
@@ -34,10 +36,10 @@ spec:
             threads: {{ vm_cpu_threads }}
         resources:
           requests:
-            memory: {{ vm_memory * memory_allocation_ratio }}
+            memory: "{{ vm_memory * memory_allocation_ratio }}Mi"
             cpu: {{ vm_cpu_cores * cpu_allocation_ratio }}
           limits:
-            memory: {{ vm_memory }}
+            memory: "{{ vm_memory }}Mi"
             cpu: {{ vm_cpu_cores }}
       networks:
       - name: default

tests/files/packet_almalinux8-calico-ha-ebpf.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: almalinux-8
 mode: ha
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Kubespray settings
 calico_bpf_enabled: true

tests/files/packet_almalinux8-calico-nodelocaldns-secondary.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: almalinux-8
 mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Kubespray settings
 enable_nodelocaldns_secondary: true

tests/files/packet_almalinux8-calico.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: almalinux-8
 mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Kubespray settings
 metrics_server_enabled: true

tests/files/packet_almalinux8-docker.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: almalinux-8
 mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Use docker
 container_manager: docker

tests/files/packet_almalinux8-kube-ovn.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: almalinux-8
 mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Kubespray settings
 kube_network_plugin: kube-ovn

tests/files/packet_rockylinux8-calico.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: rockylinux-8
 mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Kubespray settings
 metrics_server_enabled: true

tests/files/packet_rockylinux9-calico.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: rockylinux-9
 mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Kubespray settings
 metrics_server_enabled: true

tests/files/packet_rockylinux9-cilium.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: rockylinux-9
 mode: default
-vm_memory: 3072Mi
+vm_memory: 3072
 
 # Kubespray settings
 kube_network_plugin: cilium

tests/files/packet_ubuntu22-all-in-one-docker.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: ubuntu-2204
 mode: all-in-one
-vm_memory: 1600Mi
+vm_memory: 1600
 
 # Kubespray settings
 auto_renew_certificates: true

tests/files/packet_ubuntu22-calico-all-in-one.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: ubuntu-2204
 mode: all-in-one
-vm_memory: 1600Mi
+vm_memory: 1600
 
 # Kubespray settings
 auto_renew_certificates: true

tests/files/packet_ubuntu24-all-in-one-docker.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: ubuntu-2404
 mode: all-in-one
-vm_memory: 1600Mi
+vm_memory: 1600
 
 # Kubespray settings
 auto_renew_certificates: true

tests/files/packet_ubuntu24-calico-all-in-one.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: ubuntu-2404
 mode: all-in-one
-vm_memory: 1600Mi
+vm_memory: 1600
 
 # Kubespray settings
 auto_renew_certificates: true

tests/files/packet_ubuntu24-calico-etcd-datastore.yml

@@ -2,7 +2,7 @@
 # Instance settings
 cloud_image: ubuntu-2404
 mode: node-etcd-client
-vm_memory: 1600Mi
+vm_memory: 1600
 
 # Kubespray settings
 auto_renew_certificates: true

tests/files/vagrant_ubuntu20-flannel.rb

@@ -2,7 +2,6 @@ $os = "ubuntu2004"
 
 # For CI we are not worries about data persistence across reboot
 $libvirt_volume_cache = "unsafe"
-
 # Checking for box update can trigger API rate limiting
 # https://www.vagrantup.com/docs/vagrant-cloud/request-limits.html
 $box_check_update = false

tests/files/vagrant_ubuntu20-flannel.yml

@@ -1,3 +1,4 @@
 ---
 # Kubespray settings
 kube_network_plugin: flannel
+ansible_ssh_private_key: .vagrant.d/insecure_private_key

tests/scripts/vagrant-install.sh

@@ -0,0 +1,122 @@
+#!/bin/bash
+
+# install_vagrant() {
+#  sudo apt install vagrant-libvirt vagrant -y
+#  sudo vagrant plugin install vagrant-libvirt
+# }
+
+# prep(){
+# 	sudo apt-get update -y
+# 	sudo apt-get install ca-certificates curl libvirt-daemon-system\
+# 		libvirt-clients qemu-utils qemu-kvm htop atop -y
+
+# 	sudo install -m 0755 -d /etc/apt/keyrings
+# }
+# install_docker() {
+# 	VERSION_STRING=5:26.1.0-1~ubuntu.24.04~noble
+# 	sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+# 	sudo chmod a+r /etc/apt/keyrings/docker.asc
+
+# 	# Add the repository to Apt sources:
+# 	echo \
+# 		"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+# 		$(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+# 			sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+# 	sudo apt-get update -y
+
+# 	sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y
+# }
+# install_docker_auto () {
+# 	 curl -fsSL https://get.docker.com -o get-docker.sh
+# 	 sudo sh ./get-docker.sh --dry-run
+# }
+
+
+
+VAGRANT_VERSION=2.4.1
+VAGRANT_DEFAULT_PROVIDER=libvirt
+VAGRANT_ANSIBLE_TAGS=facts
+LANG=C.UTF-8
+DEBIAN_FRONTEND=noninteractive
+PYTHONDONTWRITEBYTECODE=1
+KUBE_VERSION=1.29.5
+pipeline_install() {
+    cp /etc/apt/sources.list /etc/apt/sources.list."$(date +"%F")"
+    sed -i -e '/^# deb-src.*universe$/s/# //g' /etc/apt/sources.list
+    sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources
+
+    apt update
+    # libssl-dev \
+        # python3-dev \
+        #         # jq \
+        # moreutils \
+        # libvirt-dev \
+        #         # rsync \
+        # git \
+        #                 # htop \
+        # gpg \
+        # atop
+
+        # gnupg2 \
+# software-properties-common
+#
+    apt install --no-install-recommends -y \
+        git \
+        make \
+        python3-pip \
+        sshpass \
+        apt-transport-https \
+        openssh-client \
+        ca-certificates \
+        curl \
+        libfuse2 \
+        unzip \
+        qemu-utils \
+        libvirt-daemon-system \
+        libvirt-clients \
+        qemu-kvm \
+        ebtables libguestfs-tools \
+        ruby-fog-libvirt \
+        libvirt-dev \
+        gcc \
+        build-essential \
+        ruby-libvirt \
+        libxslt-dev libxml2-dev zlib1g-dev \
+        python3-venv python3-full \
+        dnsmasq
+
+    apt-get build-dep -y ruby-libvirt ruby-dev
+    ### VAGRANT ###
+    # apt-get install -y unzip
+    curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}_linux_amd64.zip
+    unzip vagrant_${VAGRANT_VERSION}_linux_amd64.zip
+    mv vagrant /usr/local/bin/vagrant
+    chmod a+x /usr/local/bin/vagrant
+    # ls -la /usr/local/bin/vagrant
+    /usr/local/bin/vagrant plugin install vagrant-libvirt
+    usermod -aG kvm kubespray
+    usermod -aG libvirt kubespray
+
+    ### DOCKER ###
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+    add-apt-repository -y "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+    apt update
+    apt install --no-install-recommends -y docker-ce
+    apt autoremove -y --purge && apt clean && rm -rf /var/lib/apt/lists/* /var/log/*
+
+    ### KUBECTL ###
+    curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+    mv kubectl /usr/local/bin/kubectl
+    chmod a+x /usr/local/bin/kubectl
+    systemctl restart libvirtd
+    # Install Vagrant
+     # apt update -y
+    # echo apt-get install -y unzip libfuse2 vagrant vagrant-libvirt
+    # apt --fix-broken install -y
+    # dpkg --configure -a -y
+
+
+}
+# wrapped up in a function so that we have some protection against only getting
+# half the file during "curl | sh"
+pipeline_install