Patch versions updates

When joining a control plane node and "upgrading" the cluster setup (for
example, to update etcd addresses after adding a new etcd) in the same
playbook run, the node can take a bit of time to become ready after
joining.
This triggers a kubeadm preflight check (ControlPlaneNodesReady) in
kubeadm upgrade, which is run directly after the join tasks.

Add a configurable wait for the control plane node to become Ready to
fix this race condition.

Co-authored-by: Max Gautier <mg@max.gautier.name>

.ansible-lint

@@ -11,12 +11,10 @@ skip_list:
   # (Disabled in June 2021)
   - 'role-name'
 
-  # [var-naming]
+  # [var-naming] "defaults/main.yml" File defines variable 'apiVersion' that violates variable naming standards
   # In Kubespray we use variables that use camelCase to match their k8s counterparts
   # (Disabled in June 2021)
-  - 'var-naming[pattern]'
+  - 'var-naming'
-  # Variables names from within roles in kubespray don't need role name as a prefix
-  - 'var-naming[no-role-prefix]'
 
   # [fqcn-builtins]
   # Roles in kubespray don't need fully qualified collection names

.github/ISSUE_TEMPLATE/bug-report.yaml

@@ -108,6 +108,7 @@ body:
         - meta
         - multus
         - ovn4nfv
+        - weave
     validations:
       required: true
 

.github/workflows/auto-label-os.yml

@@ -13,7 +13,7 @@ jobs:
       issues: write
 
     steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
 
       - name: Parse issue form
         uses: stefanbuck/github-issue-parser@2ea9b35a8c584529ed00891a8f7e41dc46d0441e
@@ -22,7 +22,7 @@ jobs:
           template-path: .github/ISSUE_TEMPLATE/bug-report.yaml
 
       - name: Set labels based on OS field
-        uses: redhat-plumbers-in-action/advanced-issue-labeler@e38e6809c5420d038eed380d49ee9a6ca7c92dbf
+        uses: redhat-plumbers-in-action/advanced-issue-labeler@39087a4b30cb98d57f25f34d617a6af8163c17d9
         with:
           issue-form: ${{ steps.issue-parser.outputs.jsonString }}
           section: os

.github/workflows/upgrade-patch-versions-schedule.yml

@@ -20,7 +20,7 @@ jobs:
           query get_release_branches($owner:String!, $name:String!) {
             repository(owner:$owner, name:$name) {
               refs(refPrefix: "refs/heads/",
-                   first: 1, # TODO increment once we have release branch with the new checksums format
+                   first: 0, # TODO increment once we have release branch with the new checksums format
                    query: "release-",
                    orderBy: {
                      field: ALPHABETICAL,

.github/workflows/upgrade-patch-versions.yml

@@ -11,10 +11,10 @@ jobs:
   update-patch-versions:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
       with:
         ref: ${{ inputs.branch }}
-    - uses: actions/setup-python@v6
+    - uses: actions/setup-python@v5
       with:
         python-version: '3.13'
         cache: 'pip'

.gitlab-ci.yml

@@ -1,9 +1,9 @@
 ---
 stages:
-  - build # build docker image used in most other jobs
+  - build
-  - test # unit tests
+  - test
-  - deploy-part1 # kubespray runs - common setup
+  - deploy-part1
-  - deploy-extended # kubespray runs - rarer or costlier (to test) setups
+  - deploy-extended
 
 variables:
   FAILFASTCI_NAMESPACE: 'kargo-ci'
@@ -24,7 +24,6 @@ variables:
   ANSIBLE_REMOTE_USER: kubespray
   ANSIBLE_PRIVATE_KEY_FILE: /tmp/id_rsa
   ANSIBLE_INVENTORY: /tmp/inventory
-  ANSIBLE_STDOUT_CALLBACK: "default"
   RESET_CHECK: "false"
   REMOVE_NODE_CHECK: "false"
   UPGRADE_TEST: "false"
@@ -32,7 +31,7 @@ variables:
   ANSIBLE_VERBOSITY: 2
   RECOVER_CONTROL_PLANE_TEST: "false"
   RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
-  OPENTOFU_VERSION: v1.9.1
+  TF_VERSION: 1.3.7
   PIPELINE_IMAGE: "$CI_REGISTRY_IMAGE/pipeline:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"
 
 before_script:
@@ -49,6 +48,8 @@ before_script:
       - cluster-dump/
   needs:
     - pipeline-image
+  variables:
+    ANSIBLE_STDOUT_CALLBACK: "debug"
 
 .job-moderated:
   extends: .job

.gitlab-ci/build.yml

@@ -7,24 +7,23 @@ pipeline-image:
   tags:
     - ffci
   stage: build
-  image: moby/buildkit:rootless
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: ['']
   variables:
-    BUILDKITD_FLAGS: --oci-worker-no-process-sandbox
+    GODEBUG: "http2client=0"
-    CACHE_IMAGE: $CI_REGISTRY_IMAGE/pipeline:cache
   # TODO: remove the override
   # currently rebase.sh depends on bash (not available in the kaniko image)
   # once we have a simpler rebase (which should be easy if the target branch ref is available as variable
   # we'll be able to rebase here as well hopefully
-  before_script:
+  before_script: []
-    - mkdir -p ~/.docker
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > ~/.docker/config.json
   script:
-    - |
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json
-      buildctl-daemonless.sh build \
+    - /kaniko/executor --cache=true
-        --frontend dockerfile.v0 \
+                       --cache-dir=image-cache
-        --local context=$CI_PROJECT_DIR \
+                       --context $CI_PROJECT_DIR
-        --local dockerfile=$CI_PROJECT_DIR \
+                       --dockerfile $CI_PROJECT_DIR/pipeline.Dockerfile
-        --opt filename=pipeline.Dockerfile \
+                       --label 'git-branch'=$CI_COMMIT_REF_SLUG
-        --export-cache type=registry,ref=$CACHE_IMAGE \
+                       --label 'git-tag=$CI_COMMIT_TAG'
-        --import-cache type=registry,ref=$CACHE_IMAGE \
+                       --destination $PIPELINE_IMAGE
-        --output type=image,name=$PIPELINE_IMAGE,push=true
+                       --log-timestamp=true

.gitlab-ci/kubevirt.yml

@@ -34,33 +34,34 @@ pr:
   parallel:
     matrix:
       - TESTCASE:
+          - almalinux8-calico
           - almalinux9-crio
           - almalinux9-kube-ovn
           - debian11-calico-collection
           - debian11-macvlan
           - debian12-cilium
-          - debian13-cilium
           - fedora39-kube-router
+            # FIXME: this test if broken (perma-failing)
           - openeuler24-calico
+          - opensuse15-6-calico
+          - rockylinux8-calico
           - rockylinux9-cilium
+          - ubuntu20-calico-all-in-one-hardening
+          - ubuntu20-cilium-sep
+          - ubuntu20-flannel-collection
+          - ubuntu20-kube-router-sep
+          - ubuntu20-kube-router-svc-proxy
           - ubuntu22-calico-all-in-one
           - ubuntu22-calico-all-in-one-upgrade
           - ubuntu24-calico-etcd-datastore
-          - ubuntu24-calico-all-in-one-hardening
-          - ubuntu24-cilium-sep
-          - ubuntu24-flannel-collection
-          - ubuntu24-kube-router-sep
-          - ubuntu24-kube-router-svc-proxy
           - ubuntu24-ha-separate-etcd
-          - flatcar4081-calico
-          - fedora40-flannel-crio-collection-scale
 
-# The ubuntu24-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
+# The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
-ubuntu24-calico-all-in-one:
+ubuntu20-calico-all-in-one:
   stage: deploy-part1
   extends: .kubevirt
   variables:
-    TESTCASE: ubuntu24-calico-all-in-one
+    TESTCASE: ubuntu20-calico-all-in-one
   rules:
     - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
       when: on_success
@@ -90,9 +91,10 @@ pr_full:
           - debian12-custom-cni-helm
           - fedora39-calico-swap-selinux
           - fedora39-crio
-          - ubuntu24-calico-ha-wireguard
+          - ubuntu20-all-in-one-docker
-          - ubuntu24-flannel-ha
+          - ubuntu20-calico-ha-wireguard
-          - ubuntu24-flannel-ha-once
+          - ubuntu20-flannel-ha
+          - ubuntu20-flannel-ha-once
 
 # Need an update of the container image to use schema v2
 # update: quay.io/kubespray/vm-amazon-linux-2:latest
@@ -125,13 +127,13 @@ pr_extended:
           - debian11-docker
           - debian12-calico
           - debian12-docker
-          - debian13-calico
+          - opensuse15-6-docker-cilium
           - rockylinux9-calico
+          - ubuntu20-calico-etcd-kubeadm
+          - ubuntu20-flannel
           - ubuntu22-all-in-one-docker
           - ubuntu24-all-in-one-docker
           - ubuntu24-calico-all-in-one
-          - ubuntu24-calico-etcd-kubeadm
-          - ubuntu24-flannel
 
 # TODO: migrate to pr-full, fix the broken ones
 periodic:
@@ -148,6 +150,6 @@ periodic:
           - debian12-cilium-svc-proxy
           - fedora39-calico-selinux
           - fedora40-docker-calico
-          - ubuntu24-calico-etcd-kubeadm-upgrade-ha
+          - ubuntu20-calico-etcd-kubeadm-upgrade-ha
-          - ubuntu24-calico-ha-recover
+          - ubuntu20-calico-ha-recover
-          - ubuntu24-calico-ha-recover-noquorum
+          - ubuntu20-calico-ha-recover-noquorum

.gitlab-ci/lint.yml

@@ -6,7 +6,6 @@ pre-commit:
   image: 'ghcr.io/pre-commit-ci/runner-image@sha256:fe01a6ec51b298412990b88627c3973b1146c7304f930f469bafa29ba60bcde9'
   variables:
     PRE_COMMIT_HOME: ${CI_PROJECT_DIR}/.cache/pre-commit
-    ANSIBLE_STDOUT_CALLBACK: default
   script:
   - pre-commit run --all-files --show-diff-on-failure
   cache:
@@ -24,3 +23,4 @@ vagrant-validate:
     VAGRANT_VERSION: 2.3.7
   script:
   - ./tests/scripts/vagrant-validate.sh
+  except: ['triggers', 'master']

.gitlab-ci/molecule.yml

@@ -34,8 +34,6 @@ molecule:
       - container-engine/cri-dockerd
       - container-engine/containerd
       - container-engine/cri-o
-      - container-engine/gvisor
-      - container-engine/youki
       - adduser
       - bastion-ssh-config
       - bootstrap_os
@@ -53,3 +51,5 @@ molecule_full:
     - ROLE:
       # FIXME : tests below are perma-failing
       - container-engine/kata-containers
+      - container-engine/gvisor
+      - container-engine/youki

.gitlab-ci/terraform.yml

@@ -13,18 +13,18 @@
   before_script:
     - ./tests/scripts/rebase.sh
     - mkdir -p cluster-dump $ANSIBLE_INVENTORY
-    - ./tests/scripts/opentofu_install.sh
+    - ./tests/scripts/terraform_install.sh
     - cp contrib/terraform/$PROVIDER/sample-inventory/cluster.tfvars .
     - ln -rs -t $ANSIBLE_INVENTORY contrib/terraform/$PROVIDER/hosts
-    - tofu -chdir="contrib/terraform/$PROVIDER" init
+    - terraform -chdir="contrib/terraform/$PROVIDER" init
 
 terraform_validate:
   extends: .terraform_install
   tags: [ffci]
   only: ['master', /^pr-.*$/]
   script:
-    - tofu -chdir="contrib/terraform/$PROVIDER" validate
+    - terraform -chdir="contrib/terraform/$PROVIDER" validate
-    - tofu -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
+    - terraform -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
   stage: test
   needs:
     - pipeline-image
@@ -58,11 +58,11 @@ terraform_validate:
     - mkdir -p contrib/terraform/$PROVIDER/group_vars
     # Random subnet to avoid routing conflicts
     - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
-    - tofu -chdir="contrib/terraform/$PROVIDER" apply -auto-approve -parallelism=1
+    - terraform -chdir="contrib/terraform/$PROVIDER" apply -auto-approve -parallelism=1
     - tests/scripts/testcases_run.sh
   after_script:
     # Cleanup regardless of exit code
-    - tofu -chdir="contrib/terraform/$PROVIDER" destroy -auto-approve
+    - terraform -chdir="contrib/terraform/$PROVIDER" destroy -auto-approve
 
 # Elastx is generously donating resources for Kubespray on Openstack CI
 # Contacts: @gix @bl0m1

.gitlab-ci/vagrant.yml

@@ -40,8 +40,6 @@ vagrant:
       when: on_success
     - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
       when: on_success
-    - when: manual
-      allow_failure: true
   parallel:
     matrix:
       - TESTCASE:

CONTRIBUTING.md

@@ -40,7 +40,7 @@ Vagrant with VirtualBox or libvirt driver helps you to quickly spin test cluster
 2. The [repo owners](OWNERS) will respond to your issue promptly.
 3. Fork the desired repo, develop and test your code changes.
 4. Install [pre-commit](https://pre-commit.com) and install it in your development repo.
-5. Address any pre-commit validation failures.
+5. Addess any pre-commit validation failures.
 6. Sign the CNCF CLA (<https://git.k8s.io/community/CLA.md#the-contributor-license-agreement>)
 7. Submit a pull request.
 8. Work with the reviewers on their suggestions.

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1
 
-# Use immutable image tags rather than mutable tags (like ubuntu:22.04)
+# Use imutable image tags rather than mutable tags (like ubuntu:22.04)
 FROM ubuntu:22.04@sha256:149d67e29f765f4db62aa52161009e99e389544e25a8f43c8c89d4a445a7ca37
 
 # Some tools like yamllint need this
@@ -35,8 +35,8 @@ RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 RUN OS_ARCHITECTURE=$(dpkg --print-architecture) \
-    && curl -L "https://dl.k8s.io/release/v1.33.7/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
+    && curl -L "https://dl.k8s.io/release/v1.32.11/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
-    && echo "$(curl -L "https://dl.k8s.io/release/v1.33.7/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
+    && echo "$(curl -L "https://dl.k8s.io/release/v1.32.11/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
     && chmod a+x /usr/local/bin/kubectl
 
 COPY *.yml ./

OWNERS_ALIASES

@@ -8,6 +8,7 @@ aliases:
   kubespray-reviewers:
     - cyclinder
     - erikjiang
+    - mrfreezeex
     - mzaian
     - tico88612
     - vannten

README.md

@@ -22,7 +22,7 @@ Ensure you have installed Docker then
 ```ShellSession
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
   --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.29.0 bash
+  quay.io/kubespray/kubespray:v2.27.0 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```
@@ -87,8 +87,8 @@ vagrant up
 ## Supported Linux Distributions
 
 - **Flatcar Container Linux by Kinvolk**
-- **Debian** Bookworm, Bullseye, Trixie
+- **Debian** Bookworm, Bullseye
-- **Ubuntu** 22.04, 24.04
+- **Ubuntu** 20.04, 22.04, 24.04
 - **CentOS/RHEL** [8, 9](docs/operating_systems/rhel.md#rhel-8)
 - **Fedora** 39, 40
 - **Fedora CoreOS** (see [fcos Note](docs/operating_systems/fcos.md))
@@ -111,26 +111,27 @@ Note:
 <!-- BEGIN ANSIBLE MANAGED BLOCK -->
 
 - Core
-  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.33.7
+  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.32.11
   - [etcd](https://github.com/etcd-io/etcd) 3.5.26
-  - [docker](https://www.docker.com/) 28.3
+  - [docker](https://www.docker.com/) 28.0
-  - [containerd](https://containerd.io/) 2.1.6
+  - [containerd](https://containerd.io/) 2.0.7
-  - [cri-o](http://cri-o.io/) 1.33.8 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
+  - [cri-o](http://cri-o.io/) 1.32.0 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
-  - [cni-plugins](https://github.com/containernetworking/plugins) 1.8.0
+  - [cni-plugins](https://github.com/containernetworking/plugins) 1.4.1
-  - [calico](https://github.com/projectcalico/calico) 3.30.6
+  - [calico](https://github.com/projectcalico/calico) 3.29.7
-  - [cilium](https://github.com/cilium/cilium) 1.18.5
+  - [cilium](https://github.com/cilium/cilium) 1.17.9
-  - [flannel](https://github.com/flannel-io/flannel) 0.27.3
+  - [flannel](https://github.com/flannel-io/flannel) 0.22.0
   - [kube-ovn](https://github.com/alauda/kube-ovn) 1.12.21
   - [kube-router](https://github.com/cloudnativelabs/kube-router) 2.1.1
-  - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) 4.2.2
+  - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) 4.1.0
+  - [weave](https://github.com/rajch/weave) 2.8.7
   - [kube-vip](https://github.com/kube-vip/kube-vip) 0.8.0
 - Application
   - [cert-manager](https://github.com/jetstack/cert-manager) 1.15.3
-  - [coredns](https://github.com/coredns/coredns) 1.12.0
+  - [coredns](https://github.com/coredns/coredns) 1.11.3
-  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) 1.13.3
+  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) 1.12.1
   - [argocd](https://argoproj.github.io/) 2.14.5
-  - [helm](https://helm.sh/) 3.18.4
+  - [helm](https://helm.sh/) 3.16.4
   - [metallb](https://metallb.universe.tf/) 0.13.9
   - [registry](https://github.com/distribution/distribution) 2.8.1
 - Storage Plugin
@@ -138,7 +139,7 @@ Note:
   - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) 1.10.0
   - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) 1.30.0
   - [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) 1.9.2
-  - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) 0.0.32
+  - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) 0.0.24
   - [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) 2.5.0
   - [node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) 0.16.4
 
@@ -182,6 +183,9 @@ You can choose among ten network plugins. (default: `calico`, except Vagrant use
 
 - [cilium](http://docs.cilium.io/en/latest/): layer 3/4 networking (as well as layer 7 to protect and secure application protocols), supports dynamic insertion of BPF bytecode into the Linux kernel to implement security services, networking and visibility logic.
 
+- [weave](docs/CNI/weave.md): Weave is a lightweight container overlay network that doesn't require an external K/V database cluster.
+    (Please refer to `weave` [troubleshooting documentation](https://www.weave.works/docs/net/latest/troubleshooting/)).
+
 - [kube-ovn](docs/CNI/kube-ovn.md): Kube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It offers an advanced Container Network Fabric for Enterprises.
 
 - [kube-router](docs/CNI/kube-router.md): Kube-router is a L3 CNI for Kubernetes networking aiming to provide operational

Vagrantfile

@@ -4,8 +4,6 @@
 # For help on using kubespray with vagrant, check out docs/developers/vagrant.md
 
 require 'fileutils'
-require 'ipaddr'
-require 'socket'
 
 Vagrant.require_version ">= 2.0.0"
 
@@ -101,33 +99,6 @@ $extra_vars ||= {}
 
 host_vars = {}
 
-def collect_networks(subnet, subnet_ipv6)
-  Socket.getifaddrs.filter_map do |iface|
-    next unless iface&.netmask&.ip_address && iface.addr
-
-    is_ipv6 = iface.addr.ipv6?
-    ip      = IPAddr.new(iface.addr.ip_address.split('%').first)
-    ip_test = is_ipv6 ? IPAddr.new("#{subnet_ipv6}::0") : IPAddr.new("#{subnet}.0")
-
-    prefix  = IPAddr.new(iface.netmask.ip_address).to_i.to_s(2).count('1')
-    network = ip.mask(prefix)
-
-    [IPAddr.new("#{network}/#{prefix}"), ip_test]
-  end
-end
-
-def subnet_in_use?(network_ips)
-  network_ips.any? { |net, test_ip| net.include?(test_ip) && test_ip != net }
-end
-
-network_ips = collect_networks($subnet, $subnet_ipv6)
-
-if subnet_in_use?(network_ips)
-  puts "Invalid subnet provided, subnet is already in use: #{$subnet}.0"
-  puts "Subnets in use: #{network_ips.inspect}"
-  exit 1
-end
-
 # throw error if os is not supported
 if ! SUPPORTED_OS.key?($os)
   puts "Unsupported OS: #{$os}"

ansible.cfg

@@ -15,7 +15,7 @@ timeout = 300
 stdout_callback = default
 display_skipped_hosts = no
 library = ./library
-callbacks_enabled = profile_tasks
+callbacks_enabled = profile_tasks,ara_default
 roles_path = roles:$VIRTUAL_ENV/usr/local/share/kubespray/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:/usr/share/kubespray/roles
 deprecation_warnings=False
 inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo, .creds, .gpg

contrib/offline/manage-offline-container-images.sh

@@ -36,7 +36,7 @@ function create_container_image_tar() {
 	mkdir  ${IMAGE_DIR}
 	cd     ${IMAGE_DIR}
 
-	sudo --preserve-env=http_proxy,https_proxy,no_proxy ${runtime} pull registry:latest
+	sudo ${runtime} pull registry:latest
 	sudo ${runtime} save -o registry-latest.tar registry:latest
 
 	while read -r image
@@ -45,7 +45,7 @@ function create_container_image_tar() {
 		set +e
 		for step in $(seq 1 ${RETRY_COUNT})
 		do
-			sudo --preserve-env=http_proxy,https_proxy,no_proxy ${runtime} pull ${image}
+			sudo ${runtime} pull ${image}
 			if [ $? -eq 0 ]; then
 				break
 			fi

contrib/offline/manage-offline-files.sh

@@ -41,7 +41,7 @@ fi
 
 sudo "${runtime}" container inspect nginx >/dev/null 2>&1
 if [ $? -ne 0 ]; then
-    sudo --preserve-env=http_proxy,https_proxy,no_proxy "${runtime}" run \
+    sudo "${runtime}" run \
         --restart=always -d -p ${NGINX_PORT}:80 \
         --volume "${OFFLINE_FILES_DIR}":/usr/share/nginx/html/download \
         --volume "${CURRENT_DIR}"/nginx.conf:/etc/nginx/nginx.conf \

contrib/terraform/openstack/README.md

@@ -624,7 +624,7 @@ Edit `inventory/$CLUSTER/group_vars/k8s_cluster/k8s_cluster.yml`:
   - **calico** requires [configuring OpenStack Neutron ports](/docs/cloud_controllers/openstack.md) to allow service and pod subnets
 
 ```yml
-# Choose network plugin (calico or flannel)
+# Choose network plugin (calico, weave or flannel)
 # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
 kube_network_plugin: flannel
 ```

docs/CNI/calico.md

@@ -180,7 +180,7 @@ calico_group_id=rr1
 The inventory above will deploy the following topology assuming that calico's
 `global_as_num` is set to `65400`:
 
-![Image](../figures/kubespray-calico-rr.png?raw=true)
+![Image](figures/kubespray-calico-rr.png?raw=true)
 
 ### Optional : Define default endpoint to host action
 

docs/CNI/cilium.md

@@ -237,7 +237,7 @@ cilium_operator_extra_volume_mounts:
 ## Choose Cilium version
 
 ```yml
-cilium_version: "1.18.5"
+cilium_version: "1.17.9"
 ```
 
 ## Add variable to config

docs/CNI/weave.md

@@ -0,0 +1,79 @@
+# Weave
+
+Weave 2.0.1 is supported by kubespray
+
+Weave uses [**consensus**](https://www.weave.works/docs/net/latest/ipam/##consensus) mode (default mode) and [**seed**](https://www.weave.works/docs/net/latest/ipam/#seed) mode.
+
+`Consensus` mode is best to use on static size cluster and `seed` mode is best to use on dynamic size cluster
+
+Weave encryption is supported for all communication
+
+* To use Weave encryption, specify a strong password (if no password, no encryption)
+
+```ShellSession
+# In file ./inventory/sample/group_vars/k8s_cluster.yml
+weave_password: EnterPasswordHere
+```
+
+This password is used to set an environment variable inside weave container.
+
+Weave is deployed by kubespray using a daemonSet
+
+* Check the status of Weave containers
+
+```ShellSession
+# From client
+kubectl -n kube-system get pods | grep weave
+# output
+weave-net-50wd2                       2/2       Running   0          2m
+weave-net-js9rb                       2/2       Running   0          2m
+```
+
+There must be as many pods as nodes (here kubernetes have 2 nodes so there are 2 weave pods).
+
+* Check status of weave (connection,encryption ...) for each node
+
+```ShellSession
+# On nodes
+curl http://127.0.0.1:6784/status
+# output on node1
+Version: 2.0.1 (up to date; next check at 2017/08/01 13:51:34)
+
+        Service: router
+       Protocol: weave 1..2
+           Name: fa:16:3e:b3:d6:b2(node1)
+     Encryption: enabled
+  PeerDiscovery: enabled
+        Targets: 2
+    Connections: 2 (1 established, 1 failed)
+          Peers: 2 (with 2 established connections)
+ TrustedSubnets: none
+
+        Service: ipam
+         Status: ready
+          Range: 10.233.64.0/18
+  DefaultSubnet: 10.233.64.0/18
+```
+
+* Check parameters of weave for each node
+
+```ShellSession
+# On nodes
+ps -aux | grep weaver
+# output on node1 (here its use seed mode)
+root      8559  0.2  3.0 365280 62700 ?        Sl   08:25   0:00 /home/weave/weaver --name=fa:16:3e:b3:d6:b2 --port=6783 --datapath=datapath --host-root=/host --http-addr=127.0.0.1:6784 --status-addr=0.0.0.0:6782 --docker-api= --no-dns --db-prefix=/weavedb/weave-net --ipalloc-range=10.233.64.0/18 --nickname=node1 --ipalloc-init seed=fa:16:3e:b3:d6:b2,fa:16:3e:f0:50:53 --conn-limit=30 --expect-npc 192.168.208.28 192.168.208.19
+```
+
+## Consensus mode (default mode)
+
+This mode is best to use on static size cluster
+
+### Seed mode
+
+This mode is best to use on dynamic size cluster
+
+The seed mode also allows multi-clouds and hybrid on-premise/cloud clusters deployment.
+
+* Switch from consensus mode to seed/Observation mode
+
+See [weave ipam documentation](https://www.weave.works/docs/net/latest/tasks/ipam/ipam/) and use `weave_extra_args` to enable.

docs/CRI/containerd.md

@@ -149,11 +149,3 @@ following configuration:
 ```yaml
 nri_enabled: true
 ```
-
-### Optional : Static Binary
-
-To ensure compatibility with older distributions (such as Debian 11), you can use a static containerd binary. By default, this is static binary if the system's glibc version is less than 2.34; otherwise, it is the default binary.
-
-```yaml
-containerd_static_binary: true
-```

docs/_sidebar.md

@@ -32,6 +32,7 @@
   * [Kube-router](/docs/CNI/kube-router.md)
   * [Macvlan](/docs/CNI/macvlan.md)
   * [Multus](/docs/CNI/multus.md)
+  * [Weave](/docs/CNI/weave.md)
 * CRI
   * [Containerd](/docs/CRI/containerd.md)
   * [Cri-o](/docs/CRI/cri-o.md)

docs/advanced/arch.md

@@ -9,6 +9,7 @@ The following table shows the impact of the CPU architecture on compatible featu
 | kube_network_plugin | amd64 | arm64 | amd64 + arm64 |
 |---------------------|-------|-------|---------------|
 | Calico              | Y     | Y     | Y             |
+| Weave               | Y     | Y     | Y             |
 | Flannel             | Y     | N     | N             |
 | Canal               | Y     | N     | N             |
 | Cilium              | Y     | Y     | N             |

docs/ansible/ansible.md

@@ -13,7 +13,7 @@ KUBESPRAYDIR=kubespray
 python3 -m venv $VENVDIR
 source $VENVDIR/bin/activate
 cd $KUBESPRAYDIR
-pip install -r requirements.txt
+pip install -U -r requirements.txt
 ```
 
 In case you have a similar message when installing the requirements:
@@ -32,7 +32,7 @@ Based on the table below and the available python version for your ansible host
 
 | Ansible Version | Python Version |
 |-----------------|----------------|
-| >= 2.17.3       | 3.10-3.12      |
+| >= 2.16.4       | 3.10-3.12      |
 
 ## Customize Ansible vars
 
@@ -118,6 +118,7 @@ The following tags are defined in playbooks:
 | local-path-provisioner         | Configure External provisioner: local-path            |
 | local-volume-provisioner       | Configure External provisioner: local-volume          |
 | macvlan                        | Network plugin macvlan                                |
+| master (DEPRECATED)            | Deprecated - see `control-plane`                      |
 | metallb                        | Installing and configuring metallb                    |
 | metrics_server                 | Configuring metrics_server                            |
 | netchecker                     | Installing netchecker K8s app                         |
@@ -152,9 +153,14 @@ The following tags are defined in playbooks:
 | upgrade                        | Upgrading, f.e. container images/binaries             |
 | upload                         | Distributing images/binaries across hosts             |
 | vsphere-csi-driver             | Configuring csi driver: vsphere                       |
+| weave                          | Network plugin Weave                                  |
 | win_nodes                      | Running windows specific tasks                        |
 | youki                          | Configuring youki runtime                             |
 
+Note: Use the ``bash scripts/gen_tags.sh`` command to generate a list of all
+tags found in the codebase. New tags will be listed with the empty "Used for"
+field.
+
 ## Example commands
 
 Example command to filter and apply only DNS configuration tasks and skip
@@ -200,11 +206,11 @@ You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mou
 to access the inventory and SSH key in the container, like this:
 
 ```ShellSession
-git checkout v2.29.0
+git checkout v2.27.0
-docker pull quay.io/kubespray/kubespray:v2.29.0
+docker pull quay.io/kubespray/kubespray:v2.27.0
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
   --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.29.0 bash
+  quay.io/kubespray/kubespray:v2.27.0 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```

docs/ansible/ansible_collection.md

@@ -2,13 +2,14 @@
 
 Kubespray can be installed as an [Ansible collection](https://docs.ansible.com/ansible/latest/user_guide/collections_using.html).
 
+## Requirements
+
+- An inventory file with the appropriate host groups. See the [README](../README.md#usage).
+- A `group_vars` directory. These group variables **need** to match the appropriate variable names under `inventory/local/group_vars`. See the [README](../README.md#usage).
+
 ## Usage
 
-1. Set up an inventory with the appropriate host groups and required group vars.
+1. Add Kubespray to your requirements.yml file
-   See also the documentation on [kubespray inventories](./inventory.md) and the
-   general ["Getting started" documentation](../getting_started/getting-started.md#building-your-own-inventory).
-
-2. Add Kubespray to your requirements.yml file
 
    ```yaml
    collections:
@@ -17,20 +18,20 @@ Kubespray can be installed as an [Ansible collection](https://docs.ansible.com/a
      version: master # use the appropriate tag or branch for the version you need
    ```
 
-3. Install your collection
+2. Install your collection
 
    ```ShellSession
    ansible-galaxy install -r requirements.yml
    ```
 
-4. Create a playbook to install your Kubernetes cluster
+3. Create a playbook to install your Kubernetes cluster
 
    ```yaml
    - name: Install Kubernetes
      ansible.builtin.import_playbook: kubernetes_sigs.kubespray.cluster
    ```
 
-5. Update INVENTORY and PLAYBOOK so that they point to your inventory file and the playbook you created above, and then install Kubespray
+4. Update INVENTORY and PLAYBOOK so that they point to your inventory file and the playbook you created above, and then install Kubespray
 
    ```ShellSession
    ansible-playbook -i INVENTORY --become --become-user=root PLAYBOOK

docs/ansible/vars.md

@@ -103,13 +103,13 @@ following default cluster parameters:
 
 * *kube_service_addresses_ipv6* - Subnet for cluster IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1000/116``). Must not overlap with ``kube_pods_subnet_ipv6``.
 
-* *kube_service_subnets* - All service subnets separated by commas (default is a mix of ``kube_service_addresses`` and ``kube_service_addresses_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stack`` options),
+* *kube_service_subnets* - All service subnets separated by commas (default is a mix of ``kube_service_addresses`` and ``kube_service_addresses_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stacke`` options),
   for example ``10.233.0.0/18,fd85:ee78:d8a6:8607::1000/116`` for dual stack(ipv4_stack/ipv6_stack set to `true`).
   It is not recommended to change this variable directly.
 
 * *kube_pods_subnet_ipv6* - Subnet for Pod IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1:0000/112``). Must not overlap with ``kube_service_addresses_ipv6``.
 
-* *kube_pods_subnets* - All pods subnets separated by commas (default is a mix of ``kube_pods_subnet`` and ``kube_pod_subnet_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stack`` options),
+* *kube_pods_subnets* - All pods subnets separated by commas (default is a mix of ``kube_pods_subnet`` and ``kube_pod_subnet_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stacke`` options),
   for example ``10.233.64.0/18,fd85:ee78:d8a6:8607::1:0000/112`` for dual stack(ipv4_stack/ipv6_stack set to `true`).
   It is not recommended to change this variable directly.
 

docs/cloud_controllers/openstack.md

@@ -73,7 +73,6 @@ The cloud provider is configured to have Octavia by default in Kubespray.
   external_openstack_lbaas_method: ROUND_ROBIN
   external_openstack_lbaas_provider: amphora
   external_openstack_lbaas_subnet_id: "Neutron subnet ID to create LBaaS VIP"
-  external_openstack_lbaas_member_subnet_id: "Neutron subnet ID on which to create the members of the load balancer"
   external_openstack_lbaas_network_id: "Neutron network ID to create LBaaS VIP"
   external_openstack_lbaas_manage_security_groups: false
   external_openstack_lbaas_create_monitor: false

docs/developers/ci-setup.md

@@ -2,14 +2,19 @@
 
 ## Pipeline
 
-See [.gitlab-ci.yml](/.gitlab-ci.yml) and the included files for an overview.
+1. build: build a docker image to be used in the pipeline
+2. unit-tests: fast jobs for fast feedback (linting, etc...)
+3. deploy-part1: small number of jobs to test if the PR works with default settings
+4. deploy-extended: slow jobs testing different platforms, OS, settings, CNI, etc...
+5. deploy-extended: very slow jobs (upgrades, etc...)
 
 ## Runners
 
-Kubespray has 2 types of GitLab runners, both deployed  on the Kubespray CI cluster (hosted on Oracle Cloud Infrastucture):
+Kubespray has 3 types of GitLab runners:
 
-- pods: use the [gitlab-ci kubernetes executor](https://docs.gitlab.com/runner/executors/kubernetes/)
+- packet runners: used for E2E jobs (usually long), running on Equinix Metal (ex-packet), on kubevirt managed VMs
-- vagrant: custom executor running in pods with access to the libvirt socket on the nodes
+- light runners: used for short lived jobs, running on Equinix Metal (ex-packet), as managed pods
+- auto scaling runners (managed via docker-machine on Equinix Metal): used for on-demand resources, see [GitLab docs](https://docs.gitlab.com/runner/configuration/autoscale.html) for more info
 
 ## Vagrant
 
@@ -17,17 +22,18 @@ Vagrant jobs are using the [quay.io/kubespray/vagrant](/test-infra/vagrant-docke
 
 ## CI Variables
 
-In CI we have a [set of extra vars](/test/common_vars.yml) we use to ensure greater success of our CI jobs and avoid throttling by various APIs we depend on.
+In CI we have a set of overrides we use to ensure greater success of our CI jobs and avoid throttling by various APIs we depend on. See:
 
-## CI clusters
+- [Docker mirrors](/tests/common/_docker_hub_registry_mirror.yml)
+- [Test settings](/tests/common/_kubespray_test_settings.yml)
 
-DISCLAIMER: The following information is not fully up to date, in particular, the CI cluster is now on Oracle Cloud Infrastcture, not Equinix.
+## CI Environment
 
-The cluster is deployed with kubespray itself and maintained by the kubespray maintainers.
+The CI packet and light runners are deployed on a kubernetes cluster on Equinix Metal. The cluster is deployed with kubespray itself and maintained by the kubespray maintainers.
 
 The following files are used for that inventory:
 
-### cluster.tfvars (OBSOLETE: this section is no longer accurate)
+### cluster.tfvars
 
 ```ini
 # your Kubernetes cluster name here
@@ -160,6 +166,18 @@ kube_feature_gates:
 
 This section documents additional files used to complete a deployment of the kubespray CI, these files sit on the control-plane node and assume a working kubernetes cluster.
 
+### /root/nscleanup.sh
+
+```bash
+#!/bin/bash
+
+kubectl=/usr/local/bin/kubectl
+
+$kubectl get ns | grep -P "(\d.+-\d.+)" | awk 'match($3,/[0-9]+d/) {print $1}' | xargs -r $kubectl delete ns
+$kubectl get ns | grep -P "(\d.+-\d.+)" | awk 'match($3,/[3-9]+h/) {print $1}' | xargs -r $kubectl delete ns
+$kubectl get ns | grep Terminating | awk '{print $1}' | xargs -i $kubectl delete vmi/instance-1 vmi/instance-0 vmi/instance-2 -n {} --force --grace-period=0 &
+```
+
 ### /root/path-calico.sh
 
 ```bash

docs/developers/ci.md

@@ -6,52 +6,55 @@ To generate this Matrix run `./tests/scripts/md-table/main.py`
 
 | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |
 |---| --- | --- | --- | --- | --- | --- | --- |
+almalinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 almalinux9 |  :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
 amazon |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: |
 debian12 |  :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: |
-debian13 |  :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
 fedora40 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-flatcar4081 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+opensuse15 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+rockylinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
-ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu20 |  :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: |
 ubuntu22 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu24 |  :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: |
+ubuntu24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 
 ## crio
 
 | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |
 |---| --- | --- | --- | --- | --- | --- | --- |
+almalinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 almalinux9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian13 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora40 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora40 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-flatcar4081 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+opensuse15 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu20 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu22 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu22 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 
 ## docker
 
 | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |
 |---| --- | --- | --- | --- | --- | --- | --- |
+almalinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 almalinux9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian13 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora40 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-flatcar4081 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+opensuse15 |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
+rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu20 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu22 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |

docs/external_storage_provisioners/scheduler_plugins.md

@@ -7,7 +7,7 @@ The kube-scheduler binary includes a list of plugins:
 - [CapacityScheduling](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/capacityscheduling) [Beta]
 - [CoScheduling](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/coscheduling) [Beta]
 - [NodeResources](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/noderesources) [Beta]
-- [NodeResourceTopology](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/noderesourcetopology/README.md) [Beta]
+- [NodeResouceTopology](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/noderesourcetopology/README.md) [Beta]
 - [PreemptionToleration](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/preemptiontoleration/README.md) [Alpha]
 - [Trimaran](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/trimaran/README.md) [Alpha]
 - [NetworkAware](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/networkaware/README.md) [Sample]

docs/getting_started/setting-up-your-first-cluster.md

@@ -61,12 +61,12 @@ gcloud compute networks subnets create kubernetes \
 #### Firewall Rules
 
 Create a firewall rule that allows internal communication across all protocols.
-It is important to note that the vxlan (udp) protocol has to be allowed in order for
+It is important to note that the vxlan protocol has to be allowed in order for
 the calico (see later) networking plugin to work.
 
 ```ShellSession
 gcloud compute firewall-rules create kubernetes-the-kubespray-way-allow-internal \
-  --allow tcp,udp,icmp \
+  --allow tcp,udp,icmp,vxlan \
   --network kubernetes-the-kubespray-way \
   --source-ranges 10.240.0.0/24
 ```
@@ -88,7 +88,7 @@ cluster.
 
 ### Compute Instances
 
-The compute instances in this lab will be provisioned using [Ubuntu Server](https://www.ubuntu.com/server) 24.04.
+The compute instances in this lab will be provisioned using [Ubuntu Server](https://www.ubuntu.com/server) 18.04.
 Each compute instance will be provisioned with a fixed private IP address and
  a public IP address (that can be fixed - see [guide](https://cloud.google.com/compute/docs/ip-addresses/reserve-static-external-ip-address)).
 Using fixed public IP addresses has the advantage that our cluster node
@@ -103,7 +103,7 @@ for i in 0 1 2; do
     --async \
     --boot-disk-size 200GB \
     --can-ip-forward \
-    --image-family ubuntu-2404-lts-amd64 \
+    --image-family ubuntu-1804-lts \
     --image-project ubuntu-os-cloud \
     --machine-type e2-standard-2 \
     --private-network-ip 10.240.0.1${i} \
@@ -124,7 +124,7 @@ for i in 0 1 2; do
     --async \
     --boot-disk-size 200GB \
     --can-ip-forward \
-    --image-family ubuntu-2404-lts-amd64 \
+    --image-family ubuntu-1804-lts \
     --image-project ubuntu-os-cloud \
     --machine-type e2-standard-2 \
     --private-network-ip 10.240.0.2${i} \

docs/ingress/ingress_nginx.md

@@ -35,7 +35,7 @@ kubectl create clusterrolebinding cluster-admin-binding \
 The following **Mandatory Command** is required for all deployments except for AWS. See below for the AWS version.
 
 ```console
-kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.13.3/deploy/static/provider/cloud/deploy.yaml
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.0/deploy/static/provider/cloud/deploy.yaml
 ```
 
 ### Provider Specific Steps

docs/operations/cgroups.md

@@ -30,6 +30,11 @@ kube_memory_reserved: 256Mi
 kube_cpu_reserved: 100m
 # kube_ephemeral_storage_reserved: 2Gi
 # kube_pid_reserved: "1000"
+# Reservation for master hosts
+kube_master_memory_reserved: 512Mi
+kube_master_cpu_reserved: 200m
+# kube_master_ephemeral_storage_reserved: 2Gi
+# kube_master_pid_reserved: "1000"
 
 # Set to true to reserve resources for system daemons
 system_reserved: true
@@ -39,6 +44,11 @@ system_memory_reserved: 512Mi
 system_cpu_reserved: 500m
 # system_ephemeral_storage_reserved: 2Gi
 # system_pid_reserved: "1000"
+# Reservation for master hosts
+system_master_memory_reserved: 256Mi
+system_master_cpu_reserved: 250m
+# system_master_ephemeral_storage_reserved: 2Gi
+# system_master_pid_reserved: "1000"
 ```
 
 After the setup, the cgroups hierarchy is as follows:

docs/operations/hardening.md

@@ -18,6 +18,8 @@ The **kubernetes** version should be at least `v1.23.6` to have all the most rec
 
 ## kube-apiserver
 authorization_modes: ['Node', 'RBAC']
+# AppArmor-based OS
+# kube_apiserver_feature_gates: ['AppArmor=true']
 kube_apiserver_request_timeout: 120s
 kube_apiserver_service_account_lookup: true
 
@@ -75,17 +77,17 @@ remove_anonymous_access: true
 ## kube-controller-manager
 kube_controller_manager_bind_address: 127.0.0.1
 kube_controller_terminated_pod_gc_threshold: 50
+# AppArmor-based OS
+# kube_controller_feature_gates: ["RotateKubeletServerCertificate=true", "AppArmor=true"]
 kube_controller_feature_gates: ["RotateKubeletServerCertificate=true"]
 
 ## kube-scheduler
 kube_scheduler_bind_address: 127.0.0.1
+# AppArmor-based OS
+# kube_scheduler_feature_gates: ["AppArmor=true"]
 
 ## etcd
-# Running etcd (on dedicated hosts) outside the Kubernetes cluster is the most secure deployment option,
+etcd_deployment_type: kubeadm
-# as it isolates etcd from the cluster's CNI network and removes direct pod-level attack vectors.
-# This approach prevents RBAC misconfigurations that potentially compromise etcd,
-# creating an additional security boundary that protects the cluster's critical state store.
-etcd_deployment_type: host
 
 ## kubelet
 kubelet_authorization_mode_webhook: true
@@ -100,8 +102,6 @@ kubelet_make_iptables_util_chains: true
 kubelet_feature_gates: ["RotateKubeletServerCertificate=true"]
 kubelet_seccomp_default: true
 kubelet_systemd_hardening: true
-# To disable kubelet's staticPodPath (for nodes that don't use static pods like worker nodes)
-kubelet_static_pod_path: ""
 # In case you have multiple interfaces in your
 # control plane nodes and you want to specify the right
 # IP addresses, kubelet_secure_addresses allows you
@@ -126,8 +126,9 @@ Let's take a deep look to the resultant **kubernetes** configuration:
 * The `encryption-provider-config` provide encryption at rest. This means that the `kube-apiserver` encrypt data that is going to be stored before they reach `etcd`. So the data is completely unreadable from `etcd` (in case an attacker is able to exploit this).
 * The `rotateCertificates` in `KubeletConfiguration` is set to `true` along with `serverTLSBootstrap`. This could be used in alternative to `tlsCertFile` and `tlsPrivateKeyFile` parameters. Additionally it automatically generates certificates by itself. By default the CSRs are approved automatically via [kubelet-csr-approver](https://github.com/postfinance/kubelet-csr-approver). You can customize approval configuration by modifying Helm values via `kubelet_csr_approver_values`.
   See <https://kubernetes.io/docs/reference/access-authn-authz/kubelet-tls-bootstrapping/> for more information on the subject.
+* If you are installing **kubernetes** in an AppArmor-based OS (eg. Debian/Ubuntu) you can enable the `AppArmor` feature gate uncommenting the lines with the comment `# AppArmor-based OS` on top.
 * The `kubelet_systemd_hardening`, both with `kubelet_secure_addresses` setup a minimal firewall on the system. To better understand how these variables work, here's an explanatory image:
-  ![kubelet hardening](../img/kubelet-hardening.png)
+  ![kubelet hardening](img/kubelet-hardening.png)
 
 Once you have the file properly filled, you can run the **Ansible** command to start the installation:
 

docs/operations/nodes.md

@@ -2,6 +2,58 @@
 
 Modified from [comments in #3471](https://github.com/kubernetes-sigs/kubespray/issues/3471#issuecomment-530036084)
 
+## Limitation: Removal of first kube_control_plane and etcd-master
+
+Currently you can't remove the first node in your kube_control_plane and etcd-master list. If you still want to remove this node you have to:
+
+### 1) Change order of current control planes
+
+Modify the order of your control plane list by pushing your first entry to any other position. E.g. if you want to remove `node-1` of the following example:
+
+```yaml
+  children:
+    kube_control_plane:
+      hosts:
+        node-1:
+        node-2:
+        node-3:
+    kube_node:
+      hosts:
+        node-1:
+        node-2:
+        node-3:
+    etcd:
+      hosts:
+        node-1:
+        node-2:
+        node-3:
+```
+
+change your inventory to:
+
+```yaml
+  children:
+    kube_control_plane:
+      hosts:
+        node-2:
+        node-3:
+        node-1:
+    kube_node:
+      hosts:
+        node-2:
+        node-3:
+        node-1:
+    etcd:
+      hosts:
+        node-2:
+        node-3:
+        node-1:
+```
+
+## 2) Upgrade the cluster
+
+run `upgrade-cluster.yml` or `cluster.yml`. Now you are good to go on with the removal.
+
 ## Adding/replacing a worker node
 
 This should be the easiest.
@@ -48,74 +100,40 @@ crictl ps | grep nginx-proxy | awk '{print $1}' | xargs crictl stop
 With the old node still in the inventory, run `remove-node.yml`. You need to pass `-e node=NODE_NAME` to the playbook to limit the execution to the node being removed.
 If the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars.
 
-## Adding/Removal of first `kube_control_plane` and etcd-master
+## Replacing a first control plane node
 
-Currently you can't remove the first node in your `kube_control_plane` and etcd-master list. If you still want to remove this node you have to:
+### 1) Change control plane nodes order in inventory
 
-### 1) Change order of current control planes
+from
 
-Modify the order of your control plane list by pushing your first entry to any other position. E.g. if you want to remove `node-1` of the following example:
+```ini
-
+[kube_control_plane]
-```yaml
+ node-1
-all:
+ node-2
-  hosts:
+ node-3
-  children:
-    kube_control_plane:
-      hosts:
-        node-1:
-        node-2:
-        node-3:
-    kube_node:
-      hosts:
-        node-1:
-        node-2:
-        node-3:
-    etcd:
-      hosts:
-        node-1:
-        node-2:
-        node-3:
 ```
 
-change your inventory to:
+to
 
-```yaml
+```ini
-all:
+[kube_control_plane]
-  hosts:
+ node-2
-  children:
+ node-3
-    kube_control_plane:
+ node-1
-      hosts:
-        node-2:
-        node-3:
-        node-1:
-    kube_node:
-      hosts:
-        node-2:
-        node-3:
-        node-1:
-    etcd:
-      hosts:
-        node-2:
-        node-3:
-        node-1:
 ```
 
-### 2) Upgrade the cluster
+### 2) Remove old first control plane node from cluster
-
-run `upgrade-cluster.yml` or `cluster.yml`. Now you are good to go on with the removal.
-
-### 3) Remove old first control plane node from cluster
 
 With the old node still in the inventory, run `remove-node.yml`. You need to pass `-e node=node-1` to the playbook to limit the execution to the node being removed.
 If the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars.
 
-### 4) Edit cluster-info configmap in kube-public namespace
+### 3) Edit cluster-info configmap in kube-public namespace
 
 `kubectl  edit cm -n kube-public cluster-info`
 
 Change ip of old kube_control_plane node with ip of live kube_control_plane node (`server` field). Also, update `certificate-authority-data` field if you changed certs.
 
-### 5) Add new control plane node
+### 4) Add new control plane node
 
 Update inventory (if needed)
 

docs/operations/upgrades.md

@@ -13,6 +13,7 @@ versions. Here are all version vars for each component:
 * etcd_version
 * calico_version
 * calico_cni_version
+* weave_version
 * flannel_version
 
 > **Warning**
@@ -355,7 +356,7 @@ follows:
 * Containerd
 * etcd
 * kubelet and kube-proxy
-* network_plugin (such as Calico)
+* network_plugin (such as Calico or Weave)
 * kube-apiserver, kube-scheduler, and kube-controller-manager
 * Add-ons (such as KubeDNS)
 

galaxy.yml

@@ -2,7 +2,7 @@
 namespace: kubernetes_sigs
 description: Deploy a production ready Kubernetes cluster
 name: kubespray
-version: 2.29.2
+version: 2.28.2
 readme: README.md
 authors:
   - The Kubespray maintainers (https://kubernetes.slack.com/channels/kubespray)

inventory/sample/group_vars/all/all.yml

@@ -115,9 +115,6 @@ no_proxy_exclude_workers: false
 # sysctl_file_path to add sysctl conf to
 # sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf"
 
-# ignore sysctl errors about unknown keys
-# sysctl_ignore_unknown_keys: false
-
 ## Variables for webhook token auth https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication
 kube_webhook_token_auth: false
 kube_webhook_token_auth_url_skip_tls_verify: false

inventory/sample/group_vars/all/containerd.yml

@@ -50,8 +50,6 @@
 #     - host: https://registry-1.docker.io
 #       capabilities: ["pull", "resolve"]
 #       skip_verify: false
-#       header:
-#         Authorization: "Basic XXX"
 
 # containerd_max_container_log_line_size: 16384
 

inventory/sample/group_vars/etcd.yml

@@ -0,0 +1,38 @@
+---
+## Etcd auto compaction retention for mvcc key value store in hour
+# etcd_compaction_retention: 0
+
+## Set level of detail for etcd exported metrics, specify 'extensive' to include histogram metrics.
+# etcd_metrics: basic
+
+## Etcd is restricted by default to 512M on systems under 4GB RAM, 512MB is not enough for much more than testing.
+## Set this if your etcd nodes have less than 4GB but you want more RAM for etcd. Set to 0 for unrestricted RAM.
+## This value is only relevant when deploying etcd with `etcd_deployment_type: docker`
+# etcd_memory_limit: "512M"
+
+## Etcd has a default of 2G for its space quota. If you put a value in etcd_memory_limit which is less than
+## etcd_quota_backend_bytes, you may encounter out of memory terminations of the etcd cluster. Please check
+## etcd documentation for more information.
+# 8G is a suggested maximum size for normal environments and etcd warns at startup if the configured value exceeds it.
+# etcd_quota_backend_bytes: "2147483648"
+
+# Maximum client request size in bytes the server will accept.
+# etcd is designed to handle small key value pairs typical for metadata.
+# Larger requests will work, but may increase the latency of other requests
+# etcd_max_request_bytes: "1572864"
+
+### ETCD: disable peer client cert authentication.
+# This affects ETCD_PEER_CLIENT_CERT_AUTH variable
+# etcd_peer_client_auth: true
+
+## Enable distributed tracing
+## To enable this experimental feature, set the etcd_experimental_enable_distributed_tracing: true, along with the
+## etcd_experimental_distributed_tracing_sample_rate to choose how many samples to collect per million spans,
+## the default sampling rate is 0 https://etcd.io/docs/v3.5/op-guide/monitoring/#distributed-tracing
+# etcd_experimental_enable_distributed_tracing: false
+# etcd_experimental_distributed_tracing_sample_rate: 100
+# etcd_experimental_distributed_tracing_address: "localhost:4317"
+# etcd_experimental_distributed_tracing_service_name: etcd
+
+## The interval for etcd watch progress notify events
+# etcd_experimental_watch_progress_notify_interval: 5s

inventory/sample/group_vars/k8s_cluster/addons.yml

@@ -29,6 +29,7 @@ local_path_provisioner_enabled: false
 # local_path_provisioner_claim_root: /opt/local-path-provisioner/
 # local_path_provisioner_debug: false
 # local_path_provisioner_image_repo: "{{ docker_image_repo }}/rancher/local-path-provisioner"
+# local_path_provisioner_image_tag: "v0.0.24"
 # local_path_provisioner_helper_image_repo: "busybox"
 # local_path_provisioner_helper_image_tag: "latest"
 

inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml

@@ -62,7 +62,7 @@ credentials_dir: "{{ inventory_dir }}/credentials"
 # kube_webhook_authorization_url: https://...
 # kube_webhook_authorization_url_skip_tls_verify: false
 
-# Choose network plugin (cilium, calico, kube-ovn or flannel. Use cni for generic cni plugin)
+# Choose network plugin (cilium, calico, kube-ovn, weave or flannel. Use cni for generic cni plugin)
 # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
 kube_network_plugin: calico
 
@@ -346,7 +346,7 @@ event_ttl_duration: "1h0m0s"
 ## Automatically renew K8S control plane certificates on first Monday of each month
 auto_renew_certificates: false
 # First Monday of each month
-# auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:00:00"
+# auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:{{ groups['kube_control_plane'].index(inventory_hostname) }}0:00"
 
 kubeadm_patches_dir: "{{ kube_config_dir }}/patches"
 kubeadm_patches: []

inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml

@@ -25,9 +25,15 @@ calico_pool_blocksize: 26
 # add default ippool CIDR (must be inside kube_pods_subnet, defaults to kube_pods_subnet otherwise)
 # calico_pool_cidr: 1.2.3.4/5
 
+# add default ippool CIDR to CNI config
+# calico_cni_pool: true
+
 # Add default IPV6 IPPool CIDR. Must be inside kube_pods_subnet_ipv6. Defaults to kube_pods_subnet_ipv6 if not set.
 # calico_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112
 
+# Add default IPV6 IPPool CIDR to CNI config
+# calico_cni_pool_ipv6: true
+
 # Global as_num (/calico/bgp/v1/global/as_num)
 # global_as_num: "64512"
 

inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml

@@ -56,8 +56,8 @@ cilium_l2announcements: false
 #
 # Only effective when monitor aggregation is set to "medium" or higher.
 # cilium_monitor_aggregation_flags: "all"
-# Kube Proxy Replacement mode (true/false)
+# Kube Proxy Replacement mode (strict/partial)
-# cilium_kube_proxy_replacement: false
+# cilium_kube_proxy_replacement: partial
 
 # If upgrading from Cilium < 1.5, you may want to override some of these options
 # to prevent service disruptions. See also:
@@ -384,7 +384,3 @@ cilium_l2announcements: false
 #       resourceNames:
 #       - toto
 # cilium_clusterrole_rules_operator_extra_vars: []
-
-# Cilium extra values, use any values from cilium Helm Chart
-# ref: https://docs.cilium.io/en/stable/helm-reference/
-# cilium_extra_values: {}

inventory/sample/group_vars/k8s_cluster/k8s-net-weave.yml

@@ -0,0 +1,64 @@
+# see roles/network_plugin/weave/defaults/main.yml
+
+# Weave's network password for encryption, if null then no network encryption.
+# weave_password: ~
+
+# If set to 1, disable checking for new Weave Net versions (default is blank,
+# i.e. check is enabled)
+# weave_checkpoint_disable: false
+
+# Soft limit on the number of connections between peers. Defaults to 100.
+# weave_conn_limit: 100
+
+# Weave Net defaults to enabling hairpin on the bridge side of the veth pair
+# for containers attached. If you need to disable hairpin, e.g. your kernel is
+# one of those that can panic if hairpin is enabled, then you can disable it by
+# setting `HAIRPIN_MODE=false`.
+# weave_hairpin_mode: true
+
+# The range of IP addresses used by Weave Net and the subnet they are placed in
+# (CIDR format; default 10.32.0.0/12)
+# weave_ipalloc_range: "{{ kube_pods_subnet }}"
+
+# Set to 0 to disable Network Policy Controller (default is on)
+# weave_expect_npc: "{{ enable_network_policy }}"
+
+# List of addresses of peers in the Kubernetes cluster (default is to fetch the
+# list from the api-server)
+# weave_kube_peers: ~
+
+# Set the initialization mode of the IP Address Manager (defaults to consensus
+# amongst the KUBE_PEERS)
+# weave_ipalloc_init: ~
+
+# Set the IP address used as a gateway from the Weave network to the host
+# network - this is useful if you are configuring the addon as a static pod.
+# weave_expose_ip: ~
+
+# Address and port that the Weave Net daemon will serve Prometheus-style
+# metrics on (defaults to 0.0.0.0:6782)
+# weave_metrics_addr: ~
+
+# Address and port that the Weave Net daemon will serve status requests on
+# (defaults to disabled)
+# weave_status_addr: ~
+
+# Weave Net defaults to 1376 bytes, but you can set a smaller size if your
+# underlying network has a tighter limit, or set a larger size for better
+# performance if your network supports jumbo frames (e.g. 8916)
+# weave_mtu: 1376
+
+# Set to 1 to preserve the client source IP address when accessing Service
+# annotated with `service.spec.externalTrafficPolicy=Local`. The feature works
+# only with Weave IPAM (default).
+# weave_no_masq_local: true
+
+# set to nft to use nftables backend for iptables (default is iptables)
+# weave_iptables_backend: iptables
+
+# Extra variables that passing to launch.sh, useful for enabling seed mode, see
+# https://www.weave.works/docs/net/latest/tasks/ipam/ipam/
+# weave_extra_args: ~
+
+# Extra variables for weave_npc that passing to launch.sh, useful for change log level, ex --log-level=error
+# weave_npc_extra_args: ~

meta/runtime.yml

@@ -1,2 +1,2 @@
 ---
-requires_ansible: ">=2.17.3"
+requires_ansible: '>=2.16.4'

pipeline.Dockerfile

@@ -1,4 +1,4 @@
-# Use immutable image tags rather than mutable tags (like ubuntu:22.04)
+# Use imutable image tags rather than mutable tags (like ubuntu:22.04)
 FROM ubuntu:jammy-20230308
 # Some tools like yamllint need this
 # Pip needs this as well at the moment to install ansible
@@ -47,8 +47,8 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
     && pip install --no-compile --no-cache-dir pip -U \
     && pip install --no-compile --no-cache-dir -r tests/requirements.txt \
     && pip install --no-compile --no-cache-dir -r requirements.txt \
-    && curl -L https://dl.k8s.io/release/v1.33.7/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
+    && curl -L https://dl.k8s.io/release/v1.32.11/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
-    && echo $(curl -L https://dl.k8s.io/release/v1.33.7/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
+    && echo $(curl -L https://dl.k8s.io/release/v1.32.11/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
     && chmod a+x /usr/local/bin/kubectl \
     # Install Vagrant
     && curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \

playbooks/ansible_version.yml

@@ -5,8 +5,8 @@
   become: false
   run_once: true
   vars:
-    minimal_ansible_version: 2.17.3
+    minimal_ansible_version: 2.16.4
-    maximal_ansible_version: 2.18.0
+    maximal_ansible_version: 2.17.0
   tags: always
   tasks:
     - name: "Check {{ minimal_ansible_version }} <= Ansible version < {{ maximal_ansible_version }}"

playbooks/boilerplate.yml

@@ -6,12 +6,35 @@
 # - to ensure we keep compatibility with old style group names
 # - to reduce inventory boilerplate (defining parent groups / empty groups)
 
-- name: Inventory setup and validation
+- name: Define groups for legacy less structured inventories
+  hosts: all
+  gather_facts: false
+  tags: always
+  tasks:
+    - name: Match needed groups by their old names or definition
+      vars:
+        group_mappings:
+          kube_control_plane:
+            - kube-master
+          kube_node:
+            - kube-node
+          calico_rr:
+            - calico-rr
+          no_floating:
+            - no-floating
+          k8s_cluster:
+            - kube_node
+            - kube_control_plane
+            - calico_rr
+      group_by:
+        key: "{{ (group_names | intersect(item.value) | length > 0) | ternary(item.key, '_all') }}"
+      loop: "{{ group_mappings | dict2items }}"
+
+- name: Check inventory settings
   hosts: all
   gather_facts: false
   tags: always
   roles:
-    - dynamic_groups
     - validate_inventory
 
 - name: Install bastion ssh config

playbooks/cluster.yml

@@ -3,7 +3,7 @@
   import_playbook: boilerplate.yml
 
 - name: Gather facts
-  import_playbook: internal_facts.yml
+  import_playbook: facts.yml
 
 - name: Prepare for etcd install
   hosts: k8s_cluster:etcd
@@ -38,7 +38,7 @@
   environment: "{{ proxy_disable_env }}"
   roles:
     - { role: kubespray_defaults }
-    - { role: kubernetes/control-plane, tags: control-plane }
+    - { role: kubernetes/control-plane, tags: master }
     - { role: kubernetes/client, tags: client }
     - { role: kubernetes-apps/cluster_roles, tags: cluster-roles }
 
@@ -52,7 +52,11 @@
     - { role: kubernetes/kubeadm, tags: kubeadm}
     - { role: kubernetes/node-label, tags: node-label }
     - { role: kubernetes/node-taint, tags: node-taint }
-    - { role: kubernetes-apps/common_crds }
+    - role: kubernetes-apps/gateway_api
+      when: gateway_api_enabled
+      tags: gateway_api
+      delegate_to: "{{ groups['kube_control_plane'][0] }}"
+      run_once: true
     - { role: network_plugin, tags: network }
 
 - name: Install Calico Route Reflector
@@ -71,7 +75,7 @@
   environment: "{{ proxy_disable_env }}"
   roles:
     - { role: kubespray_defaults }
-    - { role: win_nodes/kubernetes_patch, tags: ["control-plane", "win_nodes"] }
+    - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"] }
 
 - name: Install Kubernetes apps
   hosts: kube_control_plane
@@ -81,6 +85,7 @@
   roles:
     - { role: kubespray_defaults }
     - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller }
+    - { role: kubernetes-apps/network_plugin, tags: network }
     - { role: kubernetes-apps/policy_controller, tags: policy-controller }
     - { role: kubernetes-apps/ingress_controller, tags: ingress-controller }
     - { role: kubernetes-apps/external_provisioner, tags: external-provisioner }

playbooks/facts.yml

@@ -1,6 +1,39 @@
 ---
-- name: Common tasks for every playbooks
+- name: Bootstrap hosts for Ansible
-  import_playbook: boilerplate.yml
+  hosts: k8s_cluster:etcd:calico_rr
+  strategy: linear
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  gather_facts: false
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: bootstrap_os, tags: bootstrap_os}
 
 - name: Gather facts
-  import_playbook: internal_facts.yml
+  hosts: k8s_cluster:etcd:calico_rr
+  gather_facts: false
+  tags: always
+  tasks:
+    - name: Gather and compute network facts
+      import_role:
+        name: network_facts
+    - name: Gather minimal facts
+      setup:
+        gather_subset: '!all'
+
+    # filter match the following variables:
+    # ansible_default_ipv4
+    # ansible_default_ipv6
+    # ansible_all_ipv4_addresses
+    # ansible_all_ipv6_addresses
+    - name: Gather necessary facts (network)
+      setup:
+        gather_subset: '!all,!min,network'
+        filter: "ansible_*_ipv[46]*"
+
+    # filter match the following variables:
+    # ansible_memtotal_mb
+    # ansible_swaptotal_mb
+    - name: Gather necessary facts (hardware)
+      setup:
+        gather_subset: '!all,!min,hardware'
+        filter: "ansible_*total_mb"

playbooks/internal_facts.yml

@@ -1,39 +0,0 @@
----
-- name: Bootstrap hosts for Ansible
-  hosts: k8s_cluster:etcd:calico_rr
-  strategy: linear
-  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
-  gather_facts: false
-  environment: "{{ proxy_disable_env }}"
-  roles:
-    - { role: bootstrap_os, tags: bootstrap_os}
-
-- name: Gather facts
-  hosts: k8s_cluster:etcd:calico_rr
-  gather_facts: false
-  tags: always
-  tasks:
-    - name: Gather and compute network facts
-      import_role:
-        name: network_facts
-    - name: Gather minimal facts
-      setup:
-        gather_subset: '!all'
-
-    # filter match the following variables:
-    # ansible_default_ipv4
-    # ansible_default_ipv6
-    # ansible_all_ipv4_addresses
-    # ansible_all_ipv6_addresses
-    - name: Gather necessary facts (network)
-      setup:
-        gather_subset: '!all,!min,network'
-        filter: "ansible_*_ipv[46]*"
-
-    # filter match the following variables:
-    # ansible_memtotal_mb
-    # ansible_swaptotal_mb
-    - name: Gather necessary facts (hardware)
-      setup:
-        gather_subset: '!all,!min,hardware'
-        filter: "ansible_*total_mb"

playbooks/remove_node.yml

@@ -1,8 +1,6 @@
 ---
 - name: Validate nodes for removal
   hosts: localhost
-  gather_facts: false
-  become: false
   tasks:
     - name: Assert that nodes are specified for removal
       assert:
@@ -32,7 +30,7 @@
       when: pause_result.user_input | default('yes') != 'yes'
 
 - name: Gather facts
-  import_playbook: internal_facts.yml
+  import_playbook: facts.yml
   when: reset_nodes | default(True) | bool
 
 - name: Reset node

playbooks/reset.yml

@@ -3,7 +3,7 @@
   import_playbook: boilerplate.yml
 
 - name: Gather facts
-  import_playbook: internal_facts.yml
+  import_playbook: facts.yml
 
 - name: Reset cluster
   hosts: etcd:k8s_cluster:calico_rr

playbooks/scale.yml

@@ -3,7 +3,7 @@
   import_playbook: boilerplate.yml
 
 - name: Gather facts
-  import_playbook: internal_facts.yml
+  import_playbook: facts.yml
 
 - name: Install etcd
   vars:

playbooks/upgrade_cluster.yml

@@ -3,7 +3,7 @@
   import_playbook: boilerplate.yml
 
 - name: Gather facts
-  import_playbook: internal_facts.yml
+  import_playbook: facts.yml
 
 - name: Download images to ansible host cache via first kube_control_plane node
   hosts: kube_control_plane[0]
@@ -73,6 +73,7 @@
     - { role: kubespray_defaults }
     - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller }
     - { role: network_plugin, tags: network }
+    - { role: kubernetes-apps/network_plugin, tags: network }
     - { role: kubernetes-apps/policy_controller, tags: policy-controller }
 
 - name: Finally handle worker upgrades, based on given batch size

remove_node.yml

@@ -1,3 +0,0 @@
----
-- name: Remove node
-  ansible.builtin.import_playbook: playbooks/remove_node.yml

requirements.txt

@@ -1,6 +1,6 @@
-ansible==10.7.0
+ansible==9.13.0
 # Needed for community.crypto module
-cryptography==46.0.2
+cryptography==45.0.2
 # Needed for jinja2 json_query templating
 jmespath==1.0.1
 # Needed for ansible.utils.ipaddr

roles/adduser/molecule/default/molecule.yml

@@ -3,8 +3,8 @@ role_name_check: 1
 dependency:
   name: galaxy
 platforms:
-  - name: ubuntu22
+  - name: ubuntu20
-    cloud_image: ubuntu-2204
+    cloud_image: ubuntu-2004
     vm_cpu_cores: 1
     vm_memory: 512
 provisioner:

roles/bastion-ssh-config/molecule/default/molecule.yml

@@ -4,7 +4,7 @@ dependency:
   name: galaxy
 platforms:
   - name: bastion-01
-    cloud_image: ubuntu-2204
+    cloud_image: ubuntu-2004
     vm_cpu_cores: 1
     vm_memory: 512
 provisioner:

roles/bootstrap_os/molecule/default/molecule.yml

@@ -3,12 +3,12 @@ role_name_check: 1
 dependency:
   name: galaxy
 platforms:
-  - name: ubuntu22
+  - name: ubuntu20
-    cloud_image: ubuntu-2204
+    cloud_image: ubuntu-2004
     vm_cpu_cores: 1
     vm_memory: 512
-  - name: ubuntu24
+  - name: ubuntu22
-    cloud_image: ubuntu-2404
+    cloud_image: ubuntu-2204
     vm_cpu_cores: 1
     vm_memory: 512
   - name: almalinux9

roles/bootstrap_os/tasks/almalinux.yml

@@ -1,3 +0,0 @@
----
-- name: Import Centos boostrap for Alma Linux
-  import_tasks: centos.yml

roles/bootstrap_os/tasks/centos.yml

@@ -17,6 +17,32 @@
   when: not skip_http_proxy_on_os_packages
 
 # For Oracle Linux install public repo
+- name: Download Oracle Linux public yum repo
+  get_url:
+    url: https://yum.oracle.com/public-yum-ol7.repo
+    dest: /etc/yum.repos.d/public-yum-ol7.repo
+    mode: "0644"
+  when:
+    - use_oracle_public_repo | default(true)
+    - '''ID="ol"'' in os_release.stdout_lines'
+    - (ansible_distribution_version | float) < 7.6
+  environment: "{{ proxy_env }}"
+
+- name: Enable Oracle Linux repo
+  community.general.ini_file:
+    dest: /etc/yum.repos.d/public-yum-ol7.repo
+    section: "{{ item }}"
+    option: enabled
+    value: "1"
+    mode: "0644"
+  with_items:
+    - ol7_latest
+    - ol7_addons
+    - ol7_developer_EPEL
+  when:
+    - use_oracle_public_repo | default(true)
+    - '''ID="ol"'' in os_release.stdout_lines'
+    - (ansible_distribution_version | float) < 7.6
 
 - name: Install EPEL for Oracle Linux repo package
   package:

roles/bootstrap_os/tasks/clear-linux-os.yml

@@ -0,0 +1,16 @@
+---
+# ClearLinux ships with Python installed
+
+- name: Install basic package to run containers
+  package:
+    name: containers-basic
+    state: present
+
+- name: Make sure docker service is enabled
+  systemd_service:
+    name: docker
+    masked: false
+    enabled: true
+    daemon_reload: true
+    state: started
+  become: true

roles/bootstrap_os/tasks/debian.yml

@@ -50,3 +50,15 @@
   become: true
   when:
     - need_bootstrap.rc != 0
+
+- name: Update Apt cache
+  raw: apt-get update --allow-releaseinfo-change
+  become: true
+  when:
+    - os_release_dict['ID'] == 'debian'
+    - os_release_dict['VERSION_ID'] in ["10", "11"]
+  register: bootstrap_update_apt_result
+  changed_when:
+    - '"changed its" in bootstrap_update_apt_result.stdout'
+    - '"value from" in bootstrap_update_apt_result.stdout'
+  ignore_errors: true

roles/bootstrap_os/tasks/main.yml

@@ -16,7 +16,8 @@
     tags:
     - facts
     with_first_found:
-    - files: &search_files
+    - &search
+      files:
       - "{{ os_release_dict['ID'] }}-{{ os_release_dict['VARIANT_ID'] }}.yml"
       - "{{ os_release_dict['ID'] }}.yml"
       paths:
@@ -25,8 +26,8 @@
   - name: Include tasks
     include_tasks: "{{ included_tasks_file }}"
     with_first_found:
-    - files: *search_files
+    - <<: *search
-      skip: true
+      paths: []
     loop_control:
       loop_var: included_tasks_file
 

roles/bootstrap_os/tasks/opensuse.yml

@@ -55,3 +55,31 @@
   become: true
   tags:
     - facts
+
+# Without this package, the get_url module fails when trying to handle https
+- name: Install python-cryptography
+  community.general.zypper:
+    name: python-cryptography
+    state: present
+    update_cache: true
+  become: true
+  when:
+    - ansible_distribution_version is version('15.4', '<')
+
+- name: Install python3-cryptography
+  community.general.zypper:
+    name: python3-cryptography
+    state: present
+    update_cache: true
+  become: true
+  when:
+    - ansible_distribution_version is version('15.4', '>=')
+
+# Nerdctl needs some basic packages to get an environment up
+- name: Install basic dependencies
+  community.general.zypper:
+    name:
+      - iptables
+      - apparmor-parser
+    state: present
+  become: true

roles/bootstrap_os/tasks/rocky.yml

@@ -1,3 +0,0 @@
----
-- name: Import Centos boostrap for Rocky Linux
-  import_tasks: centos.yml

roles/container-engine/containerd/defaults/main.yml

@@ -64,8 +64,7 @@ containerd_registries_mirrors:
         skip_verify: false
 #        ca: ["/etc/certs/mirror.pem"]
 #        client: [["/etc/certs/client.pem", ""],["/etc/certs/client.cert", "/etc/certs/client.key"]]
-#        header:
+
-#          Authorization: "Basic XXX"
 containerd_max_container_log_line_size: 16384
 
 # If enabled it will allow non root users to use port numbers <1024
@@ -84,15 +83,6 @@ containerd_cfg_dir: /etc/containerd
 # Extra config to be put in {{ containerd_cfg_dir }}/config.toml literally
 containerd_extra_args: ''
 
-# Extra runtime configuration options to be injected into the containerd CRI runtime plugin section
-# [plugins."io.containerd.cri.v1.runtime"]. This is useful for adding containerd runtime
-# configuration options that aren't explicitly supported by Kubespray's default variables.
-# Example:
-#   containerd_extra_runtime_args:
-#     device_ownership_from_security_context: true
-#     another_option: "value"
-containerd_extra_runtime_args: {}
-
 # Configure registry auth (if applicable to secure/insecure registries)
 containerd_registry_auth: []
 #  - registry: 10.0.0.2:5000

roles/container-engine/containerd/molecule/default/molecule.yml

@@ -1,16 +1,16 @@
 ---
 role_name_check: 1
 platforms:
-  - cloud_image: ubuntu-2404
+  - cloud_image: ubuntu-2004
-    name: ubuntu24
+    name: ubuntu20
     vm_cpu_cores: 1
     vm_memory: 1024
     node_groups:
       - kube_control_plane
       - kube_node
       - k8s_cluster
-  - cloud_image: debian-12
+  - cloud_image: debian-11
-    name: debian12
+    name: debian11
     vm_cpu_cores: 1
     vm_memory: 1024
     node_groups:
@@ -35,6 +35,5 @@ provisioner:
       timeout: 120
   playbooks:
     create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml
-    prepare: ../../../molecule/prepare.yml
 verifier:
-  name: ansible
+  name: testinfra

roles/container-engine/containerd/molecule/default/prepare.yml

@@ -0,0 +1,30 @@
+---
+- name: Prepare
+  hosts: all
+  gather_facts: false
+  become: true
+  vars:
+    ignore_assert_errors: true
+  roles:
+    - role: kubespray_defaults
+    - role: bootstrap_os
+    - role: network_facts
+    - role: kubernetes/preinstall
+    - role: adduser
+      user: "{{ addusers.kube }}"
+  tasks:
+    - name: Download CNI
+      include_tasks: "../../../../download/tasks/download_file.yml"
+      vars:
+        download: "{{ download_defaults | combine(downloads.cni) }}"
+
+- name: Prepare CNI
+  hosts: all
+  gather_facts: false
+  become: true
+  vars:
+    ignore_assert_errors: true
+    kube_network_plugin: cni
+  roles:
+    - role: kubespray_defaults
+    - role: network_plugin/cni

roles/container-engine/containerd/molecule/default/tests/test_default.py

@@ -0,0 +1,55 @@
+import os
+import pytest
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_service(host):
+    svc = host.service("containerd")
+    assert svc.is_running
+    assert svc.is_enabled
+
+
+def test_version(host):
+    crictl = "/usr/local/bin/crictl"
+    path = "unix:///var/run/containerd/containerd.sock"
+    with host.sudo():
+        cmd = host.command(crictl + " --runtime-endpoint " + path + " version")
+    assert cmd.rc == 0
+    assert "RuntimeName:  containerd" in cmd.stdout
+
+
+@pytest.mark.parametrize('image, dest', [
+    ('quay.io/kubespray/hello-world:latest', '/tmp/hello-world.tar')
+])
+def test_image_pull_save_load(host, image, dest):
+    nerdctl = "/usr/local/bin/nerdctl"
+    dest_file = host.file(dest)
+
+    with host.sudo():
+        pull_cmd = host.command(nerdctl + " pull " + image)
+    assert pull_cmd.rc ==0
+
+    with host.sudo():
+        save_cmd = host.command(nerdctl + " save -o " + dest + " " + image)
+    assert save_cmd.rc == 0
+    assert dest_file.exists
+
+    with host.sudo():
+        load_cmd = host.command(nerdctl + " load < " + dest)
+    assert load_cmd.rc == 0
+
+
+@pytest.mark.parametrize('image', [
+    ('quay.io/kubespray/hello-world:latest')
+])
+def test_run(host, image):
+    nerdctl = "/usr/local/bin/nerdctl"
+
+    with host.sudo():
+        cmd = host.command(nerdctl + " -n k8s.io run " + image)
+    assert cmd.rc == 0
+    assert "Hello from Docker" in cmd.stdout

roles/container-engine/containerd/molecule/default/verify.yml

@@ -1,39 +0,0 @@
----
-- name: Test containerd CRI
-  import_playbook: ../../../molecule/test_cri.yml
-  vars:
-    container_manager: containerd
-    cri_socket: unix:///var/run/containerd/containerd.sock
-    cri_name: containerd
-
-- name: Test nerdctl
-  hosts: all
-  gather_facts: false
-  become: true
-  tasks:
-  - name: Get kubespray defaults
-    import_role:
-      name: ../../../../../kubespray_defaults
-  - name: Test nerdctl commands
-    command: "{{ bin_dir }}/nerdctl {{ item | join(' ') }}"
-    vars:
-      image: quay.io/kubespray/hello-world:latest
-    loop:
-    - - pull
-      - "{{ image }}"
-    - - save
-      - -o
-      - /tmp/hello-world.tar
-      - "{{ image }}"
-    - - load
-      - -i
-      - /tmp/hello-world.tar
-    - - -n
-      - k8s.io
-      - run
-      - "{{ image }}"
-    register: nerdctl
-  - name: Check log from running a container
-    assert:
-      that:
-      - ('Hello from Docker' in nerdctl.results[3].stdout)

roles/container-engine/containerd/tasks/main.yml

@@ -71,8 +71,6 @@
   notify: Restart containerd
 
 - name: Containerd | Configure containerd registries
-  # mirror configuration can contain sensitive information on headers configuration
-  no_log: "{{ not (unsafe_show_logs | bool) }}"
   block:
     - name: Containerd | Create registry directories
       file:

roles/container-engine/containerd/templates/config.toml.j2

@@ -36,15 +36,6 @@ oom_score = {{ containerd_oom_score }}
     enable_cdi = true
     cdi_spec_dirs = ["/etc/cdi", "/var/run/cdi"]
 {% endif %}
-{% for key, value in containerd_extra_runtime_args.items() %}
-{% if value is string %}
-    {{ key }} = "{{ value }}"
-{% elif value is boolean %}
-    {{ key }} = {{ value | lower }}
-{% else %}
-    {{ key }} = {{ value }}
-{% endif %}
-{% endfor %}
 
    [plugins."io.containerd.cri.v1.runtime".containerd]
      default_runtime_name = "{{ containerd_default_runtime }}"

roles/container-engine/containerd/templates/hosts.toml.j2

@@ -10,10 +10,4 @@ server = "{{ item.server | default("https://" + item.prefix) }}"
 {% if mirror.client is defined %}
   client = [{% for pair in mirror.client %}["{{ pair[0] }}", "{{ pair[1] }}"]{% if not loop.last %},{% endif %}{% endfor %}]
 {% endif %}
-{% if mirror.header is defined %}
-  [host."{{ mirror.host }}".header]
-{% for key, value in mirror.header.items() %}
-    {{ key }} = ["{{ ([ value ] | flatten ) | join('","') }}"]
-{% endfor %}
-{% endif %}
 {% endfor %}

roles/container-engine/cri-dockerd/molecule/default/molecule.yml

@@ -7,8 +7,8 @@ platforms:
     vm_memory: 1024
     node_groups:
       - kube_control_plane
-  - name: ubuntu22
+  - name: ubuntu20
-    cloud_image: ubuntu-2204
+    cloud_image: ubuntu-2004
     vm_cpu_cores: 1
     vm_memory: 1024
     node_groups:
@@ -27,6 +27,5 @@ provisioner:
         become: true
   playbooks:
     create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml
-    prepare: ../../../molecule/prepare.yml
 verifier:
-  name: ansible
+  name: testinfra

roles/container-engine/cri-dockerd/molecule/default/prepare.yml

@@ -0,0 +1,48 @@
+---
+- name: Prepare
+  hosts: all
+  become: true
+  roles:
+    - role: kubespray_defaults
+    - role: bootstrap_os
+    - role: adduser
+      user: "{{ addusers.kube }}"
+  tasks:
+    - name: Download CNI
+      include_tasks: "../../../../download/tasks/download_file.yml"
+      vars:
+        download: "{{ download_defaults | combine(downloads.cni) }}"
+
+- name: Prepare container runtime
+  hosts: all
+  become: true
+  vars:
+    container_manager: containerd
+    kube_network_plugin: cni
+  roles:
+    - role: kubespray_defaults
+    - role: network_plugin/cni
+  tasks:
+    - name: Copy test container files
+      copy:
+        src: "{{ item }}"
+        dest: "/tmp/{{ item }}"
+        owner: root
+        mode: "0644"
+      with_items:
+        - container.json
+        - sandbox.json
+    - name: Create /etc/cni/net.d directory
+      file:
+        path: /etc/cni/net.d
+        state: directory
+        owner: "{{ kube_owner }}"
+        mode: "0755"
+    - name: Setup CNI
+      copy:
+        src: "{{ item }}"
+        dest: "/etc/cni/net.d/{{ item }}"
+        owner: root
+        mode: "0644"
+      with_items:
+        - 10-mynet.conf

roles/container-engine/cri-dockerd/molecule/default/tests/test_default.py

@@ -0,0 +1,19 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_run_pod(host):
+    run_command = "/usr/local/bin/crictl run --with-pull /tmp/container.json /tmp/sandbox.json"
+    with host.sudo():
+        cmd = host.command(run_command)
+    assert cmd.rc == 0
+
+    with host.sudo():
+      log_f = host.file("/tmp/cri-dockerd1.0.log")
+
+      assert log_f.exists
+      assert b"Hello from Docker" in log_f.content

roles/container-engine/cri-dockerd/molecule/default/verify.yml

@@ -1,15 +0,0 @@
----
-- name: Test cri-dockerd
-  import_playbook: ../../../molecule/test_cri.yml
-  vars:
-    container_manager: cri-dockerd
-    cri_socket: unix:///var/run/cri-dockerd.sock
-    cri_name: docker
-
-- name: Test running a container with docker
-  import_playbook: ../../../molecule/test_runtime.yml
-  vars:
-    container_runtime: docker
-  # cri-dockerd does not support multiple runtime handler before 0.4.0
-  # https://github.com/Mirantis/cri-dockerd/pull/350
-  # TODO: check this when we upgrade cri-dockerd

roles/container-engine/cri-o/defaults/main.yml

@@ -4,7 +4,6 @@ crio_cgroup_manager: "{{ kubelet_cgroup_driver | default('systemd') }}"
 crio_conmon: "{{ bin_dir }}/conmon"
 crio_default_runtime: "crun"
 crio_libexec_dir: "/usr/libexec/crio"
-crio_runtime_switch: false
 crio_enable_metrics: false
 crio_log_level: "info"
 crio_metrics_port: "9090"
@@ -45,7 +44,7 @@ crio_root: "/var/lib/containers/storage"
 # The crio_runtimes variable defines a list of OCI compatible runtimes.
 crio_runtimes:
   - name: crun
-    path: "{{ crio_runtime_bin_dir }}/crun"  # Use crun in cri-o distributions, don't use 'crun' role
+    path: "{{ crio_runtime_bin_dir }}/crun"
     type: oci
     root: /run/crun
 
@@ -112,4 +111,3 @@ crio_default_capabilities:
   - SETPCAP
   - NET_BIND_SERVICE
   - KILL
-crio_additional_mounts: []

roles/container-engine/cri-o/files/mounts.conf

@@ -0,0 +1 @@
+/usr/share/rhel/secrets:/run/secrets

roles/container-engine/cri-o/meta/main.yml

@@ -1,4 +1,5 @@
 ---
 dependencies:
+  - role: container-engine/crun
   - role: container-engine/crictl
   - role: container-engine/skopeo

roles/container-engine/cri-o/molecule/default/files/container.json

@@ -1,10 +1,10 @@
 {
   "metadata": {
-    "name": "{{ container_runtime }}1"
+    "name": "runc1"
   },
   "image": {
     "image": "quay.io/kubespray/hello-world:latest"
   },
-  "log_path": "{{ container_runtime }}1.0.log",
+  "log_path": "runc1.0.log",
   "linux": {}
 }

roles/container-engine/cri-o/molecule/default/files/sandbox.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "name": "{{ container_runtime }}1",
+    "name": "runc1",
     "namespace": "default",
     "attempt": 1,
     "uid": "hdishd83djaidwnduwk28bcsb"

roles/container-engine/cri-o/molecule/default/molecule.yml

@@ -1,8 +1,8 @@
 ---
 role_name_check: 1
 platforms:
-  - name: ubuntu22
+  - name: ubuntu20
-    cloud_image: ubuntu-2204
+    cloud_image: ubuntu-2004
     vm_cpu_cores: 2
     vm_memory: 1024
     node_groups:
@@ -43,6 +43,5 @@ provisioner:
       timeout: 120
   playbooks:
     create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml
-    prepare: ../../../molecule/prepare.yml
 verifier:
-  name: ansible
+  name: testinfra

roles/container-engine/cri-o/molecule/default/prepare.yml

@@ -6,7 +6,7 @@
   vars:
     ignore_assert_errors: true
   roles:
-    - role: dynamic_groups
+    - role: kubespray_defaults
     - role: bootstrap_os
     - role: network_facts
     - role: kubernetes/preinstall
@@ -14,7 +14,7 @@
       user: "{{ addusers.kube }}"
   tasks:
     - name: Download CNI
-      include_tasks: "../../download/tasks/download_file.yml"
+      include_tasks: "../../../../download/tasks/download_file.yml"
       vars:
         download: "{{ download_defaults | combine(downloads.cni) }}"
 
@@ -29,15 +29,26 @@
     - role: kubespray_defaults
     - role: network_plugin/cni
   tasks:
+    - name: Copy test container files
+      copy:
+        src: "{{ item }}"
+        dest: "/tmp/{{ item }}"
+        owner: root
+        mode: "0644"
+      with_items:
+        - container.json
+        - sandbox.json
     - name: Create /etc/cni/net.d directory
       file:
         path: /etc/cni/net.d
         state: directory
-        owner: root
+        owner: "{{ kube_owner }}"
         mode: "0755"
-    - name: Config bridge host-local CNI
+    - name: Setup CNI
       copy:
-        src: "10-mynet.conf"
+        src: "{{ item }}"
-        dest: "/etc/cni/net.d/"
+        dest: "/etc/cni/net.d/{{ item }}"
         owner: root
         mode: "0644"
+      with_items:
+        - 10-mynet.conf

roles/container-engine/cri-o/molecule/default/tests/test_default.py

@@ -0,0 +1,35 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_service(host):
+    svc = host.service("crio")
+    assert svc.is_running
+    assert svc.is_enabled
+
+
+def test_run(host):
+    crictl = "/usr/local/bin/crictl"
+    path = "unix:///var/run/crio/crio.sock"
+    with host.sudo():
+        cmd = host.command(crictl + " --runtime-endpoint " + path + " version")
+    assert cmd.rc == 0
+    assert "RuntimeName:  cri-o" in cmd.stdout
+
+def test_run_pod(host):
+    runtime = "crun"
+
+    run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime)
+    with host.sudo():
+        cmd = host.command(run_command)
+    assert cmd.rc == 0
+
+    with host.sudo():
+      log_f = host.file("/tmp/runc1.0.log")
+
+      assert log_f.exists
+      assert b"Hello from Docker" in log_f.content

roles/container-engine/cri-o/molecule/default/verify.yml

@@ -1,11 +0,0 @@
----
-- name: Test CRI-O cri
-  import_playbook: ../../../molecule/test_cri.yml
-  vars:
-    container_manager: crio
-    cri_socket: unix:///var/run/crio/crio.sock
-    cri_name: cri-o
-- name: Test running a container with crun
-  import_playbook: ../../../molecule/test_runtime.yml
-  vars:
-    container_runtime: crun

roles/container-engine/cri-o/tasks/main.yaml

@@ -55,46 +55,6 @@
   when:
     - youki_enabled
 
-- name: Cri-o | Stop kubelet service if running
-  service:
-    name: kubelet
-    state: stopped
-  when:
-    - crio_runtime_switch
-    - ansible_facts.services['kubelet.service'] is defined and ansible_facts.services['kubelet.service'].state == 'running'
-
-- name: Cri-o | Get all pods
-  ansible.builtin.command: "{{ bin_dir }}/crictl pods -o json"
-  changed_when: false
-  register: crio_pods
-  when:
-    - crio_runtime_switch
-    - ansible_facts.services['crio.service'] is defined
-
-- name: Cri-o | Stop and remove pods not on host network
-  ansible.builtin.command: "{{ bin_dir }}/crictl rmp -f {{ item.id }}"
-  loop: "{{ (crio_pods.stdout | from_json).items | default([]) | selectattr('metadata.namespace', 'ne', 'NODE') }}"
-  changed_when: true
-  when:
-    - crio_runtime_switch
-    - ansible_facts.services['crio.service'] is defined
-    - crio_pods.stdout is defined
-
-- name: Cri-o | Stop and remove all remaining pods
-  ansible.builtin.command: "{{ bin_dir }}/crictl rmp -fa"
-  changed_when: true
-  when:
-    - crio_runtime_switch
-    - ansible_facts.services['crio.service'] is defined
-
-- name: Cri-o | stop crio service if running
-  service:
-    name: crio
-    state: stopped
-  when:
-    - crio_runtime_switch
-    - ansible_facts.services['crio.service'] is defined and ansible_facts.services['crio.service'].state == 'running'
-
 - name: Cri-o | make sure needed folders exist in the system
   with_items:
     - /etc/crio
@@ -184,8 +144,8 @@
   notify: Restart crio
 
 - name: Cri-o | copy mounts.conf
-  template:
+  copy:
-    src: mounts.conf.j2
+    src: mounts.conf
     dest: /etc/containers/mounts.conf
     mode: "0644"
   when:
@@ -290,16 +250,3 @@
   changed_when: false
   retries: 5
   delay: "{{ retry_stagger | random + 3 }}"
-
-# The kubelet service status can be 'not-found' if something depends on it.
-# This check prevents failures when the service is in this indeterminate state,
-# which can occur when adding new nodes to a cluster.
-# See: https://superuser.com/questions/1755211/cleaning-debugging-services/1755215#1755215
-
-- name: Cri-o | ensure kubelet service is started if present and stopped
-  service:
-    name: kubelet
-    state: started
-  when:
-    - crio_runtime_switch
-    - ansible_facts.services['kubelet.service'] is defined and ansible_facts.services['kubelet.service']['status'] != 'not-found'

roles/container-engine/cri-o/templates/crio.conf.j2

@@ -204,7 +204,7 @@ default_mounts = [
 # limit is never exceeded.
 log_size_max = -1
 
-# Whether container output should be logged to journald in addition to the kubernetes log file
+# Whether container output should be logged to journald in addition to the kuberentes log file
 log_to_journald = false
 
 # Path to directory in which container exit files are written to by conmon.

roles/container-engine/cri-o/templates/mounts.conf.j2

@@ -1,4 +0,0 @@
-/usr/share/rhel/secrets:/run/secrets
-{% for mount in crio_additional_mounts %}
-{{ mount }}
-{% endfor %}

roles/container-engine/docker/defaults/main.yml

@@ -1,5 +1,5 @@
 ---
-docker_version: '28.3'
+docker_version: '28.0'
 docker_cli_version: "{{ docker_version }}"
 
 docker_package_info:

roles/container-engine/docker/tasks/main.yml

@@ -60,17 +60,16 @@
   environment: "{{ proxy_env }}"
   when: ansible_pkg_mgr == 'apt'
 
-# ref to https://github.com/kubernetes-sigs/kubespray/issues/11086 & 12424
+# ref to https://github.com/kubernetes-sigs/kubespray/issues/11086
-- name: Convert -backports sources to archive.debian.org for bullseye and older
+- name: Remove the archived debian apt repository
-  replace:
+  lineinfile:
-    path: "{{ item }}"
+    path: /etc/apt/sources.list
-    regexp: '^(deb(?:-src)?\s+)(?:https?://)?(?:[^ ]+debian\.org)?([^ ]*/debian)(\s+{{ ansible_distribution_release }}-backports\b.*)'
+    regexp: 'buster-backports'
-    replace: '\1http://archive.debian.org/debian\3'
+    state: absent
     backup: true
-  loop: "{{ query('fileglob', '/etc/apt/sources.list') }}"
   when:
     - ansible_os_family == 'Debian'
-    - ansible_distribution_release in ['bullseye', 'buster']
+    - ansible_distribution_release == "buster"
 
 - name: Ensure docker-ce repository is enabled
   apt_repository:

roles/container-engine/docker/vars/debian.yml

@@ -34,10 +34,8 @@ containerd_versioned_pkg:
   '1.7.23': "{{ containerd_package }}=1.7.23-1"
   '1.7.24': "{{ containerd_package }}=1.7.24-1"
   '1.7.25': "{{ containerd_package }}=1.7.25-1"
-  '1.7.26': "{{ containerd_package }}=1.7.26-1"
+  'stable': "{{ containerd_package }}=1.7.25-1"
-  '1.7.27': "{{ containerd_package }}=1.7.27-1"
+  'edge': "{{ containerd_package }}=1.7.25-1"
-  'stable': "{{ containerd_package }}=1.7.27-1"
-  'edge': "{{ containerd_package }}=1.7.27-1"
 
 # https://download.docker.com/linux/debian/
 docker_versioned_pkg:
@@ -55,13 +53,10 @@ docker_versioned_pkg:
   '27.2': docker-ce=5:27.2.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
   '27.3': docker-ce=5:27.3.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
   '27.4': docker-ce=5:27.4.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  '27.5': docker-ce=5:27.5.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '27.5': docker-ce=5:27.5.4-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  '28.0': docker-ce=5:28.0.4-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '28.0': docker-ce=5:28.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  '28.1': docker-ce=5:28.1.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  'stable': docker-ce=5:28.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  '28.2': docker-ce=5:28.2.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  'edge': docker-ce=5:28.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  '28.3': docker-ce=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  'stable': docker-ce=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  'edge': docker-ce=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
 
 docker_cli_versioned_pkg:
   'latest': docker-ce-cli
@@ -78,13 +73,10 @@ docker_cli_versioned_pkg:
   '27.2': docker-ce-cli=5:27.2.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
   '27.3': docker-ce-cli=5:27.3.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
   '27.4': docker-ce-cli=5:27.4.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  '27.5': docker-ce-cli=5:27.5.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '27.5': docker-ce-cli=5:27.5.4-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  '28.0': docker-ce-cli=5:28.0.4-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  '28.0': docker-ce-cli=5:28.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  '28.1': docker-ce-cli=5:28.1.1-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  'stable': docker-ce-cli=5:28.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  '28.2': docker-ce-cli=5:28.2.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
+  'edge': docker-ce-cli=5:28.0.2-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  '28.3': docker-ce-cli=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  'stable': docker-ce-cli=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
-  'edge': docker-ce-cli=5:28.3.3-1~debian.{{ ansible_distribution_major_version }}~{{ ansible_distribution_release | lower }}
 
 docker_package_info:
   pkgs: