.ansible-lint

@@ -1,4 +1,5 @@
 ---
+parseable: true
 skip_list:
   # see https://docs.ansible.com/ansible-lint/rules/default_rules.html for a list of all default rules
 
@@ -11,12 +12,10 @@ skip_list:
   # (Disabled in June 2021)
   - 'role-name'
 
-  # [var-naming]
+  # [var-naming] "defaults/main.yml" File defines variable 'apiVersion' that violates variable naming standards
   # In Kubespray we use variables that use camelCase to match their k8s counterparts
   # (Disabled in June 2021)
-  - 'var-naming[pattern]'
+  - 'var-naming'
-  # Variables names from within roles in kubespray don't need role name as a prefix
-  - 'var-naming[no-role-prefix]'
 
   # [fqcn-builtins]
   # Roles in kubespray don't need fully qualified collection names
@@ -33,8 +32,6 @@ skip_list:
   # Disable run-once check with free strategy
   # (Disabled in June 2023 after ansible upgrade; FIXME)
   - 'run-once[task]'
-
-  - 'jinja[spacing]'
 exclude_paths:
   # Generated files
   - tests/files/custom_cni/cilium.yaml

.github/ISSUE_TEMPLATE/bug-report.yaml

@@ -108,6 +108,7 @@ body:
         - meta
         - multus
         - ovn4nfv
+        - weave
     validations:
       required: true
 

.github/workflows/auto-label-os.yml

@@ -13,16 +13,16 @@ jobs:
       issues: write
 
     steps:
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
 
       - name: Parse issue form
-        uses: stefanbuck/github-issue-parser@25f1485edffc1fee3ea68eb9f59a72e58720ffc4
+        uses: stefanbuck/github-issue-parser@2ea9b35a8c584529ed00891a8f7e41dc46d0441e
         id: issue-parser
         with:
           template-path: .github/ISSUE_TEMPLATE/bug-report.yaml
 
       - name: Set labels based on OS field
-        uses: redhat-plumbers-in-action/advanced-issue-labeler@b80ae64e3e156e9c111b075bfa04b295d54e8e2e
+        uses: redhat-plumbers-in-action/advanced-issue-labeler@39087a4b30cb98d57f25f34d617a6af8163c17d9
         with:
           issue-form: ${{ steps.issue-parser.outputs.jsonString }}
           section: os

.github/workflows/upgrade-patch-versions-schedule.yml

@@ -13,14 +13,14 @@ jobs:
     outputs:
       branches: ${{ steps.get-branches.outputs.data }}
     steps:
-    - uses: octokit/graphql-action@ddde8ebb2493e79f390e6449c725c21663a67505
+    - uses: octokit/graphql-action@8ad880e4d437783ea2ab17010324de1075228110
       id: get-branches
       with:
         query: |
           query get_release_branches($owner:String!, $name:String!) {
             repository(owner:$owner, name:$name) {
               refs(refPrefix: "refs/heads/",
-                   first: 2, # TODO increment once we have release branch with the new checksums format
+                   first: 0, # TODO increment once we have release branch with the new checksums format
                    query: "release-",
                    orderBy: {
                      field: ALPHABETICAL,

.github/workflows/upgrade-patch-versions.yml

@@ -11,10 +11,10 @@ jobs:
   update-patch-versions:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
       with:
         ref: ${{ inputs.branch }}
-    - uses: actions/setup-python@v6
+    - uses: actions/setup-python@v5
       with:
         python-version: '3.13'
         cache: 'pip'
@@ -22,14 +22,14 @@ jobs:
     - run: update-hashes
       env:
         API_KEY: ${{ secrets.GITHUB_TOKEN }}
-    - uses: actions/cache@v5
+    - uses: actions/cache@v4
       with:
         key: pre-commit-hook-propagate
         path: |
           ~/.cache/pre-commit
     - run: pre-commit run --all-files propagate-ansible-variables
       continue-on-error: true
-    - uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725
+    - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e
       with:
         commit-message: Patch versions updates
         title: Patch versions updates - ${{ inputs.branch }}

.gitlab-ci.yml

@@ -1,9 +1,9 @@
 ---
 stages:
-  - build # build docker image used in most other jobs
+  - build
-  - test # unit tests
+  - test
-  - deploy-part1 # kubespray runs - common setup
+  - deploy-part1
-  - deploy-extended # kubespray runs - rarer or costlier (to test) setups
+  - deploy-extended
 
 variables:
   FAILFASTCI_NAMESPACE: 'kargo-ci'
@@ -24,7 +24,6 @@ variables:
   ANSIBLE_REMOTE_USER: kubespray
   ANSIBLE_PRIVATE_KEY_FILE: /tmp/id_rsa
   ANSIBLE_INVENTORY: /tmp/inventory
-  ANSIBLE_STDOUT_CALLBACK: "default"
   RESET_CHECK: "false"
   REMOVE_NODE_CHECK: "false"
   UPGRADE_TEST: "false"
@@ -32,7 +31,7 @@ variables:
   ANSIBLE_VERBOSITY: 2
   RECOVER_CONTROL_PLANE_TEST: "false"
   RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
-  OPENTOFU_VERSION: v1.9.1
+  TF_VERSION: 1.3.7
   PIPELINE_IMAGE: "$CI_REGISTRY_IMAGE/pipeline:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"
 
 before_script:
@@ -49,6 +48,8 @@ before_script:
       - cluster-dump/
   needs:
     - pipeline-image
+  variables:
+    ANSIBLE_STDOUT_CALLBACK: "debug"
 
 .job-moderated:
   extends: .job

.gitlab-ci/build.yml

@@ -7,24 +7,23 @@ pipeline-image:
   tags:
     - ffci
   stage: build
-  image: moby/buildkit:rootless
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: ['']
   variables:
-    BUILDKITD_FLAGS: --oci-worker-no-process-sandbox
+    GODEBUG: "http2client=0"
-    CACHE_IMAGE: $CI_REGISTRY_IMAGE/pipeline:cache
   # TODO: remove the override
   # currently rebase.sh depends on bash (not available in the kaniko image)
   # once we have a simpler rebase (which should be easy if the target branch ref is available as variable
   # we'll be able to rebase here as well hopefully
-  before_script:
+  before_script: []
-    - mkdir -p ~/.docker
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > ~/.docker/config.json
   script:
-    - |
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json
-      buildctl-daemonless.sh build \
+    - /kaniko/executor --cache=true
-        --frontend dockerfile.v0 \
+                       --cache-dir=image-cache
-        --local context=$CI_PROJECT_DIR \
+                       --context $CI_PROJECT_DIR
-        --local dockerfile=$CI_PROJECT_DIR \
+                       --dockerfile $CI_PROJECT_DIR/pipeline.Dockerfile
-        --opt filename=pipeline.Dockerfile \
+                       --label 'git-branch'=$CI_COMMIT_REF_SLUG
-        --export-cache type=registry,ref=$CACHE_IMAGE \
+                       --label 'git-tag=$CI_COMMIT_TAG'
-        --import-cache type=registry,ref=$CACHE_IMAGE \
+                       --destination $PIPELINE_IMAGE
-        --output type=image,name=$PIPELINE_IMAGE,push=true
+                       --log-timestamp=true

.gitlab-ci/kubevirt.yml

@@ -4,7 +4,7 @@
   interruptible: true
   script:
     - ansible-playbook tests/cloud_playbooks/create-kubevirt.yml
-                              -e @"tests/files/${TESTCASE}.yml"
+                      -c local -e @"tests/files/${TESTCASE}.yml"
     - ./tests/scripts/testcases_run.sh
   variables:
     ANSIBLE_TIMEOUT: "120"
@@ -34,34 +34,34 @@ pr:
   parallel:
     matrix:
       - TESTCASE:
+          - almalinux8-calico
           - almalinux9-crio
           - almalinux9-kube-ovn
           - debian11-calico-collection
           - debian11-macvlan
           - debian12-cilium
-          - debian13-cilium
           - fedora39-kube-router
+            # FIXME: this test if broken (perma-failing)
           - openeuler24-calico
+          - opensuse15-6-calico
+          - rockylinux8-calico
           - rockylinux9-cilium
-          - rockylinux10-cilium
+          - ubuntu20-calico-all-in-one-hardening
+          - ubuntu20-cilium-sep
+          - ubuntu20-flannel-collection
+          - ubuntu20-kube-router-sep
+          - ubuntu20-kube-router-svc-proxy
           - ubuntu22-calico-all-in-one
           - ubuntu22-calico-all-in-one-upgrade
           - ubuntu24-calico-etcd-datastore
-          - ubuntu24-calico-all-in-one-hardening
-          - ubuntu24-cilium-sep
-          - ubuntu24-flannel-collection
-          - ubuntu24-kube-router-sep
-          - ubuntu24-kube-router-svc-proxy
           - ubuntu24-ha-separate-etcd
-          - flatcar4081-calico
-          - fedora40-flannel-crio-collection-scale
 
-# The ubuntu24-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
+# The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
-ubuntu24-calico-all-in-one:
+ubuntu20-calico-all-in-one:
   stage: deploy-part1
   extends: .kubevirt
   variables:
-    TESTCASE: ubuntu24-calico-all-in-one
+    TESTCASE: ubuntu20-calico-all-in-one
   rules:
     - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
       when: on_success
@@ -91,9 +91,10 @@ pr_full:
           - debian12-custom-cni-helm
           - fedora39-calico-swap-selinux
           - fedora39-crio
-          - ubuntu24-calico-ha-wireguard
+          - ubuntu20-all-in-one-docker
-          - ubuntu24-flannel-ha
+          - ubuntu20-calico-ha-wireguard
-          - ubuntu24-flannel-ha-once
+          - ubuntu20-flannel-ha
+          - ubuntu20-flannel-ha-once
 
 # Need an update of the container image to use schema v2
 # update: quay.io/kubespray/vm-amazon-linux-2:latest
@@ -126,14 +127,13 @@ pr_extended:
           - debian11-docker
           - debian12-calico
           - debian12-docker
-          - debian13-calico
+          - opensuse15-6-docker-cilium
           - rockylinux9-calico
-          - rockylinux10-calico
+          - ubuntu20-calico-etcd-kubeadm
+          - ubuntu20-flannel
           - ubuntu22-all-in-one-docker
           - ubuntu24-all-in-one-docker
           - ubuntu24-calico-all-in-one
-          - ubuntu24-calico-etcd-kubeadm
-          - ubuntu24-flannel
 
 # TODO: migrate to pr-full, fix the broken ones
 periodic:
@@ -150,6 +150,6 @@ periodic:
           - debian12-cilium-svc-proxy
           - fedora39-calico-selinux
           - fedora40-docker-calico
-          - ubuntu24-calico-etcd-kubeadm-upgrade-ha
+          - ubuntu20-calico-etcd-kubeadm-upgrade-ha
-          - ubuntu24-calico-ha-recover
+          - ubuntu20-calico-ha-recover
-          - ubuntu24-calico-ha-recover-noquorum
+          - ubuntu20-calico-ha-recover-noquorum

.gitlab-ci/lint.yml

@@ -6,7 +6,6 @@ pre-commit:
   image: 'ghcr.io/pre-commit-ci/runner-image@sha256:fe01a6ec51b298412990b88627c3973b1146c7304f930f469bafa29ba60bcde9'
   variables:
     PRE_COMMIT_HOME: ${CI_PROJECT_DIR}/.cache/pre-commit
-    ANSIBLE_STDOUT_CALLBACK: default
   script:
   - pre-commit run --all-files --show-diff-on-failure
   cache:
@@ -24,3 +23,4 @@ vagrant-validate:
     VAGRANT_VERSION: 2.3.7
   script:
   - ./tests/scripts/vagrant-validate.sh
+  except: ['triggers', 'master']

.gitlab-ci/molecule.yml

@@ -34,8 +34,6 @@ molecule:
       - container-engine/cri-dockerd
       - container-engine/containerd
       - container-engine/cri-o
-      - container-engine/gvisor
-      - container-engine/youki
       - adduser
       - bastion-ssh-config
       - bootstrap_os
@@ -53,3 +51,5 @@ molecule_full:
     - ROLE:
       # FIXME : tests below are perma-failing
       - container-engine/kata-containers
+      - container-engine/gvisor
+      - container-engine/youki

.gitlab-ci/terraform.yml

@@ -13,18 +13,18 @@
   before_script:
     - ./tests/scripts/rebase.sh
     - mkdir -p cluster-dump $ANSIBLE_INVENTORY
-    - ./tests/scripts/opentofu_install.sh
+    - ./tests/scripts/terraform_install.sh
     - cp contrib/terraform/$PROVIDER/sample-inventory/cluster.tfvars .
     - ln -rs -t $ANSIBLE_INVENTORY contrib/terraform/$PROVIDER/hosts
-    - tofu -chdir="contrib/terraform/$PROVIDER" init
+    - terraform -chdir="contrib/terraform/$PROVIDER" init
 
 terraform_validate:
   extends: .terraform_install
   tags: [ffci]
   only: ['master', /^pr-.*$/]
   script:
-    - tofu -chdir="contrib/terraform/$PROVIDER" validate
+    - terraform -chdir="contrib/terraform/$PROVIDER" validate
-    - tofu -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
+    - terraform -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
   stage: test
   needs:
     - pipeline-image
@@ -58,11 +58,11 @@ terraform_validate:
     - mkdir -p contrib/terraform/$PROVIDER/group_vars
     # Random subnet to avoid routing conflicts
     - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
-    - tofu -chdir="contrib/terraform/$PROVIDER" apply -auto-approve -parallelism=1
+    - terraform -chdir="contrib/terraform/$PROVIDER" apply -auto-approve -parallelism=1
     - tests/scripts/testcases_run.sh
   after_script:
     # Cleanup regardless of exit code
-    - tofu -chdir="contrib/terraform/$PROVIDER" destroy -auto-approve
+    - terraform -chdir="contrib/terraform/$PROVIDER" destroy -auto-approve
 
 # Elastx is generously donating resources for Kubespray on Openstack CI
 # Contacts: @gix @bl0m1
@@ -89,10 +89,11 @@ tf-elastx_cleanup:
     - ./scripts/openstack-cleanup/main.py
   allow_failure: true
 
-tf-elastx_ubuntu24-calico:
+tf-elastx_ubuntu20-calico:
   extends: .terraform_apply
   stage: deploy-part1
   when: on_success
+  allow_failure: true
   variables:
     <<: *elastx_variables
     PROVIDER: openstack
@@ -115,5 +116,5 @@ tf-elastx_ubuntu24-calico:
     TF_VAR_az_list_node: '["sto1"]'
     TF_VAR_flavor_k8s_master: 3f73fc93-ec61-4808-88df-2580d94c1a9b    # v1-standard-2
     TF_VAR_flavor_k8s_node: 3f73fc93-ec61-4808-88df-2580d94c1a9b      # v1-standard-2
-    TF_VAR_image: ubuntu-24.04-server-latest
+    TF_VAR_image: ubuntu-20.04-server-latest
     TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'

.gitlab-ci/vagrant.yml

@@ -36,12 +36,10 @@ vagrant:
     policy: pull-push # TODO: change to "pull" when not on main
   stage: deploy-extended
   rules:
-    - if: $PR_LABELS =~ /.*ci-full.*/
+    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
       when: on_success
     - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
       when: on_success
-    - when: manual
-      allow_failure: true
   parallel:
     matrix:
       - TESTCASE:

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v6.0.0
+    rev: v5.0.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -15,13 +15,13 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.37.1
+    rev: v1.35.1
     hooks:
       - id: yamllint
         args: [--strict]
 
   - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: v0.11.0.1
+    rev: v0.10.0.1
     hooks:
       - id: shellcheck
         args: ["--severity=error"]
@@ -29,7 +29,7 @@ repos:
         files: "\\.sh$"
 
   - repo: https://github.com/ansible/ansible-lint
-    rev: v25.11.0
+    rev: v25.1.1
     hooks:
       - id: ansible-lint
         additional_dependencies:
@@ -38,7 +38,7 @@ repos:
           - distlib
 
   - repo: https://github.com/golangci/misspell
-    rev: v0.7.0
+    rev: v0.6.0
     hooks:
       - id: misspell
         exclude: "OWNERS_ALIASES$"

CONTRIBUTING.md

@@ -40,7 +40,7 @@ Vagrant with VirtualBox or libvirt driver helps you to quickly spin test cluster
 2. The [repo owners](OWNERS) will respond to your issue promptly.
 3. Fork the desired repo, develop and test your code changes.
 4. Install [pre-commit](https://pre-commit.com) and install it in your development repo.
-5. Address any pre-commit validation failures.
+5. Addess any pre-commit validation failures.
 6. Sign the CNCF CLA (<https://git.k8s.io/community/CLA.md#the-contributor-license-agreement>)
 7. Submit a pull request.
 8. Work with the reviewers on their suggestions.

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1
 
-# Use immutable image tags rather than mutable tags (like ubuntu:22.04)
+# Use imutable image tags rather than mutable tags (like ubuntu:22.04)
 FROM ubuntu:22.04@sha256:149d67e29f765f4db62aa52161009e99e389544e25a8f43c8c89d4a445a7ca37
 
 # Some tools like yamllint need this
@@ -35,8 +35,8 @@ RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 RUN OS_ARCHITECTURE=$(dpkg --print-architecture) \
-    && curl -L "https://dl.k8s.io/release/v1.34.3/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
+    && curl -L "https://dl.k8s.io/release/v1.32.5/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
-    && echo "$(curl -L "https://dl.k8s.io/release/v1.34.3/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
+    && echo "$(curl -L "https://dl.k8s.io/release/v1.32.5/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
     && chmod a+x /usr/local/bin/kubectl
 
 COPY *.yml ./

OWNERS_ALIASES

@@ -2,12 +2,12 @@ aliases:
   kubespray-approvers:
     - ant31
     - mzaian
-    - tico88612
     - vannten
     - yankay
   kubespray-reviewers:
     - cyclinder
     - erikjiang
+    - mrfreezeex
     - mzaian
     - tico88612
     - vannten

README.md

@@ -22,7 +22,7 @@ Ensure you have installed Docker then
 ```ShellSession
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
   --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.29.0 bash
+  quay.io/kubespray/kubespray:v2.27.0 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```
@@ -87,15 +87,15 @@ vagrant up
 ## Supported Linux Distributions
 
 - **Flatcar Container Linux by Kinvolk**
-- **Debian** Bookworm, Bullseye, Trixie
+- **Debian** Bookworm, Bullseye
-- **Ubuntu** 22.04, 24.04
+- **Ubuntu** 20.04, 22.04, 24.04
-- **CentOS Stream / RHEL** [9, 10](docs/operating_systems/rhel.md#rhel-8)
+- **CentOS/RHEL** [8, 9](docs/operating_systems/rhel.md#rhel-8)
 - **Fedora** 39, 40
 - **Fedora CoreOS** (see [fcos Note](docs/operating_systems/fcos.md))
 - **openSUSE** Leap 15.x/Tumbleweed
-- **Oracle Linux** [9, 10](docs/operating_systems/rhel.md#rhel-8)
+- **Oracle Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
-- **Alma Linux** [9, 10](docs/operating_systems/rhel.md#rhel-8)
+- **Alma Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
-- **Rocky Linux** [9, 10](docs/operating_systems/rhel.md#rhel-8) (experimental in 10: see [Rocky Linux 10 notes](docs/operating_systems/rhel.md#rocky-linux-10))
+- **Rocky Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
 - **Kylin Linux Advanced Server V10** (experimental: see [kylin linux notes](docs/operating_systems/kylinlinux.md))
 - **Amazon Linux 2** (experimental: see [amazon linux notes](docs/operating_systems/amazonlinux.md))
 - **UOS Linux** (experimental: see [uos linux notes](docs/operating_systems/uoslinux.md))
@@ -111,26 +111,27 @@ Note:
 <!-- BEGIN ANSIBLE MANAGED BLOCK -->
 
 - Core
-  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.34.3
+  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.32.5
-  - [etcd](https://github.com/etcd-io/etcd) 3.5.26
+  - [etcd](https://github.com/etcd-io/etcd) 3.5.16
-  - [docker](https://www.docker.com/) 28.3
+  - [docker](https://www.docker.com/) 28.0
-  - [containerd](https://containerd.io/) 2.2.1
+  - [containerd](https://containerd.io/) 2.0.5
-  - [cri-o](http://cri-o.io/) 1.34.3 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
+  - [cri-o](http://cri-o.io/) 1.32.0 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
-  - [cni-plugins](https://github.com/containernetworking/plugins) 1.8.0
+  - [cni-plugins](https://github.com/containernetworking/plugins) 1.4.1
-  - [calico](https://github.com/projectcalico/calico) 3.30.5
+  - [calico](https://github.com/projectcalico/calico) 3.29.3
-  - [cilium](https://github.com/cilium/cilium) 1.18.5
+  - [cilium](https://github.com/cilium/cilium) 1.17.3
-  - [flannel](https://github.com/flannel-io/flannel) 0.27.3
+  - [flannel](https://github.com/flannel-io/flannel) 0.22.0
   - [kube-ovn](https://github.com/alauda/kube-ovn) 1.12.21
   - [kube-router](https://github.com/cloudnativelabs/kube-router) 2.1.1
-  - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) 4.2.2
+  - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) 4.1.0
-  - [kube-vip](https://github.com/kube-vip/kube-vip) 1.0.3
+  - [weave](https://github.com/rajch/weave) 2.8.7
+  - [kube-vip](https://github.com/kube-vip/kube-vip) 0.8.0
 - Application
   - [cert-manager](https://github.com/jetstack/cert-manager) 1.15.3
-  - [coredns](https://github.com/coredns/coredns) 1.12.1
+  - [coredns](https://github.com/coredns/coredns) 1.11.3
-  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) 1.13.3
+  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) 1.12.1
   - [argocd](https://argoproj.github.io/) 2.14.5
-  - [helm](https://helm.sh/) 3.18.4
+  - [helm](https://helm.sh/) 3.16.4
   - [metallb](https://metallb.universe.tf/) 0.13.9
   - [registry](https://github.com/distribution/distribution) 2.8.1
 - Storage Plugin
@@ -138,7 +139,7 @@ Note:
   - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) 1.10.0
   - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) 1.30.0
   - [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) 1.9.2
-  - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) 0.0.32
+  - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) 0.0.24
   - [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) 2.5.0
   - [node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) 0.16.4
 
@@ -182,6 +183,9 @@ You can choose among ten network plugins. (default: `calico`, except Vagrant use
 
 - [cilium](http://docs.cilium.io/en/latest/): layer 3/4 networking (as well as layer 7 to protect and secure application protocols), supports dynamic insertion of BPF bytecode into the Linux kernel to implement security services, networking and visibility logic.
 
+- [weave](docs/CNI/weave.md): Weave is a lightweight container overlay network that doesn't require an external K/V database cluster.
+    (Please refer to `weave` [troubleshooting documentation](https://www.weave.works/docs/net/latest/troubleshooting/)).
+
 - [kube-ovn](docs/CNI/kube-ovn.md): Kube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It offers an advanced Container Network Fabric for Enterprises.
 
 - [kube-router](docs/CNI/kube-router.md): Kube-router is a L3 CNI for Kubernetes networking aiming to provide operational

Vagrantfile

@@ -4,8 +4,6 @@
 # For help on using kubespray with vagrant, check out docs/developers/vagrant.md
 
 require 'fileutils'
-require 'ipaddr'
-require 'socket'
 
 Vagrant.require_version ">= 2.0.0"
 
@@ -101,33 +99,6 @@ $extra_vars ||= {}
 
 host_vars = {}
 
-def collect_networks(subnet, subnet_ipv6)
-  Socket.getifaddrs.filter_map do |iface|
-    next unless iface&.netmask&.ip_address && iface.addr
-
-    is_ipv6 = iface.addr.ipv6?
-    ip      = IPAddr.new(iface.addr.ip_address.split('%').first)
-    ip_test = is_ipv6 ? IPAddr.new("#{subnet_ipv6}::0") : IPAddr.new("#{subnet}.0")
-
-    prefix  = IPAddr.new(iface.netmask.ip_address).to_i.to_s(2).count('1')
-    network = ip.mask(prefix)
-
-    [IPAddr.new("#{network}/#{prefix}"), ip_test]
-  end
-end
-
-def subnet_in_use?(network_ips)
-  network_ips.any? { |net, test_ip| net.include?(test_ip) && test_ip != net }
-end
-
-network_ips = collect_networks($subnet, $subnet_ipv6)
-
-if subnet_in_use?(network_ips)
-  puts "Invalid subnet provided, subnet is already in use: #{$subnet}.0"
-  puts "Subnets in use: #{network_ips.inspect}"
-  exit 1
-end
-
 # throw error if os is not supported
 if ! SUPPORTED_OS.key?($os)
   puts "Unsupported OS: #{$os}"

ansible.cfg

@@ -15,7 +15,7 @@ timeout = 300
 stdout_callback = default
 display_skipped_hosts = no
 library = ./library
-callbacks_enabled = profile_tasks
+callbacks_enabled = profile_tasks,ara_default
 roles_path = roles:$VIRTUAL_ENV/usr/local/share/kubespray/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:/usr/share/kubespray/roles
 deprecation_warnings=False
 inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo, .creds, .gpg

contrib/collection.sh

@@ -1,9 +0,0 @@
-#!/bin/bash -eux
-# Install collection from source assuming dependencies are present.
-# Run in SemaphoreUI this bash script can install Kubespray from the repo
-NAMESPACE=kubernetes_sigs
-COLLECTION=kubespray
-MY_VER=$(grep '^version:' galaxy.yml|cut -d: -f2|sed 's/ //')
-
-ansible-galaxy collection build --force --output-path .
-ansible-galaxy collection install --offline --force $NAMESPACE-$COLLECTION-$MY_VER.tar.gz

contrib/offline/manage-offline-container-images.sh

@@ -20,6 +20,7 @@ function create_container_image_tar() {
 
 		kubectl describe cronjobs,jobs,pods --all-namespaces | grep " Image:" | awk '{print $2}' | sort | uniq > "${IMAGES}"
 		# NOTE: etcd and pause cannot be seen as pods.
+		# The pause image is used for --pod-infra-container-image option of kubelet.
 		kubectl cluster-info dump | grep -E "quay.io/coreos/etcd:|registry.k8s.io/pause:" | sed s@\"@@g >> "${IMAGES}"
 	else
 		echo "Getting images from file \"${IMAGES_FROM_FILE}\""
@@ -35,7 +36,7 @@ function create_container_image_tar() {
 	mkdir  ${IMAGE_DIR}
 	cd     ${IMAGE_DIR}
 
-	sudo --preserve-env=http_proxy,https_proxy,no_proxy ${runtime} pull registry:latest
+	sudo ${runtime} pull registry:latest
 	sudo ${runtime} save -o registry-latest.tar registry:latest
 
 	while read -r image
@@ -44,7 +45,7 @@ function create_container_image_tar() {
 		set +e
 		for step in $(seq 1 ${RETRY_COUNT})
 		do
-			sudo --preserve-env=http_proxy,https_proxy,no_proxy ${runtime} pull ${image}
+			sudo ${runtime} pull ${image}
 			if [ $? -eq 0 ]; then
 				break
 			fi
@@ -147,7 +148,7 @@ function register_container_images() {
 		if [ "${org_image}" == "ID:" ]; then
 		  org_image=$(echo "${load_image}"  | awk '{print $4}')
 		fi
-		image_id=$(sudo ${runtime} image inspect --format "{{.Id}}" "${org_image}")
+		image_id=$(sudo ${runtime} image inspect ${org_image} | grep "\"Id\":" | awk -F: '{print $3}'| sed s/'\",'//)
 		if [ -z "${file_name}" ]; then
 			echo "Failed to get file_name for line ${line}"
 			exit 1

contrib/offline/manage-offline-files.sh

@@ -41,7 +41,7 @@ fi
 
 sudo "${runtime}" container inspect nginx >/dev/null 2>&1
 if [ $? -ne 0 ]; then
-    sudo --preserve-env=http_proxy,https_proxy,no_proxy "${runtime}" run \
+    sudo "${runtime}" run \
         --restart=always -d -p ${NGINX_PORT}:80 \
         --volume "${OFFLINE_FILES_DIR}":/usr/share/nginx/html/download \
         --volume "${CURRENT_DIR}"/nginx.conf:/etc/nginx/nginx.conf \

contrib/terraform/aws/create-infrastructure.tf

@@ -1,11 +1,5 @@
 terraform {
   required_version = ">= 0.12.0"
-  required_providers {
-    aws = {
-      source  = "hashicorp/aws"
-      version = "~> 5.0"
-    }
-  }
 }
 
 provider "aws" {

contrib/terraform/openstack/README.md

@@ -281,9 +281,9 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
 |`k8s_allowed_remote_ips_ipv6` | List of IPv6 CIDR allowed to initiate a SSH connection, empty by default |
 |`k8s_allowed_egress_ipv6_ips` | List of IPv6 CIDRs allowed for egress traffic, `["::/0"]` by default |
 |`worker_allowed_ports` | List of ports to open on worker nodes, `[{ "protocol" = "tcp", "port_range_min" = 30000, "port_range_max" = 32767, "remote_ip_prefix" = "0.0.0.0/0"}]` by default |
-|`worker_allowed_ports_ipv6` | List of ports to open on worker nodes for IPv6 CIDR blocks, `[{ "protocol" = "tcp", "port_range_min" = 30000, "port_range_max" = 32767, "remote_ip_prefix" = "::/0"}, { "protocol" = "ipv6-icmp", "port_range_min" = 0, "port_range_max" = 0, "remote_ip_prefix" = "::/0"}]` by default |
+|`worker_allowed_ports_ipv6` | List of ports to open on worker nodes for IPv6 CIDR blocks, `[{ "protocol" = "tcp", "port_range_min" = 30000, "port_range_max" = 32767, "remote_ip_prefix" = "::/0"}]` by default |
 |`master_allowed_ports` | List of ports to open on master nodes, expected format is `[{ "protocol" = "tcp", "port_range_min" = 443, "port_range_max" = 443, "remote_ip_prefix" = "0.0.0.0/0"}]`, empty by default |
-|`master_allowed_ports_ipv6` | List of ports to open on master nodes for IPv6 CIDR blocks, `[{ "protocol" = "ipv6-icmp", "port_range_min" = 0, "port_range_max" = 0, "remote_ip_prefix" = "::/0"}]` by default |
+|`master_allowed_ports_ipv6` | List of ports to open on master nodes for IPv6 CIDR blocks, expected format is `[{ "protocol" = "tcp", "port_range_min" = 443, "port_range_max" = 443, "remote_ip_prefix" = "::/0"}]`, empty by default |
 |`node_root_volume_size_in_gb` | Size of the root volume for nodes, 0 to use ephemeral storage |
 |`master_root_volume_size_in_gb` | Size of the root volume for masters, 0 to use ephemeral storage |
 |`master_volume_type` | Volume type of the root volume for control_plane, 'Default' by default |
@@ -624,7 +624,7 @@ Edit `inventory/$CLUSTER/group_vars/k8s_cluster/k8s_cluster.yml`:
   - **calico** requires [configuring OpenStack Neutron ports](/docs/cloud_controllers/openstack.md) to allow service and pod subnets
 
 ```yml
-# Choose network plugin (calico or flannel)
+# Choose network plugin (calico, weave or flannel)
 # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
 kube_network_plugin: flannel
 ```

contrib/terraform/openstack/variables.tf

@@ -271,14 +271,7 @@ variable "master_allowed_ports" {
 variable "master_allowed_ports_ipv6" {
   type = list(any)
 
-  default = [
+  default = []
-    {
-      "protocol"         = "ipv6-icmp"
-      "port_range_min"   = 0
-      "port_range_max"   = 0
-      "remote_ip_prefix" = "::/0"
-    },
-  ]
 }
 
 variable "worker_allowed_ports" {
@@ -304,12 +297,6 @@ variable "worker_allowed_ports_ipv6" {
       "port_range_max"   = 32767
       "remote_ip_prefix" = "::/0"
     },
-    {
-      "protocol"         = "ipv6-icmp"
-      "port_range_min"   = 0
-      "port_range_max"   = 0
-      "remote_ip_prefix" = "::/0"
-    },
   ]
 }
 

contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf

@@ -3,7 +3,7 @@ terraform {
   required_providers {
     upcloud = {
       source  = "UpCloudLtd/upcloud"
-      version = "~>5.29.1"
+      version = "~>5.9.0"
     }
   }
   required_version = ">= 0.13"

contrib/terraform/upcloud/versions.tf

@@ -3,7 +3,7 @@ terraform {
   required_providers {
     upcloud = {
       source  = "UpCloudLtd/upcloud"
-      version = "~>5.29.1"
+      version = "~>5.9.0"
     }
   }
   required_version = ">= 0.13"

docs/CNI/calico.md

@@ -180,7 +180,7 @@ calico_group_id=rr1
 The inventory above will deploy the following topology assuming that calico's
 `global_as_num` is set to `65400`:
 
-![Image](../figures/kubespray-calico-rr.png?raw=true)
+![Image](figures/kubespray-calico-rr.png?raw=true)
 
 ### Optional : Define default endpoint to host action
 

docs/CNI/cilium.md

@@ -1,13 +1,5 @@
 # Cilium
 
-## Unprivileged agent configuration
-
-By default, Cilium is installed with `securityContext.privileged: false`. You need to set the `kube_owner` variable to `root` in the inventory:
-
-```yml
-kube_owner: root
-```
-
 ## IP Address Management (IPAM)
 
 IP Address Management (IPAM) is responsible for the allocation and management of IP addresses used by network endpoints (container and others) managed by Cilium. The default mode is "Cluster Scope".
@@ -245,7 +237,7 @@ cilium_operator_extra_volume_mounts:
 ## Choose Cilium version
 
 ```yml
-cilium_version: "1.18.5"
+cilium_version: "1.17.3"
 ```
 
 ## Add variable to config

docs/CNI/macvlan.md

@@ -32,7 +32,7 @@ add `kube_proxy_masquerade_all: true` in `group_vars/all/all.yml`
 
 * Disable nodelocaldns
 
-The nodelocal dns IP is not reachable.
+The nodelocal dns IP is not reacheable.
 
 Disable it in `sample/group_vars/k8s_cluster/k8s_cluster.yml`
 

docs/CNI/weave.md

@@ -0,0 +1,79 @@
+# Weave
+
+Weave 2.0.1 is supported by kubespray
+
+Weave uses [**consensus**](https://www.weave.works/docs/net/latest/ipam/##consensus) mode (default mode) and [**seed**](https://www.weave.works/docs/net/latest/ipam/#seed) mode.
+
+`Consensus` mode is best to use on static size cluster and `seed` mode is best to use on dynamic size cluster
+
+Weave encryption is supported for all communication
+
+* To use Weave encryption, specify a strong password (if no password, no encryption)
+
+```ShellSession
+# In file ./inventory/sample/group_vars/k8s_cluster.yml
+weave_password: EnterPasswordHere
+```
+
+This password is used to set an environment variable inside weave container.
+
+Weave is deployed by kubespray using a daemonSet
+
+* Check the status of Weave containers
+
+```ShellSession
+# From client
+kubectl -n kube-system get pods | grep weave
+# output
+weave-net-50wd2                       2/2       Running   0          2m
+weave-net-js9rb                       2/2       Running   0          2m
+```
+
+There must be as many pods as nodes (here kubernetes have 2 nodes so there are 2 weave pods).
+
+* Check status of weave (connection,encryption ...) for each node
+
+```ShellSession
+# On nodes
+curl http://127.0.0.1:6784/status
+# output on node1
+Version: 2.0.1 (up to date; next check at 2017/08/01 13:51:34)
+
+        Service: router
+       Protocol: weave 1..2
+           Name: fa:16:3e:b3:d6:b2(node1)
+     Encryption: enabled
+  PeerDiscovery: enabled
+        Targets: 2
+    Connections: 2 (1 established, 1 failed)
+          Peers: 2 (with 2 established connections)
+ TrustedSubnets: none
+
+        Service: ipam
+         Status: ready
+          Range: 10.233.64.0/18
+  DefaultSubnet: 10.233.64.0/18
+```
+
+* Check parameters of weave for each node
+
+```ShellSession
+# On nodes
+ps -aux | grep weaver
+# output on node1 (here its use seed mode)
+root      8559  0.2  3.0 365280 62700 ?        Sl   08:25   0:00 /home/weave/weaver --name=fa:16:3e:b3:d6:b2 --port=6783 --datapath=datapath --host-root=/host --http-addr=127.0.0.1:6784 --status-addr=0.0.0.0:6782 --docker-api= --no-dns --db-prefix=/weavedb/weave-net --ipalloc-range=10.233.64.0/18 --nickname=node1 --ipalloc-init seed=fa:16:3e:b3:d6:b2,fa:16:3e:f0:50:53 --conn-limit=30 --expect-npc 192.168.208.28 192.168.208.19
+```
+
+## Consensus mode (default mode)
+
+This mode is best to use on static size cluster
+
+### Seed mode
+
+This mode is best to use on dynamic size cluster
+
+The seed mode also allows multi-clouds and hybrid on-premise/cloud clusters deployment.
+
+* Switch from consensus mode to seed/Observation mode
+
+See [weave ipam documentation](https://www.weave.works/docs/net/latest/tasks/ipam/ipam/) and use `weave_extra_args` to enable.

docs/CRI/containerd.md

@@ -65,8 +65,9 @@ In kubespray, the default runtime name is "runc", and it can be configured with
 containerd_runc_runtime:
   name: runc
   type: "io.containerd.runc.v2"
+  engine: ""
+  root: ""
   options:
-    Root: ""
     SystemdCgroup: "false"
     BinaryName: /usr/local/bin/my-runc
   base_runtime_spec: cri-base.json
@@ -148,11 +149,3 @@ following configuration:
 ```yaml
 nri_enabled: true
 ```
-
-### Optional : Static Binary
-
-To ensure compatibility with older distributions (such as Debian 11), you can use a static containerd binary. By default, this is static binary if the system's glibc version is less than 2.34; otherwise, it is the default binary.
-
-```yaml
-containerd_static_binary: true
-```

docs/CRI/cri-o.md

@@ -80,7 +80,7 @@ The `crio_remap_enable` configures the `/etc/subuid` and `/etc/subgid` files to
 By default, 16M uids and gids are reserved for user namespaces (256 pods * 65536 uids/gids) at the end of the uid/gid space.
 
 The `crio_default_capabilities` configure the default containers capabilities for the crio.
-Defaults capabilities are:
+Defaults capabilties are:
 
 ```yaml
 crio_default_capabilities:

docs/_sidebar.md

@@ -6,6 +6,7 @@
   * [Downloads](/docs/advanced/downloads.md)
   * [Gcp-lb](/docs/advanced/gcp-lb.md)
   * [Kubernetes-reliability](/docs/advanced/kubernetes-reliability.md)
+  * [Mitogen](/docs/advanced/mitogen.md)
   * [Netcheck](/docs/advanced/netcheck.md)
   * [Ntp](/docs/advanced/ntp.md)
   * [Proxy](/docs/advanced/proxy.md)
@@ -31,6 +32,7 @@
   * [Kube-router](/docs/CNI/kube-router.md)
   * [Macvlan](/docs/CNI/macvlan.md)
   * [Multus](/docs/CNI/multus.md)
+  * [Weave](/docs/CNI/weave.md)
 * CRI
   * [Containerd](/docs/CRI/containerd.md)
   * [Cri-o](/docs/CRI/cri-o.md)

docs/advanced/arch.md

@@ -9,6 +9,7 @@ The following table shows the impact of the CPU architecture on compatible featu
 | kube_network_plugin | amd64 | arm64 | amd64 + arm64 |
 |---------------------|-------|-------|---------------|
 | Calico              | Y     | Y     | Y             |
+| Weave               | Y     | Y     | Y             |
 | Flannel             | Y     | N     | N             |
 | Canal               | Y     | N     | N             |
 | Cilium              | Y     | Y     | N             |

docs/advanced/cert_manager.md

@@ -6,7 +6,7 @@
     - [Create New TLS Root CA Certificate and Key](#create-new-tls-root-ca-certificate-and-key)
       - [Install Cloudflare PKI/TLS `cfssl` Toolkit.](#install-cloudflare-pkitls-cfssl-toolkit)
       - [Create Root Certificate Authority (CA) Configuration File](#create-root-certificate-authority-ca-configuration-file)
-      - [Create Certificate Signing Request (CSR) Configuration File](#create-certificate-signing-request-csr-configuration-file)
+      - [Create Certficate Signing Request (CSR) Configuration File](#create-certficate-signing-request-csr-configuration-file)
       - [Create TLS Root CA Certificate and Key](#create-tls-root-ca-certificate-and-key)
 
 Cert-Manager is a native Kubernetes certificate management controller. It can help with issuing certificates from a variety of sources, such as Let’s Encrypt, HashiCorp Vault, Venafi, a simple signing key pair, or self signed. It will ensure certificates are valid and up to date, and attempt to renew certificates at a configured time before expiry.
@@ -134,7 +134,7 @@ $ cat > ca-config.json <<EOF
 EOF
 ```
 
-#### Create Certificate Signing Request (CSR) Configuration File
+#### Create Certficate Signing Request (CSR) Configuration File
 
 The TLS certificate `names` details can be updated to your own specific requirements.
 

docs/advanced/gcp-lb.md

@@ -1,4 +1,4 @@
-# GCP Load Balancers for type=LoadBalancer of Kubernetes Services
+# GCP Load Balancers for type=LoadBalacer of Kubernetes Services
 
 > **Removed**: Since v1.31 (the Kubespray counterpart is v2.27), Kubernetes no longer supports `cloud_provider`. (except external cloud provider)
 

docs/advanced/mitogen.md

@@ -0,0 +1,30 @@
+# Mitogen
+
+*Warning:* Mitogen support is now deprecated in kubespray due to upstream not releasing an updated version to support ansible 4.x (ansible-base 2.11.x) and above. The CI support has been stripped for mitogen and we are no longer validating any support or regressions for it. The supporting mitogen install playbook and integration documentation will be removed in a later version.
+
+[Mitogen for Ansible](https://mitogen.networkgenomics.com/ansible_detailed.html) allow a 1.25x - 7x speedup and a CPU usage reduction of at least 2x, depending on network conditions, modules executed, and time already spent by targets on useful work. Mitogen cannot improve a module once it is executing, it can only ensure the module executes as quickly as possible.
+
+## Install
+
+```ShellSession
+ansible-playbook contrib/mitogen/mitogen.yml
+```
+
+The above playbook sets the ansible `strategy` and `strategy_plugins` in `ansible.cfg` but you can also enable them if you use your own `ansible.cfg` by setting the environment varialbles:
+
+```ShellSession
+export ANSIBLE_STRATEGY=mitogen_linear
+export ANSIBLE_STRATEGY_PLUGINS=plugins/mitogen/ansible_mitogen/plugins/strategy
+```
+
+... or `ansible.cfg` setup:
+
+```ini
+[defaults]
+strategy_plugins = plugins/mitogen/ansible_mitogen/plugins/strategy
+strategy=mitogen_linear
+```
+
+## Limitation
+
+If you are experiencing problems, please see the [documentation](https://mitogen.networkgenomics.com/ansible_detailed.html#noteworthy-differences).

docs/ansible/ansible.md

@@ -13,7 +13,7 @@ KUBESPRAYDIR=kubespray
 python3 -m venv $VENVDIR
 source $VENVDIR/bin/activate
 cd $KUBESPRAYDIR
-pip install -r requirements.txt
+pip install -U -r requirements.txt
 ```
 
 In case you have a similar message when installing the requirements:
@@ -32,7 +32,7 @@ Based on the table below and the available python version for your ansible host
 
 | Ansible Version | Python Version |
 |-----------------|----------------|
-| >= 2.17.3       | 3.10-3.12      |
+| >= 2.16.4       | 3.10-3.12      |
 
 ## Customize Ansible vars
 
@@ -42,10 +42,13 @@ Kubespray expects users to use one of the following variables sources for settin
 |----------------------------------------|------------------------------------------------------------------------------|
 | inventory vars                         |                                                                              |
 |  - **inventory group_vars**            | most used                                                                    |
-|  - inventory host_vars                 | host specific vars overrides, group_vars is usually more practical           |
+|  - inventory host_vars                 | host specifc vars overrides, group_vars is usually more practical            |
 | **extra vars** (always win precedence) | override with ``ansible-playbook -e @foo.yml``                               |
 
-> Extra vars are best used to override kubespray internal variables, for instances, roles/vars/. Those vars are usually **not expected** (by Kubespray developers) to be modified by end users, and not part of Kubespray interface. Thus they can change, disappear, or break stuff unexpectedly.
+[!IMPORTANT]
+Extra vars are best used to override kubespray internal variables, for instances, roles/vars/.
+Those vars are usually **not expected** (by Kubespray developers) to be modified by end users, and not part of Kubespray
+interface. Thus they can change, disappear, or break stuff unexpectedly.
 
 ## Ansible tags
 
@@ -115,11 +118,12 @@ The following tags are defined in playbooks:
 | local-path-provisioner         | Configure External provisioner: local-path            |
 | local-volume-provisioner       | Configure External provisioner: local-volume          |
 | macvlan                        | Network plugin macvlan                                |
+| master (DEPRECATED)            | Deprecated - see `control-plane`                      |
 | metallb                        | Installing and configuring metallb                    |
 | metrics_server                 | Configuring metrics_server                            |
 | netchecker                     | Installing netchecker K8s app                         |
 | network                        | Configuring networking plugins for K8s                |
-| mounts                         | Umount kubelet dirs when resetting                    |
+| mounts                         | Umount kubelet dirs when reseting                     |
 | multus                         | Network plugin multus                                 |
 | nginx                          | Configuring LB for kube-apiserver instances           |
 | node                           | Configuring K8s minion (compute) node role            |
@@ -149,9 +153,14 @@ The following tags are defined in playbooks:
 | upgrade                        | Upgrading, f.e. container images/binaries             |
 | upload                         | Distributing images/binaries across hosts             |
 | vsphere-csi-driver             | Configuring csi driver: vsphere                       |
+| weave                          | Network plugin Weave                                  |
 | win_nodes                      | Running windows specific tasks                        |
 | youki                          | Configuring youki runtime                             |
 
+Note: Use the ``bash scripts/gen_tags.sh`` command to generate a list of all
+tags found in the codebase. New tags will be listed with the empty "Used for"
+field.
+
 ## Example commands
 
 Example command to filter and apply only DNS configuration tasks and skip
@@ -178,13 +187,17 @@ ansible-playbook -i inventory/sample/hosts.ini cluster.yml \
 
 Note: use `--tags` and `--skip-tags` wisely and only if you're 100% sure what you're doing.
 
+## Mitogen
+
+Mitogen support is deprecated, please see [mitogen related docs](/docs/advanced/mitogen.md) for usage and reasons for deprecation.
+
 ## Troubleshooting Ansible issues
 
 Having the wrong version of ansible, ansible collections or python dependencies can cause issue.
-In particular, Kubespray ship custom modules which Ansible needs to find, for which you should specify [ANSIBLE_LIBRARY](https://docs.ansible.com/ansible/latest/dev_guide/developing_locally.html#adding-a-module-or-plugin-outside-of-a-collection)
+In particular, Kubespray ship custom modules which Ansible needs to find, for which you should specify [ANSIBLE_LIBRAY](https://docs.ansible.com/ansible/latest/dev_guide/developing_locally.html#adding-a-module-or-plugin-outside-of-a-collection)
 
 ```ShellSession
-export ANSIBLE_LIBRARY=<kubespray_dir>/library`
+export ANSIBLE_LIBRAY=<kubespray_dir>/library`
 ```
 
 A simple way to ensure you get all the correct version of Ansible is to use
@@ -193,11 +206,11 @@ You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mou
 to access the inventory and SSH key in the container, like this:
 
 ```ShellSession
-git checkout v2.29.0
+git checkout v2.27.0
-docker pull quay.io/kubespray/kubespray:v2.29.0
+docker pull quay.io/kubespray/kubespray:v2.27.0
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
   --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.29.0 bash
+  quay.io/kubespray/kubespray:v2.27.0 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```

docs/ansible/ansible_collection.md

@@ -2,13 +2,14 @@
 
 Kubespray can be installed as an [Ansible collection](https://docs.ansible.com/ansible/latest/user_guide/collections_using.html).
 
+## Requirements
+
+- An inventory file with the appropriate host groups. See the [README](../README.md#usage).
+- A `group_vars` directory. These group variables **need** to match the appropriate variable names under `inventory/local/group_vars`. See the [README](../README.md#usage).
+
 ## Usage
 
-1. Set up an inventory with the appropriate host groups and required group vars.
+1. Add Kubespray to your requirements.yml file
-   See also the documentation on [kubespray inventories](./inventory.md) and the
-   general ["Getting started" documentation](../getting_started/getting-started.md#building-your-own-inventory).
-
-2. Add Kubespray to your requirements.yml file
 
    ```yaml
    collections:
@@ -17,20 +18,20 @@ Kubespray can be installed as an [Ansible collection](https://docs.ansible.com/a
      version: master # use the appropriate tag or branch for the version you need
    ```
 
-3. Install your collection
+2. Install your collection
 
    ```ShellSession
    ansible-galaxy install -r requirements.yml
    ```
 
-4. Create a playbook to install your Kubernetes cluster
+3. Create a playbook to install your Kubernetes cluster
 
    ```yaml
    - name: Install Kubernetes
      ansible.builtin.import_playbook: kubernetes_sigs.kubespray.cluster
    ```
 
-5. Update INVENTORY and PLAYBOOK so that they point to your inventory file and the playbook you created above, and then install Kubespray
+4. Update INVENTORY and PLAYBOOK so that they point to your inventory file and the playbook you created above, and then install Kubespray
 
    ```ShellSession
    ansible-playbook -i INVENTORY --become --become-user=root PLAYBOOK

docs/ansible/vars.md

@@ -103,13 +103,13 @@ following default cluster parameters:
 
 * *kube_service_addresses_ipv6* - Subnet for cluster IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1000/116``). Must not overlap with ``kube_pods_subnet_ipv6``.
 
-* *kube_service_subnets* - All service subnets separated by commas (default is a mix of ``kube_service_addresses`` and ``kube_service_addresses_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stack`` options),
+* *kube_service_subnets* - All service subnets separated by commas (default is a mix of ``kube_service_addresses`` and ``kube_service_addresses_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stacke`` options),
   for example ``10.233.0.0/18,fd85:ee78:d8a6:8607::1000/116`` for dual stack(ipv4_stack/ipv6_stack set to `true`).
   It is not recommended to change this variable directly.
 
 * *kube_pods_subnet_ipv6* - Subnet for Pod IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1:0000/112``). Must not overlap with ``kube_service_addresses_ipv6``.
 
-* *kube_pods_subnets* - All pods subnets separated by commas (default is a mix of ``kube_pods_subnet`` and ``kube_pod_subnet_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stack`` options),
+* *kube_pods_subnets* - All pods subnets separated by commas (default is a mix of ``kube_pods_subnet`` and ``kube_pod_subnet_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stacke`` options),
   for example ``10.233.64.0/18,fd85:ee78:d8a6:8607::1:0000/112`` for dual stack(ipv4_stack/ipv6_stack set to `true`).
   It is not recommended to change this variable directly.
 

docs/cloud_controllers/openstack.md

@@ -73,7 +73,6 @@ The cloud provider is configured to have Octavia by default in Kubespray.
   external_openstack_lbaas_method: ROUND_ROBIN
   external_openstack_lbaas_provider: amphora
   external_openstack_lbaas_subnet_id: "Neutron subnet ID to create LBaaS VIP"
-  external_openstack_lbaas_member_subnet_id: "Neutron subnet ID on which to create the members of the load balancer"
   external_openstack_lbaas_network_id: "Neutron network ID to create LBaaS VIP"
   external_openstack_lbaas_manage_security_groups: false
   external_openstack_lbaas_create_monitor: false

docs/developers/ci-setup.md

@@ -2,14 +2,19 @@
 
 ## Pipeline
 
-See [.gitlab-ci.yml](/.gitlab-ci.yml) and the included files for an overview.
+1. build: build a docker image to be used in the pipeline
+2. unit-tests: fast jobs for fast feedback (linting, etc...)
+3. deploy-part1: small number of jobs to test if the PR works with default settings
+4. deploy-extended: slow jobs testing different platforms, OS, settings, CNI, etc...
+5. deploy-extended: very slow jobs (upgrades, etc...)
 
 ## Runners
 
-Kubespray has 2 types of GitLab runners, both deployed on the Kubespray CI cluster (hosted on Oracle Cloud Infrastructure):
+Kubespray has 3 types of GitLab runners:
 
-- pods: use the [gitlab-ci kubernetes executor](https://docs.gitlab.com/runner/executors/kubernetes/)
+- packet runners: used for E2E jobs (usually long), running on Equinix Metal (ex-packet), on kubevirt managed VMs
-- vagrant: custom executor running in pods with access to the libvirt socket on the nodes
+- light runners: used for short lived jobs, running on Equinix Metal (ex-packet), as managed pods
+- auto scaling runners (managed via docker-machine on Equinix Metal): used for on-demand resources, see [GitLab docs](https://docs.gitlab.com/runner/configuration/autoscale.html) for more info
 
 ## Vagrant
 
@@ -17,17 +22,18 @@ Vagrant jobs are using the [quay.io/kubespray/vagrant](/test-infra/vagrant-docke
 
 ## CI Variables
 
-In CI we have a [set of extra vars](/test/common_vars.yml) we use to ensure greater success of our CI jobs and avoid throttling by various APIs we depend on.
+In CI we have a set of overrides we use to ensure greater success of our CI jobs and avoid throttling by various APIs we depend on. See:
 
-## CI clusters
+- [Docker mirrors](/tests/common/_docker_hub_registry_mirror.yml)
+- [Test settings](/tests/common/_kubespray_test_settings.yml)
 
-DISCLAIMER: The following information is not fully up to date, in particular, the CI cluster is now on Oracle Cloud Infrastcture, not Equinix.
+## CI Environment
 
-The cluster is deployed with kubespray itself and maintained by the kubespray maintainers.
+The CI packet and light runners are deployed on a kubernetes cluster on Equinix Metal. The cluster is deployed with kubespray itself and maintained by the kubespray maintainers.
 
 The following files are used for that inventory:
 
-### cluster.tfvars (OBSOLETE: this section is no longer accurate)
+### cluster.tfvars
 
 ```ini
 # your Kubernetes cluster name here
@@ -156,10 +162,22 @@ kube_feature_gates:
   - "NodeSwap=True"
 ```
 
-## Additional files
+## Aditional files
 
 This section documents additional files used to complete a deployment of the kubespray CI, these files sit on the control-plane node and assume a working kubernetes cluster.
 
+### /root/nscleanup.sh
+
+```bash
+#!/bin/bash
+
+kubectl=/usr/local/bin/kubectl
+
+$kubectl get ns | grep -P "(\d.+-\d.+)" | awk 'match($3,/[0-9]+d/) {print $1}' | xargs -r $kubectl delete ns
+$kubectl get ns | grep -P "(\d.+-\d.+)" | awk 'match($3,/[3-9]+h/) {print $1}' | xargs -r $kubectl delete ns
+$kubectl get ns | grep Terminating | awk '{print $1}' | xargs -i $kubectl delete vmi/instance-1 vmi/instance-0 vmi/instance-2 -n {} --force --grace-period=0 &
+```
+
 ### /root/path-calico.sh
 
 ```bash

docs/developers/ci.md

@@ -6,52 +6,55 @@ To generate this Matrix run `./tests/scripts/md-table/main.py`
 
 | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |
 |---| --- | --- | --- | --- | --- | --- | --- |
+almalinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 almalinux9 |  :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
 amazon |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: |
 debian12 |  :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: |
-debian13 |  :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
 fedora40 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-flatcar4081 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-rockylinux10 |  :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
+opensuse15 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+rockylinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
+ubuntu20 |  :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: |
 ubuntu22 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu24 |  :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: |
+ubuntu24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 
 ## crio
 
 | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |
 |---| --- | --- | --- | --- | --- | --- | --- |
+almalinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 almalinux9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian13 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora40 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora40 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-flatcar4081 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-rockylinux10 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+opensuse15 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu22 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu22 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 
 ## docker
 
 | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |
 |---| --- | --- | --- | --- | --- | --- | --- |
+almalinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 almalinux9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian13 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora40 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-flatcar4081 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-rockylinux10 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+opensuse15 |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
+rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu22 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |

docs/external_storage_provisioners/scheduler_plugins.md

@@ -7,7 +7,7 @@ The kube-scheduler binary includes a list of plugins:
 - [CapacityScheduling](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/capacityscheduling) [Beta]
 - [CoScheduling](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/coscheduling) [Beta]
 - [NodeResources](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/noderesources) [Beta]
-- [NodeResourceTopology](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/noderesourcetopology/README.md) [Beta]
+- [NodeResouceTopology](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/noderesourcetopology/README.md) [Beta]
 - [PreemptionToleration](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/preemptiontoleration/README.md) [Alpha]
 - [Trimaran](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/trimaran/README.md) [Alpha]
 - [NetworkAware](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/networkaware/README.md) [Sample]

docs/getting_started/setting-up-your-first-cluster.md

@@ -61,12 +61,12 @@ gcloud compute networks subnets create kubernetes \
 #### Firewall Rules
 
 Create a firewall rule that allows internal communication across all protocols.
-It is important to note that the vxlan (udp) protocol has to be allowed in order for
+It is important to note that the vxlan protocol has to be allowed in order for
 the calico (see later) networking plugin to work.
 
 ```ShellSession
 gcloud compute firewall-rules create kubernetes-the-kubespray-way-allow-internal \
-  --allow tcp,udp,icmp \
+  --allow tcp,udp,icmp,vxlan \
   --network kubernetes-the-kubespray-way \
   --source-ranges 10.240.0.0/24
 ```
@@ -88,7 +88,7 @@ cluster.
 
 ### Compute Instances
 
-The compute instances in this lab will be provisioned using [Ubuntu Server](https://www.ubuntu.com/server) 24.04.
+The compute instances in this lab will be provisioned using [Ubuntu Server](https://www.ubuntu.com/server) 18.04.
 Each compute instance will be provisioned with a fixed private IP address and
  a public IP address (that can be fixed - see [guide](https://cloud.google.com/compute/docs/ip-addresses/reserve-static-external-ip-address)).
 Using fixed public IP addresses has the advantage that our cluster node
@@ -103,7 +103,7 @@ for i in 0 1 2; do
     --async \
     --boot-disk-size 200GB \
     --can-ip-forward \
-    --image-family ubuntu-2404-lts-amd64 \
+    --image-family ubuntu-1804-lts \
     --image-project ubuntu-os-cloud \
     --machine-type e2-standard-2 \
     --private-network-ip 10.240.0.1${i} \
@@ -124,7 +124,7 @@ for i in 0 1 2; do
     --async \
     --boot-disk-size 200GB \
     --can-ip-forward \
-    --image-family ubuntu-2404-lts-amd64 \
+    --image-family ubuntu-1804-lts \
     --image-project ubuntu-os-cloud \
     --machine-type e2-standard-2 \
     --private-network-ip 10.240.0.2${i} \

docs/ingress/ingress_nginx.md

@@ -35,7 +35,7 @@ kubectl create clusterrolebinding cluster-admin-binding \
 The following **Mandatory Command** is required for all deployments except for AWS. See below for the AWS version.
 
 ```console
-kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.13.3/deploy/static/provider/cloud/deploy.yaml
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.0/deploy/static/provider/cloud/deploy.yaml
 ```
 
 ### Provider Specific Steps

docs/ingress/metallb.md

@@ -35,7 +35,7 @@ metallb_config:
       effect: "NoSchedule"
 ```
 
-If you'd like to set additional nodeSelector and tolerations values, you can do so in the following fashion:
+If you'd like to set additional nodeSelector and tolerations values, you can do so in the following fasion:
 
 ```yaml
 metallb_config:

docs/operating_systems/rhel.md

@@ -37,12 +37,4 @@ If you have containers that are using iptables in the host network namespace (`h
 you need to ensure they are using iptables-nft.
 An example how k8s do the autodetection can be found [in this PR](https://github.com/kubernetes/kubernetes/pull/82966)
 
-The kernel version is lower than the kubernetes 1.32 system validation, please refer to the [kernel requirements](../operations/kernel-requirements.md).
+The kernel version is lower than the kubenretes 1.32 system validation, please refer to the [kernel requirements](../operations/kernel-requirements.md).
-
-## Rocky Linux 10
-
-(Experimental in Kubespray CI)
-
-The official Rocky Linux 10 cloud image does not include `kernel-module-extra`. Both Kube Proxy and CNI rely on this package, and since it relates to kernel version compatibility (which may require VM reboots, etc.), we haven't found an ideal solution.
-
-However, some users report that it doesn't affect them (minimal version). Therefore, the Kubespray CI Rocky Linux 10 image is built by Kubespray maintainers using `diskimage-builder`. For detailed methods, please refer to [the comments](https://github.com/kubernetes-sigs/kubespray/pull/12355#issuecomment-3705400093).

docs/operations/cgroups.md

@@ -30,6 +30,11 @@ kube_memory_reserved: 256Mi
 kube_cpu_reserved: 100m
 # kube_ephemeral_storage_reserved: 2Gi
 # kube_pid_reserved: "1000"
+# Reservation for master hosts
+kube_master_memory_reserved: 512Mi
+kube_master_cpu_reserved: 200m
+# kube_master_ephemeral_storage_reserved: 2Gi
+# kube_master_pid_reserved: "1000"
 
 # Set to true to reserve resources for system daemons
 system_reserved: true
@@ -39,6 +44,11 @@ system_memory_reserved: 512Mi
 system_cpu_reserved: 500m
 # system_ephemeral_storage_reserved: 2Gi
 # system_pid_reserved: "1000"
+# Reservation for master hosts
+system_master_memory_reserved: 256Mi
+system_master_cpu_reserved: 250m
+# system_master_ephemeral_storage_reserved: 2Gi
+# system_master_pid_reserved: "1000"
 ```
 
 After the setup, the cgroups hierarchy is as follows:

docs/operations/hardening.md

@@ -18,6 +18,8 @@ The **kubernetes** version should be at least `v1.23.6` to have all the most rec
 
 ## kube-apiserver
 authorization_modes: ['Node', 'RBAC']
+# AppArmor-based OS
+# kube_apiserver_feature_gates: ['AppArmor=true']
 kube_apiserver_request_timeout: 120s
 kube_apiserver_service_account_lookup: true
 
@@ -75,17 +77,17 @@ remove_anonymous_access: true
 ## kube-controller-manager
 kube_controller_manager_bind_address: 127.0.0.1
 kube_controller_terminated_pod_gc_threshold: 50
+# AppArmor-based OS
+# kube_controller_feature_gates: ["RotateKubeletServerCertificate=true", "AppArmor=true"]
 kube_controller_feature_gates: ["RotateKubeletServerCertificate=true"]
 
 ## kube-scheduler
 kube_scheduler_bind_address: 127.0.0.1
+# AppArmor-based OS
+# kube_scheduler_feature_gates: ["AppArmor=true"]
 
 ## etcd
-# Running etcd (on dedicated hosts) outside the Kubernetes cluster is the most secure deployment option,
+etcd_deployment_type: kubeadm
-# as it isolates etcd from the cluster's CNI network and removes direct pod-level attack vectors.
-# This approach prevents RBAC misconfigurations that potentially compromise etcd,
-# creating an additional security boundary that protects the cluster's critical state store.
-etcd_deployment_type: host
 
 ## kubelet
 kubelet_authorization_mode_webhook: true
@@ -100,8 +102,6 @@ kubelet_make_iptables_util_chains: true
 kubelet_feature_gates: ["RotateKubeletServerCertificate=true"]
 kubelet_seccomp_default: true
 kubelet_systemd_hardening: true
-# To disable kubelet's staticPodPath (for nodes that don't use static pods like worker nodes)
-kubelet_static_pod_path: ""
 # In case you have multiple interfaces in your
 # control plane nodes and you want to specify the right
 # IP addresses, kubelet_secure_addresses allows you
@@ -126,8 +126,9 @@ Let's take a deep look to the resultant **kubernetes** configuration:
 * The `encryption-provider-config` provide encryption at rest. This means that the `kube-apiserver` encrypt data that is going to be stored before they reach `etcd`. So the data is completely unreadable from `etcd` (in case an attacker is able to exploit this).
 * The `rotateCertificates` in `KubeletConfiguration` is set to `true` along with `serverTLSBootstrap`. This could be used in alternative to `tlsCertFile` and `tlsPrivateKeyFile` parameters. Additionally it automatically generates certificates by itself. By default the CSRs are approved automatically via [kubelet-csr-approver](https://github.com/postfinance/kubelet-csr-approver). You can customize approval configuration by modifying Helm values via `kubelet_csr_approver_values`.
   See <https://kubernetes.io/docs/reference/access-authn-authz/kubelet-tls-bootstrapping/> for more information on the subject.
+* If you are installing **kubernetes** in an AppArmor-based OS (eg. Debian/Ubuntu) you can enable the `AppArmor` feature gate uncommenting the lines with the comment `# AppArmor-based OS` on top.
 * The `kubelet_systemd_hardening`, both with `kubelet_secure_addresses` setup a minimal firewall on the system. To better understand how these variables work, here's an explanatory image:
-  ![kubelet hardening](../img/kubelet-hardening.png)
+  ![kubelet hardening](img/kubelet-hardening.png)
 
 Once you have the file properly filled, you can run the **Ansible** command to start the installation:
 

docs/operations/kernel-requirements.md

@@ -11,7 +11,7 @@ kubeadm_ignore_preflight_errors:
 
 The Kernel Version Matrixs:
 
-| OS Version         | Kernel Version | Kernel >=4.19      |
+| OS Verion          | Kernel Verion  | Kernel >=4.19      |
 |---                 | ---            | ---                |
 | RHEL 9             | 5.14           | :white_check_mark: |
 | RHEL 8             | 4.18           | :x:                |

docs/operations/nodes.md

@@ -2,6 +2,58 @@
 
 Modified from [comments in #3471](https://github.com/kubernetes-sigs/kubespray/issues/3471#issuecomment-530036084)
 
+## Limitation: Removal of first kube_control_plane and etcd-master
+
+Currently you can't remove the first node in your kube_control_plane and etcd-master list. If you still want to remove this node you have to:
+
+### 1) Change order of current control planes
+
+Modify the order of your control plane list by pushing your first entry to any other position. E.g. if you want to remove `node-1` of the following example:
+
+```yaml
+  children:
+    kube_control_plane:
+      hosts:
+        node-1:
+        node-2:
+        node-3:
+    kube_node:
+      hosts:
+        node-1:
+        node-2:
+        node-3:
+    etcd:
+      hosts:
+        node-1:
+        node-2:
+        node-3:
+```
+
+change your inventory to:
+
+```yaml
+  children:
+    kube_control_plane:
+      hosts:
+        node-2:
+        node-3:
+        node-1:
+    kube_node:
+      hosts:
+        node-2:
+        node-3:
+        node-1:
+    etcd:
+      hosts:
+        node-2:
+        node-3:
+        node-1:
+```
+
+## 2) Upgrade the cluster
+
+run `upgrade-cluster.yml` or `cluster.yml`. Now you are good to go on with the removal.
+
 ## Adding/replacing a worker node
 
 This should be the easiest.
@@ -31,8 +83,6 @@ That's it.
 
 Append the new host to the inventory and run `cluster.yml`. You can NOT use `scale.yml` for that.
 
-**Note:** When adding new control plane nodes, always append them to the end of the `kube_control_plane` group in your inventory. Adding control plane nodes in the first position is not supported and will cause the playbook to fail.
-
 ### 2) Restart kube-system/nginx-proxy
 
 In all hosts, restart nginx-proxy pod. This pod is a local proxy for the apiserver. Kubespray will update its static config, but it needs to be restarted in order to reload.
@@ -50,74 +100,40 @@ crictl ps | grep nginx-proxy | awk '{print $1}' | xargs crictl stop
 With the old node still in the inventory, run `remove-node.yml`. You need to pass `-e node=NODE_NAME` to the playbook to limit the execution to the node being removed.
 If the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars.
 
-## Adding/Removal of first `kube_control_plane` and etcd-master
+## Replacing a first control plane node
 
-Currently you can't remove the first node in your `kube_control_plane` and etcd-master list. If you still want to remove this node you have to:
+### 1) Change control plane nodes order in inventory
 
-### 1) Change order of current control planes
+from
 
-Modify the order of your control plane list by pushing your first entry to any other position. E.g. if you want to remove `node-1` of the following example:
+```ini
-
+[kube_control_plane]
-```yaml
+ node-1
-all:
+ node-2
-  hosts:
+ node-3
-  children:
-    kube_control_plane:
-      hosts:
-        node-1:
-        node-2:
-        node-3:
-    kube_node:
-      hosts:
-        node-1:
-        node-2:
-        node-3:
-    etcd:
-      hosts:
-        node-1:
-        node-2:
-        node-3:
 ```
 
-change your inventory to:
+to
 
-```yaml
+```ini
-all:
+[kube_control_plane]
-  hosts:
+ node-2
-  children:
+ node-3
-    kube_control_plane:
+ node-1
-      hosts:
-        node-2:
-        node-3:
-        node-1:
-    kube_node:
-      hosts:
-        node-2:
-        node-3:
-        node-1:
-    etcd:
-      hosts:
-        node-2:
-        node-3:
-        node-1:
 ```
 
-### 2) Upgrade the cluster
+### 2) Remove old first control plane node from cluster
-
-run `upgrade-cluster.yml` or `cluster.yml`. Now you are good to go on with the removal.
-
-### 3) Remove old first control plane node from cluster
 
 With the old node still in the inventory, run `remove-node.yml`. You need to pass `-e node=node-1` to the playbook to limit the execution to the node being removed.
 If the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars.
 
-### 4) Edit cluster-info configmap in kube-public namespace
+### 3) Edit cluster-info configmap in kube-public namespace
 
 `kubectl  edit cm -n kube-public cluster-info`
 
 Change ip of old kube_control_plane node with ip of live kube_control_plane node (`server` field). Also, update `certificate-authority-data` field if you changed certs.
 
-### 5) Add new control plane node
+### 4) Add new control plane node
 
 Update inventory (if needed)
 

docs/operations/upgrades.md

@@ -13,6 +13,7 @@ versions. Here are all version vars for each component:
 * etcd_version
 * calico_version
 * calico_cni_version
+* weave_version
 * flannel_version
 
 > **Warning**
@@ -355,7 +356,7 @@ follows:
 * Containerd
 * etcd
 * kubelet and kube-proxy
-* network_plugin (such as Calico)
+* network_plugin (such as Calico or Weave)
 * kube-apiserver, kube-scheduler, and kube-controller-manager
 * Add-ons (such as KubeDNS)
 

galaxy.yml

@@ -2,7 +2,7 @@
 namespace: kubernetes_sigs
 description: Deploy a production ready Kubernetes cluster
 name: kubespray
-version: 2.30.0
+version: 2.28.0
 readme: README.md
 authors:
   - The Kubespray maintainers (https://kubernetes.slack.com/channels/kubespray)

index.html

@@ -38,7 +38,6 @@
     loadSidebar: 'docs/_sidebar.md',
     repo: 'https://github.com/kubernetes-sigs/kubespray',
     auto2top: true,
-    noCompileLinks: ['.*\.ini'],
     logo: '/logo/logo-clear.png'
   }
 </script>

inventory/sample/group_vars/all/all.yml

@@ -115,9 +115,6 @@ no_proxy_exclude_workers: false
 # sysctl_file_path to add sysctl conf to
 # sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf"
 
-# ignore sysctl errors about unknown keys
-# sysctl_ignore_unknown_keys: false
-
 ## Variables for webhook token auth https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication
 kube_webhook_token_auth: false
 kube_webhook_token_auth_url_skip_tls_verify: false

inventory/sample/group_vars/all/containerd.yml

@@ -11,15 +11,15 @@
 # containerd_runc_runtime:
 #   name: runc
 #   type: "io.containerd.runc.v2"
-#   options:
+#   engine: ""
-#     Root: ""
+#   root: ""
 
 # containerd_additional_runtimes:
 # Example for Kata Containers as additional runtime:
 #   - name: kata
 #     type: "io.containerd.kata.v2"
-#     options:
+#     engine: ""
-#       Root: ""
+#     root: ""
 
 # containerd_grpc_max_recv_message_size: 16777216
 # containerd_grpc_max_send_message_size: 16777216
@@ -50,8 +50,6 @@
 #     - host: https://registry-1.docker.io
 #       capabilities: ["pull", "resolve"]
 #       skip_verify: false
-#       header:
-#         Authorization: "Basic XXX"
 
 # containerd_max_container_log_line_size: 16384
 

inventory/sample/group_vars/etcd.yml

@@ -0,0 +1,38 @@
+---
+## Etcd auto compaction retention for mvcc key value store in hour
+# etcd_compaction_retention: 0
+
+## Set level of detail for etcd exported metrics, specify 'extensive' to include histogram metrics.
+# etcd_metrics: basic
+
+## Etcd is restricted by default to 512M on systems under 4GB RAM, 512MB is not enough for much more than testing.
+## Set this if your etcd nodes have less than 4GB but you want more RAM for etcd. Set to 0 for unrestricted RAM.
+## This value is only relevant when deploying etcd with `etcd_deployment_type: docker`
+# etcd_memory_limit: "512M"
+
+## Etcd has a default of 2G for its space quota. If you put a value in etcd_memory_limit which is less than
+## etcd_quota_backend_bytes, you may encounter out of memory terminations of the etcd cluster. Please check
+## etcd documentation for more information.
+# 8G is a suggested maximum size for normal environments and etcd warns at startup if the configured value exceeds it.
+# etcd_quota_backend_bytes: "2147483648"
+
+# Maximum client request size in bytes the server will accept.
+# etcd is designed to handle small key value pairs typical for metadata.
+# Larger requests will work, but may increase the latency of other requests
+# etcd_max_request_bytes: "1572864"
+
+### ETCD: disable peer client cert authentication.
+# This affects ETCD_PEER_CLIENT_CERT_AUTH variable
+# etcd_peer_client_auth: true
+
+## Enable distributed tracing
+## To enable this experimental feature, set the etcd_experimental_enable_distributed_tracing: true, along with the
+## etcd_experimental_distributed_tracing_sample_rate to choose how many samples to collect per million spans,
+## the default sampling rate is 0 https://etcd.io/docs/v3.5/op-guide/monitoring/#distributed-tracing
+# etcd_experimental_enable_distributed_tracing: false
+# etcd_experimental_distributed_tracing_sample_rate: 100
+# etcd_experimental_distributed_tracing_address: "localhost:4317"
+# etcd_experimental_distributed_tracing_service_name: etcd
+
+## The interval for etcd watch progress notify events
+# etcd_experimental_watch_progress_notify_interval: 5s

inventory/sample/group_vars/k8s_cluster/addons.yml

@@ -29,6 +29,7 @@ local_path_provisioner_enabled: false
 # local_path_provisioner_claim_root: /opt/local-path-provisioner/
 # local_path_provisioner_debug: false
 # local_path_provisioner_image_repo: "{{ docker_image_repo }}/rancher/local-path-provisioner"
+# local_path_provisioner_image_tag: "v0.0.24"
 # local_path_provisioner_helper_image_repo: "busybox"
 # local_path_provisioner_helper_image_tag: "latest"
 

inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml

@@ -22,8 +22,7 @@ local_release_dir: "/tmp/releases"
 # Random shifts for retrying failed ops like pushing/downloading
 retry_stagger: 5
 
-# This is the user that owns the cluster installation.
+# This is the user that owns tha cluster installation.
-# Note: cilium needs to set kube_owner to root https://kubespray.io/#/docs/CNI/cilium?id=unprivileged-agent-configuration
 kube_owner: kube
 
 # This is the group that the cert creation scripts chgrp the
@@ -63,7 +62,7 @@ credentials_dir: "{{ inventory_dir }}/credentials"
 # kube_webhook_authorization_url: https://...
 # kube_webhook_authorization_url_skip_tls_verify: false
 
-# Choose network plugin (cilium, calico, kube-ovn or flannel. Use cni for generic cni plugin)
+# Choose network plugin (cilium, calico, kube-ovn, weave or flannel. Use cni for generic cni plugin)
 # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
 kube_network_plugin: calico
 
@@ -347,7 +346,7 @@ event_ttl_duration: "1h0m0s"
 ## Automatically renew K8S control plane certificates on first Monday of each month
 auto_renew_certificates: false
 # First Monday of each month
-# auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:00:00"
+# auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:{{ groups['kube_control_plane'].index(inventory_hostname) }}0:00"
 
 kubeadm_patches_dir: "{{ kube_config_dir }}/patches"
 kubeadm_patches: []

inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml

@@ -25,9 +25,15 @@ calico_pool_blocksize: 26
 # add default ippool CIDR (must be inside kube_pods_subnet, defaults to kube_pods_subnet otherwise)
 # calico_pool_cidr: 1.2.3.4/5
 
+# add default ippool CIDR to CNI config
+# calico_cni_pool: true
+
 # Add default IPV6 IPPool CIDR. Must be inside kube_pods_subnet_ipv6. Defaults to kube_pods_subnet_ipv6 if not set.
 # calico_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112
 
+# Add default IPV6 IPPool CIDR to CNI config
+# calico_cni_pool_ipv6: true
+
 # Global as_num (/calico/bgp/v1/global/as_num)
 # global_as_num: "64512"
 

inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml

@@ -175,10 +175,6 @@ cilium_l2announcements: false
 ### Buffer size of the channel to receive monitor events.
 # cilium_hubble_event_queue_size: 50
 
-# Override the DNS suffix that Hubble-Relay uses to resolve its peer service.
-# It defaults to the inventory's `dns_domain`.
-# cilium_hubble_peer_service_cluster_domain: "{{ dns_domain }}"
-
 # IP address management mode for v1.9+.
 # https://docs.cilium.io/en/v1.9/concepts/networking/ipam/
 # cilium_ipam_mode: kubernetes
@@ -384,7 +380,3 @@ cilium_l2announcements: false
 #       resourceNames:
 #       - toto
 # cilium_clusterrole_rules_operator_extra_vars: []
-
-# Cilium extra values, use any values from cilium Helm Chart
-# ref: https://docs.cilium.io/en/stable/helm-reference/
-# cilium_extra_values: {}

inventory/sample/group_vars/k8s_cluster/k8s-net-weave.yml

@@ -0,0 +1,64 @@
+# see roles/network_plugin/weave/defaults/main.yml
+
+# Weave's network password for encryption, if null then no network encryption.
+# weave_password: ~
+
+# If set to 1, disable checking for new Weave Net versions (default is blank,
+# i.e. check is enabled)
+# weave_checkpoint_disable: false
+
+# Soft limit on the number of connections between peers. Defaults to 100.
+# weave_conn_limit: 100
+
+# Weave Net defaults to enabling hairpin on the bridge side of the veth pair
+# for containers attached. If you need to disable hairpin, e.g. your kernel is
+# one of those that can panic if hairpin is enabled, then you can disable it by
+# setting `HAIRPIN_MODE=false`.
+# weave_hairpin_mode: true
+
+# The range of IP addresses used by Weave Net and the subnet they are placed in
+# (CIDR format; default 10.32.0.0/12)
+# weave_ipalloc_range: "{{ kube_pods_subnet }}"
+
+# Set to 0 to disable Network Policy Controller (default is on)
+# weave_expect_npc: "{{ enable_network_policy }}"
+
+# List of addresses of peers in the Kubernetes cluster (default is to fetch the
+# list from the api-server)
+# weave_kube_peers: ~
+
+# Set the initialization mode of the IP Address Manager (defaults to consensus
+# amongst the KUBE_PEERS)
+# weave_ipalloc_init: ~
+
+# Set the IP address used as a gateway from the Weave network to the host
+# network - this is useful if you are configuring the addon as a static pod.
+# weave_expose_ip: ~
+
+# Address and port that the Weave Net daemon will serve Prometheus-style
+# metrics on (defaults to 0.0.0.0:6782)
+# weave_metrics_addr: ~
+
+# Address and port that the Weave Net daemon will serve status requests on
+# (defaults to disabled)
+# weave_status_addr: ~
+
+# Weave Net defaults to 1376 bytes, but you can set a smaller size if your
+# underlying network has a tighter limit, or set a larger size for better
+# performance if your network supports jumbo frames (e.g. 8916)
+# weave_mtu: 1376
+
+# Set to 1 to preserve the client source IP address when accessing Service
+# annotated with `service.spec.externalTrafficPolicy=Local`. The feature works
+# only with Weave IPAM (default).
+# weave_no_masq_local: true
+
+# set to nft to use nftables backend for iptables (default is iptables)
+# weave_iptables_backend: iptables
+
+# Extra variables that passing to launch.sh, useful for enabling seed mode, see
+# https://www.weave.works/docs/net/latest/tasks/ipam/ipam/
+# weave_extra_args: ~
+
+# Extra variables for weave_npc that passing to launch.sh, useful for change log level, ex --log-level=error
+# weave_npc_extra_args: ~

meta/runtime.yml

@@ -1,2 +1,2 @@
 ---
-requires_ansible: ">=2.17.3"
+requires_ansible: '>=2.16.4'

pipeline.Dockerfile

@@ -1,4 +1,4 @@
-# Use immutable image tags rather than mutable tags (like ubuntu:22.04)
+# Use imutable image tags rather than mutable tags (like ubuntu:22.04)
 FROM ubuntu:jammy-20230308
 # Some tools like yamllint need this
 # Pip needs this as well at the moment to install ansible
@@ -47,8 +47,8 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
     && pip install --no-compile --no-cache-dir pip -U \
     && pip install --no-compile --no-cache-dir -r tests/requirements.txt \
     && pip install --no-compile --no-cache-dir -r requirements.txt \
-    && curl -L https://dl.k8s.io/release/v1.34.3/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
+    && curl -L https://dl.k8s.io/release/v1.32.5/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
-    && echo $(curl -L https://dl.k8s.io/release/v1.34.3/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
+    && echo $(curl -L https://dl.k8s.io/release/v1.32.5/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
     && chmod a+x /usr/local/bin/kubectl \
     # Install Vagrant
     && curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \

playbooks/ansible_version.yml

@@ -5,8 +5,8 @@
   become: false
   run_once: true
   vars:
-    minimal_ansible_version: 2.17.3
+    minimal_ansible_version: 2.16.4
-    maximal_ansible_version: 2.18.0
+    maximal_ansible_version: 2.17.0
   tags: always
   tasks:
     - name: "Check {{ minimal_ansible_version }} <= Ansible version < {{ maximal_ansible_version }}"

playbooks/boilerplate.yml

@@ -6,12 +6,35 @@
 # - to ensure we keep compatibility with old style group names
 # - to reduce inventory boilerplate (defining parent groups / empty groups)
 
-- name: Inventory setup and validation
+- name: Define groups for legacy less structured inventories
+  hosts: all
+  gather_facts: false
+  tags: always
+  tasks:
+    - name: Match needed groups by their old names or definition
+      vars:
+        group_mappings:
+          kube_control_plane:
+            - kube-master
+          kube_node:
+            - kube-node
+          calico_rr:
+            - calico-rr
+          no_floating:
+            - no-floating
+          k8s_cluster:
+            - kube_node
+            - kube_control_plane
+            - calico_rr
+      group_by:
+        key: "{{ (group_names | intersect(item.value) | length > 0) | ternary(item.key, '_all') }}"
+      loop: "{{ group_mappings | dict2items }}"
+
+- name: Check inventory settings
   hosts: all
   gather_facts: false
   tags: always
   roles:
-    - dynamic_groups
     - validate_inventory
 
 - name: Install bastion ssh config

playbooks/cluster.yml

@@ -3,7 +3,7 @@
   import_playbook: boilerplate.yml
 
 - name: Gather facts
-  import_playbook: internal_facts.yml
+  import_playbook: facts.yml
 
 - name: Prepare for etcd install
   hosts: k8s_cluster:etcd
@@ -38,7 +38,7 @@
   environment: "{{ proxy_disable_env }}"
   roles:
     - { role: kubespray_defaults }
-    - { role: kubernetes/control-plane, tags: control-plane }
+    - { role: kubernetes/control-plane, tags: master }
     - { role: kubernetes/client, tags: client }
     - { role: kubernetes-apps/cluster_roles, tags: cluster-roles }
 
@@ -52,7 +52,11 @@
     - { role: kubernetes/kubeadm, tags: kubeadm}
     - { role: kubernetes/node-label, tags: node-label }
     - { role: kubernetes/node-taint, tags: node-taint }
-    - { role: kubernetes-apps/common_crds }
+    - role: kubernetes-apps/gateway_api
+      when: gateway_api_enabled
+      tags: gateway_api
+      delegate_to: "{{ groups['kube_control_plane'][0] }}"
+      run_once: true
     - { role: network_plugin, tags: network }
 
 - name: Install Calico Route Reflector
@@ -71,7 +75,7 @@
   environment: "{{ proxy_disable_env }}"
   roles:
     - { role: kubespray_defaults }
-    - { role: win_nodes/kubernetes_patch, tags: ["control-plane", "win_nodes"] }
+    - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"] }
 
 - name: Install Kubernetes apps
   hosts: kube_control_plane
@@ -81,6 +85,7 @@
   roles:
     - { role: kubespray_defaults }
     - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller }
+    - { role: kubernetes-apps/network_plugin, tags: network }
     - { role: kubernetes-apps/policy_controller, tags: policy-controller }
     - { role: kubernetes-apps/ingress_controller, tags: ingress-controller }
     - { role: kubernetes-apps/external_provisioner, tags: external-provisioner }

playbooks/facts.yml

@@ -1,6 +1,39 @@
 ---
-- name: Common tasks for every playbooks
+- name: Bootstrap hosts for Ansible
-  import_playbook: boilerplate.yml
+  hosts: k8s_cluster:etcd:calico_rr
+  strategy: linear
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  gather_facts: false
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: bootstrap_os, tags: bootstrap_os}
 
 - name: Gather facts
-  import_playbook: internal_facts.yml
+  hosts: k8s_cluster:etcd:calico_rr
+  gather_facts: false
+  tags: always
+  tasks:
+    - name: Gather and compute network facts
+      import_role:
+        name: network_facts
+    - name: Gather minimal facts
+      setup:
+        gather_subset: '!all'
+
+    # filter match the following variables:
+    # ansible_default_ipv4
+    # ansible_default_ipv6
+    # ansible_all_ipv4_addresses
+    # ansible_all_ipv6_addresses
+    - name: Gather necessary facts (network)
+      setup:
+        gather_subset: '!all,!min,network'
+        filter: "ansible_*_ipv[46]*"
+
+    # filter match the following variables:
+    # ansible_memtotal_mb
+    # ansible_swaptotal_mb
+    - name: Gather necessary facts (hardware)
+      setup:
+        gather_subset: '!all,!min,hardware'
+        filter: "ansible_*total_mb"

playbooks/internal_facts.yml

@@ -1,39 +0,0 @@
----
-- name: Bootstrap hosts for Ansible
-  hosts: k8s_cluster:etcd:calico_rr
-  strategy: linear
-  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
-  gather_facts: false
-  environment: "{{ proxy_disable_env }}"
-  roles:
-    - { role: bootstrap_os, tags: bootstrap_os}
-
-- name: Gather facts
-  hosts: k8s_cluster:etcd:calico_rr
-  gather_facts: false
-  tags: always
-  tasks:
-    - name: Gather and compute network facts
-      import_role:
-        name: network_facts
-    - name: Gather minimal facts
-      setup:
-        gather_subset: '!all'
-
-    # filter match the following variables:
-    # ansible_default_ipv4
-    # ansible_default_ipv6
-    # ansible_all_ipv4_addresses
-    # ansible_all_ipv6_addresses
-    - name: Gather necessary facts (network)
-      setup:
-        gather_subset: '!all,!min,network'
-        filter: "ansible_*_ipv[46]*"
-
-    # filter match the following variables:
-    # ansible_memtotal_mb
-    # ansible_swaptotal_mb
-    - name: Gather necessary facts (hardware)
-      setup:
-        gather_subset: '!all,!min,hardware'
-        filter: "ansible_*total_mb"

playbooks/remove_node.yml

@@ -1,8 +1,6 @@
 ---
 - name: Validate nodes for removal
   hosts: localhost
-  gather_facts: false
-  become: false
   tasks:
     - name: Assert that nodes are specified for removal
       assert:
@@ -32,7 +30,7 @@
       when: pause_result.user_input | default('yes') != 'yes'
 
 - name: Gather facts
-  import_playbook: internal_facts.yml
+  import_playbook: facts.yml
   when: reset_nodes | default(True) | bool
 
 - name: Reset node

playbooks/reset.yml

@@ -3,7 +3,7 @@
   import_playbook: boilerplate.yml
 
 - name: Gather facts
-  import_playbook: internal_facts.yml
+  import_playbook: facts.yml
 
 - name: Reset cluster
   hosts: etcd:k8s_cluster:calico_rr

playbooks/scale.yml

@@ -3,7 +3,7 @@
   import_playbook: boilerplate.yml
 
 - name: Gather facts
-  import_playbook: internal_facts.yml
+  import_playbook: facts.yml
 
 - name: Install etcd
   vars:

playbooks/upgrade_cluster.yml

@@ -3,7 +3,7 @@
   import_playbook: boilerplate.yml
 
 - name: Gather facts
-  import_playbook: internal_facts.yml
+  import_playbook: facts.yml
 
 - name: Download images to ansible host cache via first kube_control_plane node
   hosts: kube_control_plane[0]
@@ -55,7 +55,7 @@
     - { role: kubernetes-apps/kubelet-csr-approver, tags: kubelet-csr-approver }
     - { role: container-engine, tags: "container-engine", when: deploy_container_engine }
     - { role: kubernetes/node, tags: node }
-    - { role: kubernetes/control-plane, tags: control-plane, upgrade_cluster_setup: true }
+    - { role: kubernetes/control-plane, tags: master, upgrade_cluster_setup: true }
     - { role: kubernetes/client, tags: client }
     - { role: kubernetes/node-label, tags: node-label }
     - { role: kubernetes/node-taint, tags: node-taint }
@@ -73,6 +73,7 @@
     - { role: kubespray_defaults }
     - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller }
     - { role: network_plugin, tags: network }
+    - { role: kubernetes-apps/network_plugin, tags: network }
     - { role: kubernetes-apps/policy_controller, tags: policy-controller }
 
 - name: Finally handle worker upgrades, based on given batch size
@@ -100,7 +101,7 @@
   environment: "{{ proxy_disable_env }}"
   roles:
     - { role: kubespray_defaults }
-    - { role: win_nodes/kubernetes_patch, tags: ["control-plane", "win_nodes"] }
+    - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"] }
 
 - name: Install Calico Route Reflector
   hosts: calico_rr

remove_node.yml

@@ -1,3 +0,0 @@
----
-- name: Remove node
-  ansible.builtin.import_playbook: playbooks/remove_node.yml

requirements.txt

@@ -1,6 +1,6 @@
-ansible==10.7.0
+ansible==9.13.0
 # Needed for community.crypto module
-cryptography==46.0.3
+cryptography==45.0.2
 # Needed for jinja2 json_query templating
 jmespath==1.0.1
 # Needed for ansible.utils.ipaddr

roles/adduser/molecule/default/molecule.yml

@@ -3,14 +3,12 @@ role_name_check: 1
 dependency:
   name: galaxy
 platforms:
-  - name: ubuntu22
+  - name: ubuntu20
-    cloud_image: ubuntu-2204
+    cloud_image: ubuntu-2004
     vm_cpu_cores: 1
     vm_memory: 512
 provisioner:
   name: ansible
-  env:
-    ANSIBLE_ROLES_PATH: ../../../
   config_options:
     defaults:
       callbacks_enabled: profile_tasks

roles/bastion-ssh-config/molecule/default/molecule.yml

@@ -4,13 +4,11 @@ dependency:
   name: galaxy
 platforms:
   - name: bastion-01
-    cloud_image: ubuntu-2204
+    cloud_image: ubuntu-2004
     vm_cpu_cores: 1
     vm_memory: 512
 provisioner:
   name: ansible
-  env:
-    ANSIBLE_ROLES_PATH: ../../../
   config_options:
     defaults:
       callbacks_enabled: profile_tasks

roles/bootstrap_os/defaults/main.yml

@@ -37,3 +37,8 @@ override_system_hostname: true
 is_fedora_coreos: false
 
 skip_http_proxy_on_os_packages: false
+
+# If this is true, debug information will be displayed but
+# may contain some private data, so it is recommended to set it to false
+# in the production environment.
+unsafe_show_logs: false

roles/bootstrap_os/molecule/default/molecule.yml

@@ -3,12 +3,12 @@ role_name_check: 1
 dependency:
   name: galaxy
 platforms:
-  - name: ubuntu22
+  - name: ubuntu20
-    cloud_image: ubuntu-2204
+    cloud_image: ubuntu-2004
     vm_cpu_cores: 1
     vm_memory: 512
-  - name: ubuntu24
+  - name: ubuntu22
-    cloud_image: ubuntu-2404
+    cloud_image: ubuntu-2204
     vm_cpu_cores: 1
     vm_memory: 512
   - name: almalinux9
@@ -21,8 +21,6 @@ platforms:
     vm_memory: 512
 provisioner:
   name: ansible
-  env:
-    ANSIBLE_ROLES_PATH: ../../../
   config_options:
     defaults:
       callbacks_enabled: profile_tasks

roles/bootstrap_os/tasks/almalinux.yml

@@ -1,3 +0,0 @@
----
-- name: Import Centos boostrap for Alma Linux
-  import_tasks: centos.yml

roles/bootstrap_os/tasks/centos.yml

@@ -17,6 +17,32 @@
   when: not skip_http_proxy_on_os_packages
 
 # For Oracle Linux install public repo
+- name: Download Oracle Linux public yum repo
+  get_url:
+    url: https://yum.oracle.com/public-yum-ol7.repo
+    dest: /etc/yum.repos.d/public-yum-ol7.repo
+    mode: "0644"
+  when:
+    - use_oracle_public_repo | default(true)
+    - '''ID="ol"'' in os_release.stdout_lines'
+    - (ansible_distribution_version | float) < 7.6
+  environment: "{{ proxy_env }}"
+
+- name: Enable Oracle Linux repo
+  community.general.ini_file:
+    dest: /etc/yum.repos.d/public-yum-ol7.repo
+    section: "{{ item }}"
+    option: enabled
+    value: "1"
+    mode: "0644"
+  with_items:
+    - ol7_latest
+    - ol7_addons
+    - ol7_developer_EPEL
+  when:
+    - use_oracle_public_repo | default(true)
+    - '''ID="ol"'' in os_release.stdout_lines'
+    - (ansible_distribution_version | float) < 7.6
 
 - name: Install EPEL for Oracle Linux repo package
   package:

roles/bootstrap_os/tasks/clear-linux-os.yml

@@ -0,0 +1,16 @@
+---
+# ClearLinux ships with Python installed
+
+- name: Install basic package to run containers
+  package:
+    name: containers-basic
+    state: present
+
+- name: Make sure docker service is enabled
+  systemd_service:
+    name: docker
+    masked: false
+    enabled: true
+    daemon_reload: true
+    state: started
+  become: true

roles/bootstrap_os/tasks/debian.yml

@@ -50,3 +50,15 @@
   become: true
   when:
     - need_bootstrap.rc != 0
+
+- name: Update Apt cache
+  raw: apt-get update --allow-releaseinfo-change
+  become: true
+  when:
+    - os_release_dict['ID'] == 'debian'
+    - os_release_dict['VERSION_ID'] in ["10", "11"]
+  register: bootstrap_update_apt_result
+  changed_when:
+    - '"changed its" in bootstrap_update_apt_result.stdout'
+    - '"value from" in bootstrap_update_apt_result.stdout'
+  ignore_errors: true

roles/bootstrap_os/tasks/main.yml

@@ -16,7 +16,8 @@
     tags:
     - facts
     with_first_found:
-    - files: &search_files
+    - &search
+      files:
       - "{{ os_release_dict['ID'] }}-{{ os_release_dict['VARIANT_ID'] }}.yml"
       - "{{ os_release_dict['ID'] }}.yml"
       paths:
@@ -25,8 +26,8 @@
   - name: Include tasks
     include_tasks: "{{ included_tasks_file }}"
     with_first_found:
-    - files: *search_files
+    - <<: *search
-      skip: true
+      paths: []
     loop_control:
       loop_var: included_tasks_file
 

roles/bootstrap_os/tasks/opensuse.yml

@@ -55,3 +55,31 @@
   become: true
   tags:
     - facts
+
+# Without this package, the get_url module fails when trying to handle https
+- name: Install python-cryptography
+  community.general.zypper:
+    name: python-cryptography
+    state: present
+    update_cache: true
+  become: true
+  when:
+    - ansible_distribution_version is version('15.4', '<')
+
+- name: Install python3-cryptography
+  community.general.zypper:
+    name: python3-cryptography
+    state: present
+    update_cache: true
+  become: true
+  when:
+    - ansible_distribution_version is version('15.4', '>=')
+
+# Nerdctl needs some basic packages to get an environment up
+- name: Install basic dependencies
+  community.general.zypper:
+    name:
+      - iptables
+      - apparmor-parser
+    state: present
+  become: true

roles/bootstrap_os/tasks/rocky.yml

@@ -1,3 +0,0 @@
----
-- name: Import Centos boostrap for Rocky Linux
-  import_tasks: centos.yml

roles/container-engine/containerd/defaults/main.yml

@@ -13,9 +13,10 @@ containerd_snapshotter: "overlayfs"
 containerd_runc_runtime:
   name: runc
   type: "io.containerd.runc.v2"
+  engine: ""
+  root: ""
   base_runtime_spec: cri-base.json
   options:
-    Root: ""
     SystemdCgroup: "{{ containerd_use_systemd_cgroup | ternary('true', 'false') }}"
     BinaryName: "{{ bin_dir }}/runc"
 
@@ -23,8 +24,8 @@ containerd_additional_runtimes: []
 # Example for Kata Containers as additional runtime:
 #  - name: kata
 #    type: "io.containerd.kata.v2"
-#    options:
+#    engine: ""
-#      Root: ""
+#    root: ""
 
 containerd_base_runtime_spec_rlimit_nofile: 65535
 
@@ -35,8 +36,8 @@ containerd_default_base_runtime_spec_patch:
         hard: "{{ containerd_base_runtime_spec_rlimit_nofile }}"
         soft: "{{ containerd_base_runtime_spec_rlimit_nofile }}"
 
-# Only for containerd < 2.1; discard unpacked layers to save disk space
+# Can help reduce disk usage
-# https://github.com/containerd/containerd/blob/release/2.1/docs/cri/config.md#image-pull-configuration-since-containerd-v21
+# https://github.com/containerd/containerd/discussions/6295
 containerd_discard_unpacked_layers: true
 
 containerd_base_runtime_specs:
@@ -63,8 +64,7 @@ containerd_registries_mirrors:
         skip_verify: false
 #        ca: ["/etc/certs/mirror.pem"]
 #        client: [["/etc/certs/client.pem", ""],["/etc/certs/client.cert", "/etc/certs/client.key"]]
-#        header:
+
-#          Authorization: "Basic XXX"
 containerd_max_container_log_line_size: 16384
 
 # If enabled it will allow non root users to use port numbers <1024
@@ -83,15 +83,6 @@ containerd_cfg_dir: /etc/containerd
 # Extra config to be put in {{ containerd_cfg_dir }}/config.toml literally
 containerd_extra_args: ''
 
-# Extra runtime configuration options to be injected into the containerd CRI runtime plugin section
-# [plugins."io.containerd.cri.v1.runtime"]. This is useful for adding containerd runtime
-# configuration options that aren't explicitly supported by Kubespray's default variables.
-# Example:
-#   containerd_extra_runtime_args:
-#     device_ownership_from_security_context: true
-#     another_option: "value"
-containerd_extra_runtime_args: {}
-
 # Configure registry auth (if applicable to secure/insecure registries)
 containerd_registry_auth: []
 #  - registry: 10.0.0.2:5000

roles/container-engine/containerd/molecule/default/molecule.yml

@@ -1,16 +1,16 @@
 ---
 role_name_check: 1
 platforms:
-  - cloud_image: ubuntu-2404
+  - cloud_image: ubuntu-2004
-    name: ubuntu24
+    name: ubuntu20
     vm_cpu_cores: 1
     vm_memory: 1024
     node_groups:
       - kube_control_plane
       - kube_node
       - k8s_cluster
-  - cloud_image: debian-12
+  - cloud_image: debian-11
-    name: debian12
+    name: debian11
     vm_cpu_cores: 1
     vm_memory: 1024
     node_groups:
@@ -35,6 +35,5 @@ provisioner:
       timeout: 120
   playbooks:
     create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml
-    prepare: ../../../molecule/prepare.yml
 verifier:
-  name: ansible
+  name: testinfra

roles/container-engine/containerd/molecule/default/prepare.yml

@@ -0,0 +1,30 @@
+---
+- name: Prepare
+  hosts: all
+  gather_facts: false
+  become: true
+  vars:
+    ignore_assert_errors: true
+  roles:
+    - role: kubespray_defaults
+    - role: bootstrap_os
+    - role: network_facts
+    - role: kubernetes/preinstall
+    - role: adduser
+      user: "{{ addusers.kube }}"
+  tasks:
+    - name: Download CNI
+      include_tasks: "../../../../download/tasks/download_file.yml"
+      vars:
+        download: "{{ download_defaults | combine(downloads.cni) }}"
+
+- name: Prepare CNI
+  hosts: all
+  gather_facts: false
+  become: true
+  vars:
+    ignore_assert_errors: true
+    kube_network_plugin: cni
+  roles:
+    - role: kubespray_defaults
+    - role: network_plugin/cni

roles/container-engine/containerd/molecule/default/tests/test_default.py

@@ -0,0 +1,55 @@
+import os
+import pytest
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_service(host):
+    svc = host.service("containerd")
+    assert svc.is_running
+    assert svc.is_enabled
+
+
+def test_version(host):
+    crictl = "/usr/local/bin/crictl"
+    path = "unix:///var/run/containerd/containerd.sock"
+    with host.sudo():
+        cmd = host.command(crictl + " --runtime-endpoint " + path + " version")
+    assert cmd.rc == 0
+    assert "RuntimeName:  containerd" in cmd.stdout
+
+
+@pytest.mark.parametrize('image, dest', [
+    ('quay.io/kubespray/hello-world:latest', '/tmp/hello-world.tar')
+])
+def test_image_pull_save_load(host, image, dest):
+    nerdctl = "/usr/local/bin/nerdctl"
+    dest_file = host.file(dest)
+
+    with host.sudo():
+        pull_cmd = host.command(nerdctl + " pull " + image)
+    assert pull_cmd.rc ==0
+
+    with host.sudo():
+        save_cmd = host.command(nerdctl + " save -o " + dest + " " + image)
+    assert save_cmd.rc == 0
+    assert dest_file.exists
+
+    with host.sudo():
+        load_cmd = host.command(nerdctl + " load < " + dest)
+    assert load_cmd.rc == 0
+
+
+@pytest.mark.parametrize('image', [
+    ('quay.io/kubespray/hello-world:latest')
+])
+def test_run(host, image):
+    nerdctl = "/usr/local/bin/nerdctl"
+
+    with host.sudo():
+        cmd = host.command(nerdctl + " -n k8s.io run " + image)
+    assert cmd.rc == 0
+    assert "Hello from Docker" in cmd.stdout

roles/container-engine/containerd/molecule/default/verify.yml

@@ -1,39 +0,0 @@
----
-- name: Test containerd CRI
-  import_playbook: ../../../molecule/test_cri.yml
-  vars:
-    container_manager: containerd
-    cri_socket: unix:///var/run/containerd/containerd.sock
-    cri_name: containerd
-
-- name: Test nerdctl
-  hosts: all
-  gather_facts: false
-  become: true
-  tasks:
-  - name: Get kubespray defaults
-    import_role:
-      name: ../../../../../kubespray_defaults
-  - name: Test nerdctl commands
-    command: "{{ bin_dir }}/nerdctl {{ item | join(' ') }}"
-    vars:
-      image: quay.io/kubespray/hello-world:latest
-    loop:
-    - - pull
-      - "{{ image }}"
-    - - save
-      - -o
-      - /tmp/hello-world.tar
-      - "{{ image }}"
-    - - load
-      - -i
-      - /tmp/hello-world.tar
-    - - -n
-      - k8s.io
-      - run
-      - "{{ image }}"
-    register: nerdctl
-  - name: Check log from running a container
-    assert:
-      that:
-      - ('Hello from Docker' in nerdctl.results[3].stdout)

roles/container-engine/containerd/tasks/main.yml

@@ -34,6 +34,8 @@
   with_items:
     - "{{ containerd_systemd_dir }}"
     - "{{ containerd_cfg_dir }}"
+    - "{{ containerd_storage_dir }}"
+    - "{{ containerd_state_dir }}"
 
 - name: Containerd | Write containerd proxy drop-in
   template:
@@ -71,8 +73,6 @@
   notify: Restart containerd
 
 - name: Containerd | Configure containerd registries
-  # mirror configuration can contain sensitive information on headers configuration
-  no_log: "{{ not (unsafe_show_logs | bool) }}"
   block:
     - name: Containerd | Create registry directories
       file:

roles/container-engine/containerd/templates/config.toml.j2

@@ -36,15 +36,6 @@ oom_score = {{ containerd_oom_score }}
     enable_cdi = true
     cdi_spec_dirs = ["/etc/cdi", "/var/run/cdi"]
 {% endif %}
-{% for key, value in containerd_extra_runtime_args.items() %}
-{% if value is string %}
-    {{ key }} = "{{ value }}"
-{% elif value is boolean %}
-    {{ key }} = {{ value | lower }}
-{% else %}
-    {{ key }} = {{ value }}
-{% endif %}
-{% endfor %}
 
    [plugins."io.containerd.cri.v1.runtime".containerd]
      default_runtime_name = "{{ containerd_default_runtime }}"
@@ -52,6 +43,8 @@ oom_score = {{ containerd_oom_score }}
 {% for runtime in [containerd_runc_runtime] + containerd_additional_runtimes %}
        [plugins."io.containerd.cri.v1.runtime".containerd.runtimes.{{ runtime.name }}]
          runtime_type = "{{ runtime.type }}"
+         runtime_engine = "{{ runtime.engine }}"
+         runtime_root = "{{ runtime.root }}"
 {% if runtime.base_runtime_spec is defined %}
          base_runtime_spec = "{{ containerd_cfg_dir }}/{{ runtime.base_runtime_spec }}"
 {% endif %}
@@ -76,9 +69,7 @@ oom_score = {{ containerd_oom_score }}
 
   [plugins."io.containerd.cri.v1.images"]
     snapshotter = "{{ containerd_snapshotter }}"
-{% if containerd_discard_unpacked_layers and containerd_version is version('2.1.0', '<') %}
     discard_unpacked_layers = {{ containerd_discard_unpacked_layers | lower }}
-{% endif %}
     image_pull_progress_timeout = "{{ containerd_image_pull_progress_timeout }}"
   [plugins."io.containerd.cri.v1.images".pinned_images]
     sandbox = "{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"

roles/container-engine/containerd/templates/hosts.toml.j2

@@ -10,10 +10,4 @@ server = "{{ item.server | default("https://" + item.prefix) }}"
 {% if mirror.client is defined %}
   client = [{% for pair in mirror.client %}["{{ pair[0] }}", "{{ pair[1] }}"]{% if not loop.last %},{% endif %}{% endfor %}]
 {% endif %}
-{% if mirror.header is defined %}
-  [host."{{ mirror.host }}".header]
-{% for key, value in mirror.header.items() %}
-    {{ key }} = ["{{ ([ value ] | flatten ) | join('","') }}"]
-{% endfor %}
-{% endif %}
 {% endfor %}

roles/container-engine/cri-dockerd/molecule/default/molecule.yml

@@ -7,8 +7,8 @@ platforms:
     vm_memory: 1024
     node_groups:
       - kube_control_plane
-  - name: ubuntu22
+  - name: ubuntu20
-    cloud_image: ubuntu-2204
+    cloud_image: ubuntu-2004
     vm_cpu_cores: 1
     vm_memory: 1024
     node_groups:
@@ -25,10 +25,7 @@ provisioner:
     group_vars:
       all:
         become: true
-      k8s_cluster:
-        container_manager: docker
   playbooks:
     create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml
-    prepare: ../../../molecule/prepare.yml
 verifier:
-  name: ansible
+  name: testinfra

roles/container-engine/cri-dockerd/molecule/default/prepare.yml

@@ -0,0 +1,48 @@
+---
+- name: Prepare
+  hosts: all
+  become: true
+  roles:
+    - role: kubespray_defaults
+    - role: bootstrap_os
+    - role: adduser
+      user: "{{ addusers.kube }}"
+  tasks:
+    - name: Download CNI
+      include_tasks: "../../../../download/tasks/download_file.yml"
+      vars:
+        download: "{{ download_defaults | combine(downloads.cni) }}"
+
+- name: Prepare container runtime
+  hosts: all
+  become: true
+  vars:
+    container_manager: containerd
+    kube_network_plugin: cni
+  roles:
+    - role: kubespray_defaults
+    - role: network_plugin/cni
+  tasks:
+    - name: Copy test container files
+      copy:
+        src: "{{ item }}"
+        dest: "/tmp/{{ item }}"
+        owner: root
+        mode: "0644"
+      with_items:
+        - container.json
+        - sandbox.json
+    - name: Create /etc/cni/net.d directory
+      file:
+        path: /etc/cni/net.d
+        state: directory
+        owner: "{{ kube_owner }}"
+        mode: "0755"
+    - name: Setup CNI
+      copy:
+        src: "{{ item }}"
+        dest: "/etc/cni/net.d/{{ item }}"
+        owner: root
+        mode: "0644"
+      with_items:
+        - 10-mynet.conf

roles/container-engine/cri-dockerd/molecule/default/tests/test_default.py

@@ -0,0 +1,19 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_run_pod(host):
+    run_command = "/usr/local/bin/crictl run --with-pull /tmp/container.json /tmp/sandbox.json"
+    with host.sudo():
+        cmd = host.command(run_command)
+    assert cmd.rc == 0
+
+    with host.sudo():
+      log_f = host.file("/tmp/cri-dockerd1.0.log")
+
+      assert log_f.exists
+      assert b"Hello from Docker" in log_f.content

roles/container-engine/cri-dockerd/molecule/default/verify.yml

@@ -1,15 +0,0 @@
----
-- name: Test cri-dockerd
-  import_playbook: ../../../molecule/test_cri.yml
-  vars:
-    container_manager: cri-dockerd
-    cri_socket: unix:///var/run/cri-dockerd.sock
-    cri_name: docker
-
-- name: Test running a container with docker
-  import_playbook: ../../../molecule/test_runtime.yml
-  vars:
-    container_runtime: docker
-  # cri-dockerd does not support multiple runtime handler before 0.4.0
-  # https://github.com/Mirantis/cri-dockerd/pull/350
-  # TODO: check this when we upgrade cri-dockerd