Kubevirt: use Ignition cloud config

Signed-off-by: ChengHao Yang <17496418+tico88612@users.noreply.github.com>

.github/workflows/upgrade-patch-versions-schedule.yml

@@ -8,7 +8,6 @@ on:
 permissions: {}
 jobs:
   get-releases-branches:
-    if: github.repository == 'kubernetes-sigs/kubespray'
     runs-on: ubuntu-latest
     outputs:
       branches: ${{ steps.get-branches.outputs.data }}

.gitlab-ci.yml

@@ -59,6 +59,33 @@ before_script:
     - pre-commit            # lint
     - vagrant-validate      # lint
 
+# For failfast, at least 1 job must be defined in .gitlab-ci.yml
+# Premoderated with manual actions
+ci-not-authorized:
+  stage: build
+  before_script: []
+  after_script: []
+  rules:
+    # LGTM or ok-to-test labels
+    - if: $PR_LABELS =~ /.*,(lgtm|approved|ok-to-test).*|^(lgtm|approved|ok-to-test).*/i
+      variables:
+        CI_OK_TO_TEST: '0'
+      when: always
+    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "trigger"
+      variables:
+        CI_OK_TO_TEST: '0'
+    - if: $CI_COMMIT_BRANCH == "master"
+      variables:
+        CI_OK_TO_TEST: '0'
+    - when: always
+      variables:
+        CI_OK_TO_TEST: '1'
+  script:
+    - exit $CI_OK_TO_TEST
+  tags:
+    - ffci
+  needs: []
+
 include:
   - .gitlab-ci/build.yml
   - .gitlab-ci/lint.yml

.gitlab-ci/kubevirt.yml

@@ -15,7 +15,7 @@
     - ci-not-authorized
 
 # TODO: generate testcases matrixes from the files in tests/files/
-# this is needed to avoid the need for PR rebasing when a job was added or removed in the target branch
+# this is needed to avoid the need for PR rebasing when a job was added or remvoed in the target branch
 # (currently, a removed job in the target branch breaks the tests, because the
 # pipeline definition is parsed by gitlab before the rebase.sh script)
 # CI template for PRs
@@ -27,8 +27,6 @@ pr:
       allow_failure: true
     - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
       when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
     - when: manual
       allow_failure: true
   extends: .kubevirt
@@ -55,7 +53,7 @@ pr:
           - ubuntu22-calico-all-in-one
           - ubuntu22-calico-all-in-one-upgrade
           - ubuntu24-calico-etcd-datastore
-          - ubuntu24-ha-separate-etcd
+          - flatcar4081-calico
 
 # The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
 ubuntu20-calico-all-in-one:
@@ -66,8 +64,6 @@ ubuntu20-calico-all-in-one:
   rules:
     - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
       when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
     - when: manual
       allow_failure: true
 
@@ -77,8 +73,6 @@ pr_full:
   rules:
     - if: $PR_LABELS =~ /.*ci-full.*/
       when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
     # Else run as manual
     - when: manual
       allow_failure: true
@@ -115,8 +109,6 @@ pr_extended:
   rules:
     - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
       when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
     - when: manual
       allow_failure: true
   parallel:
@@ -136,13 +128,13 @@ pr_extended:
           - ubuntu24-all-in-one-docker
           - ubuntu24-calico-all-in-one
 
-# TODO: migrate to pr-full, fix the broken ones
+# Enabled when PERIODIC_CI_ENABLED var is set
 periodic:
+  only:
+    variables:
+      - $PERIODIC_CI_ENABLED
   allow_failure: true
   extends: .kubevirt
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
   parallel:
     matrix:
       - TESTCASE:

.gitlab-ci/molecule.yml

@@ -1,13 +1,8 @@
 ---
 .molecule:
   tags: [ffci]
-  rules: # run on ci-short as well
-  - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-    when: on_success
-  - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-    when: on_success
-  - when: manual
-    allow_failure: true
+  only: [/^pr-.*$/]
+  except: ['triggers']
   stage: deploy-part1
   image: $PIPELINE_IMAGE
   needs:
@@ -16,10 +11,7 @@
   script:
   - ./tests/scripts/molecule_run.sh
   after_script:
-  - rm -fr molecule_logs
-  - mkdir -p molecule_logs
-  - find ~/.cache/molecule/ \( -name '*.out' -o -name '*.err' \) -type f | xargs tar -uf molecule_logs/molecule.tar
-  - gzip molecule_logs/molecule.tar
+  - ./tests/scripts/molecule_logs.sh
   artifacts:
     when: always
     paths:
@@ -37,19 +29,25 @@ molecule:
       - container-engine/cri-o
       - adduser
       - bastion-ssh-config
-      - bootstrap_os
+      - bootstrap-os
 
+# CI template for periodic CI jobs
+# Enabled when PERIODIC_CI_ENABLED var is set
 molecule_full:
+  only:
+    variables:
+    - $PERIODIC_CI_ENABLED
   allow_failure: true
-  rules:
-  - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-    when: on_success
-  - when: manual
-    allow_failure: true
   extends: molecule
   parallel:
     matrix:
     - ROLE:
+      - container-engine/cri-dockerd
+      - container-engine/containerd
+      - container-engine/cri-o
+      - adduser
+      - bastion-ssh-config
+      - bootstrap-os
       # FIXME : tests below are perma-failing
       - container-engine/kata-containers
       - container-engine/gvisor

.gitlab-ci/vagrant.yml

@@ -13,6 +13,8 @@ vagrant:
     VAGRANT_HOME: "$CI_PROJECT_DIR/.vagrant.d"
     PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
   tags: [ffci-vm-large]
+  # only: [/^pr-.*$/]
+  # except: ['triggers']
   image: quay.io/kubespray/vm-kubespray-ci:v13
   services: []
   before_script:
@@ -40,8 +42,6 @@ vagrant:
   rules:
     - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
       when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
   parallel:
     matrix:
       - TESTCASE:

Dockerfile

@@ -35,8 +35,8 @@ RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 RUN OS_ARCHITECTURE=$(dpkg --print-architecture) \
-    && curl -L "https://dl.k8s.io/release/v1.32.4/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
-    && echo "$(curl -L "https://dl.k8s.io/release/v1.32.4/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
+    && curl -L "https://dl.k8s.io/release/v1.32.3/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
+    && echo "$(curl -L "https://dl.k8s.io/release/v1.32.3/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
     && chmod a+x /usr/local/bin/kubectl
 
 COPY *.yml ./

README.md

@@ -111,14 +111,14 @@ Note:
 <!-- BEGIN ANSIBLE MANAGED BLOCK -->
 
 - Core
-  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.32.4
+  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.32.3
   - [etcd](https://github.com/etcd-io/etcd) 3.5.16
   - [docker](https://www.docker.com/) 28.0
-  - [containerd](https://containerd.io/) 2.0.5
+  - [containerd](https://containerd.io/) 2.0.3
   - [cri-o](http://cri-o.io/) 1.32.0 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
   - [cni-plugins](https://github.com/containernetworking/plugins) 1.4.1
-  - [calico](https://github.com/projectcalico/calico) 3.29.3
+  - [calico](https://github.com/projectcalico/calico) 3.29.2
   - [cilium](https://github.com/cilium/cilium) 1.15.9
   - [flannel](https://github.com/flannel-io/flannel) 0.22.0
   - [kube-ovn](https://github.com/alauda/kube-ovn) 1.12.21

contrib/offline/README.md

@@ -31,7 +31,7 @@ manage-offline-container-images.sh   register
 
 ## generate_list.sh
 
-This script generates the list of downloaded files and the list of container images by `roles/kubespray_defaults/defaults/main/download.yml` file.
+This script generates the list of downloaded files and the list of container images by `roles/kubespray-defaults/defaults/main/download.yml` file.
 
 Run this script will execute `generate_list.yml` playbook in kubespray root directory and generate four files,
 all downloaded files url in files.list, all container images in images.list, jinja2 templates in *.template.

contrib/offline/generate_list.sh

@@ -5,7 +5,7 @@ CURRENT_DIR=$(cd $(dirname $0); pwd)
 TEMP_DIR="${CURRENT_DIR}/temp"
 REPO_ROOT_DIR="${CURRENT_DIR%/contrib/offline}"
 
-: ${DOWNLOAD_YML:="roles/kubespray_defaults/defaults/main/download.yml"}
+: ${DOWNLOAD_YML:="roles/kubespray-defaults/defaults/main/download.yml"}
 
 mkdir -p ${TEMP_DIR}
 
@@ -19,7 +19,7 @@ sed -n '/^downloads:/,/download_defaults:/p' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \
     | sed 'N;s#\n# #g' | tr ' ' ':' | sed 's/\"//g' > ${TEMP_DIR}/images.list.template
 
 # add kube-* images to images list template
-# Those container images are downloaded by kubeadm, then roles/kubespray_defaults/defaults/main/download.yml
+# Those container images are downloaded by kubeadm, then roles/kubespray-defaults/defaults/main/download.yml
 # doesn't contain those images. That is reason why here needs to put those images into the
 # list separately.
 KUBE_IMAGES="kube-apiserver kube-controller-manager kube-scheduler kube-proxy"

contrib/offline/generate_list.yml

@@ -5,7 +5,7 @@
 
   roles:
     # Just load default variables from roles.
-    - role: kubespray_defaults
+    - role: kubespray-defaults
       when: false
     - role: download
       when: false

docs/CNI/cilium.md

@@ -54,10 +54,6 @@ cilium_loadbalancer_ip_pools:
   - name: "blue-pool"
     cidrs:
       - "10.0.10.0/24"
-    ranges:
-      - start: "20.0.20.100"
-        stop: "20.0.20.200"
-      - start: "1.2.3.4"
 ```
 
 For further information, check [LB IPAM documentation](https://docs.cilium.io/en/stable/network/lb-ipam/)

docs/CRI/containerd.md

@@ -68,8 +68,8 @@ containerd_runc_runtime:
   engine: ""
   root: ""
   options:
-    SystemdCgroup: "false"
-    BinaryName: /usr/local/bin/my-runc
+    systemdCgroup: "false"
+    binaryName: /usr/local/bin/my-runc
   base_runtime_spec: cri-base.json
 ```
 

docs/advanced/proxy.md

@@ -1,6 +1,6 @@
 # Setting up Environment Proxy
 
-If you set http and https proxy, all nodes and loadbalancer will be excluded from proxy with generating no_proxy variable in `roles/kubespray_defaults/tasks/no_proxy.yml`, if you have additional resources for exclude add them to `additional_no_proxy` variable. If you want fully override your `no_proxy` setting, then fill in just `no_proxy` and no nodes or loadbalancer addresses will be added to no_proxy.
+If you set http and https proxy, all nodes and loadbalancer will be excluded from proxy with generating no_proxy variable in `roles/kubespray-defaults/tasks/no_proxy.yml`, if you have additional resources for exclude add them to `additional_no_proxy` variable. If you want fully override your `no_proxy` setting, then fill in just `no_proxy` and no nodes or loadbalancer addresses will be added to no_proxy.
 
 ## Set proxy for http and https
 

docs/ansible/ansible.md

@@ -62,7 +62,7 @@ The following tags are defined in playbooks:
 | aws-ebs-csi-driver             | Configuring csi driver: aws-ebs                       |
 | azure-csi-driver               | Configuring csi driver: azure                         |
 | bastion                        | Setup ssh config for bastion                          |
-| bootstrap_os                   | Anything related to host OS configuration             |
+| bootstrap-os                   | Anything related to host OS configuration             |
 | calico                         | Network plugin Calico                                 |
 | calico_rr                      | Configuring Calico route reflector                    |
 | cert-manager                   | Configuring certificate manager for K8s               |
@@ -167,7 +167,7 @@ Example command to filter and apply only DNS configuration tasks and skip
 everything else related to host OS configuration and downloading images of containers:
 
 ```ShellSession
-ansible-playbook -i inventory/sample/hosts.ini cluster.yml --tags preinstall,facts --skip-tags=download,bootstrap_os
+ansible-playbook -i inventory/sample/hosts.ini cluster.yml --tags preinstall,facts --skip-tags=download,bootstrap-os
 ```
 
 And this play only removes the K8s cluster DNS resolver IP from hosts' /etc/resolv.conf files:

docs/ansible/vars.md

@@ -180,7 +180,7 @@ and ``kube_pods_subnet``, for example from the ``172.18.0.0/16``.
 
 IPv4 stack enable by *ipv4_stack* is set to ``true``, by default.
 IPv6 stack enable by *ipv6_stack* is set to ``false`` by default.
-This will use the default IPv4 and IPv6 subnets specified in the defaults file in the ``kubespray_defaults`` role, unless overridden of course. The default config will give you room for up to 256 nodes with 126 pods per node, and up to 4096 services.
+This will use the default IPv4 and IPv6 subnets specified in the defaults file in the ``kubespray-defaults`` role, unless overridden of course. The default config will give you room for up to 256 nodes with 126 pods per node, and up to 4096 services.
 Set both variables to ``true`` for Dual Stack mode.
 IPv4 has higher priority in Dual Stack mode(e.g. in variables `main_ip`, `main_access_ip` and other).
 You can also make IPv6 only clusters with ``false`` in *ipv4_stack*.

docs/developers/ci.md

@@ -13,6 +13,7 @@ debian11 |  :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :w
 debian12 |  :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: |
 fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
 fedora40 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+flatcar4081 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 opensuse15 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
@@ -32,6 +33,7 @@ debian11 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora40 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+flatcar4081 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 opensuse15 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
@@ -51,6 +53,7 @@ debian11 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora40 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+flatcar4081 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 opensuse15 |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
 rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |

docs/operating_systems/bootstrap-os.md

@@ -1,4 +1,4 @@
-# bootstrap_os
+# bootstrap-os
 
 Bootstrap an Ansible host to be able to run Ansible modules.
 
@@ -48,8 +48,8 @@ Remember to disable fact gathering since Python might not be present on hosts.
 - hosts: all
   gather_facts: false  # not all hosts might be able to run modules yet
   roles:
-    - kubespray_defaults
-    - bootstrap_os
+    - kubespray-defaults
+    - bootstrap-os
 ```
 
 ## License

docs/operations/offline-environment.md

@@ -75,17 +75,17 @@ quay_image_repo: "{{ registry_host }}"
 github_image_repo: "{{ registry_host }}"
 
 local_path_provisioner_helper_image_repo: "{{ registry_host }}/busybox"
-kubeadm_download_url: "{{ files_repo }}/kubernetes/v{{ kube_version }}/kubeadm"
-kubectl_download_url: "{{ files_repo }}/kubernetes/v{{ kube_version }}/kubectl"
-kubelet_download_url: "{{ files_repo }}/kubernetes/v{{ kube_version }}/kubelet"
+kubeadm_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubeadm"
+kubectl_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubectl"
+kubelet_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubelet"
 # etcd is optional if you **DON'T** use etcd_deployment=host
-etcd_download_url: "{{ files_repo }}/kubernetes/etcd/etcd-v{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
-cni_download_url: "{{ files_repo }}/kubernetes/cni/cni-plugins-linux-{{ image_arch }}-v{{ cni_version }}.tgz"
-crictl_download_url: "{{ files_repo }}/kubernetes/cri-tools/crictl-v{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
+etcd_download_url: "{{ files_repo }}/kubernetes/etcd/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
+cni_download_url: "{{ files_repo }}/kubernetes/cni/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
+crictl_download_url: "{{ files_repo }}/kubernetes/cri-tools/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
 # If using Calico
-calicoctl_download_url: "{{ files_repo }}/kubernetes/calico/v{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
+calicoctl_download_url: "{{ files_repo }}/kubernetes/calico/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
 # If using Calico with kdd
-calico_crds_download_url: "{{ files_repo }}/kubernetes/calico/v{{ calico_version }}.tar.gz"
+calico_crds_download_url: "{{ files_repo }}/kubernetes/calico/{{ calico_version }}.tar.gz"
 # Containerd
 containerd_download_url: "{{ files_repo }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"
 runc_download_url: "{{ files_repo }}/runc.{{ image_arch }}"
@@ -136,7 +136,7 @@ If you use the settings like the one above, you'll need to define in your invent
 
 * `registry_host`: Container image registry. If you _don't_ use the same repository path for the container images that
   the ones defined
-  in [kubesprays-defaults's role defaults](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/kubespray_defaults/defaults/main/download.yml)
+  in [kubesprays-defaults's role defaults](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/kubespray-defaults/defaults/main/download.yml)
   , you need to override the `*_image_repo` for these container images. If you want to make your life easier, use the
   same repository path, you won't have to override anything else.
 * `registry_addr`: Container image registry, but only have [domain or ip]:[port].

docs/operations/upgrades.md

@@ -15,6 +15,7 @@ versions. Here are all version vars for each component:
 * calico_cni_version
 * weave_version
 * flannel_version
+* kubedns_version
 
 > **Warning**
 > [Attempting to upgrade from an older release straight to the latest release is unsupported and likely to break something](https://github.com/kubernetes-sigs/kubespray/issues/3849#issuecomment-451386515)
@@ -83,7 +84,7 @@ If you don't want to upgrade all nodes in one run, you can use `--limit` [patter
 Before using `--limit` run playbook `facts.yml` without the limit to refresh facts cache for all nodes:
 
 ```ShellSession
-ansible-playbook playbooks/facts.yml -b -i inventory/sample/hosts.ini
+ansible-playbook facts.yml -b -i inventory/sample/hosts.ini
 ```
 
 After this upgrade control plane and etcd groups [#5147](https://github.com/kubernetes-sigs/kubespray/issues/5147):

extra_playbooks/migrate_openstack_provider.yml

@@ -12,7 +12,7 @@
   hosts: kube_control_plane[0]
   tasks:
     - name: Include kubespray-default variables
-      include_vars: ../roles/kubespray_defaults/defaults/main/main.yml
+      include_vars: ../roles/kubespray-defaults/defaults/main/main.yml
     - name: Copy get_cinder_pvs.sh to first control plane node
       copy:
         src: get_cinder_pvs.sh

extra_playbooks/upgrade-only-k8s.yml

@@ -14,7 +14,7 @@
   hosts: localhost
   gather_facts: false
   roles:
-    - { role: kubespray_defaults}
+    - { role: kubespray-defaults}
     - { role: bastion-ssh-config, tags: ["localhost", "bastion"]}
 
 - name: Bootstrap hosts OS for Ansible
@@ -22,18 +22,18 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   gather_facts: false
   vars:
-    # Need to disable pipelining for bootstrap_os as some systems have requiretty in sudoers set, which makes pipelining
-    # fail. bootstrap_os fixes this on these systems, so in later plays it can be enabled.
+    # Need to disable pipelining for bootstrap-os as some systems have requiretty in sudoers set, which makes pipelining
+    # fail. bootstrap-os fixes this on these systems, so in later plays it can be enabled.
     ansible_ssh_pipelining: false
   roles:
-    - { role: kubespray_defaults}
-    - { role: bootstrap_os, tags: bootstrap_os}
+    - { role: kubespray-defaults}
+    - { role: bootstrap-os, tags: bootstrap-os}
 
 - name: Preinstall
   hosts: k8s_cluster:etcd:calico_rr
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
-    - { role: kubespray_defaults}
+    - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
 
 - name: Handle upgrades to control plane components first to maintain backwards compat.
@@ -41,7 +41,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   serial: 1
   roles:
-    - { role: kubespray_defaults}
+    - { role: kubespray-defaults}
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
     - { role: kubernetes/node, tags: node }
     - { role: kubernetes/control-plane, tags: master, upgrade_cluster_setup: true }
@@ -54,8 +54,8 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   serial: "{{ serial | default('20%') }}"
   roles:
-    - { role: kubespray_defaults}
+    - { role: kubespray-defaults}
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
     - { role: kubernetes/node, tags: node }
     - { role: upgrade/post-upgrade, tags: post-upgrade }
-    - { role: kubespray_defaults}
+    - { role: kubespray-defaults}

inventory/sample/group_vars/all/all.yml

@@ -57,7 +57,7 @@ loadbalancer_apiserver_healthcheck_port: 8081
 # https_proxy: ""
 # https_proxy_cert_file: ""
 
-## Refer to roles/kubespray_defaults/defaults/main/main.yml before modifying no_proxy
+## Refer to roles/kubespray-defaults/defaults/main/main.yml before modifying no_proxy
 # no_proxy: ""
 
 ## Some problems may occur when downloading files over https proxy due to ansible bug

inventory/sample/group_vars/all/oci.yml

@@ -43,6 +43,7 @@
 #   ocid1.subnet.oc1.phx.aaaaaaaahuxrgvs65iwdz7ekwgg3l5gyah7ww5klkwjcso74u3e4i64hvtvq: ocid1.securitylist.oc1.iad.aaaaaaaaqti5jsfvyw6ejahh7r4okb2xbtuiuguswhs746mtahn72r7adt7q
 ## If oci_use_instance_principals is true, you do not need to set the region, tenancy, user, key, passphrase, or fingerprint
 # oci_use_instance_principals: false
+# oci_cloud_controller_version: 0.6.0
 ## If you would like to control OCI query rate limits for the controller
 # oci_rate_limit:
 #   rate_limit_qps_read:

inventory/sample/group_vars/all/offline.yml

@@ -18,9 +18,9 @@
 # quay_image_repo: "{{ registry_host }}"
 
 ## Kubernetes components
-# kubeadm_download_url: "{{ files_repo }}/dl.k8s.io/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubeadm"
-# kubectl_download_url: "{{ files_repo }}/dl.k8s.io/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubectl"
-# kubelet_download_url: "{{ files_repo }}/dl.k8s.io/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubelet"
+# kubeadm_download_url: "{{ files_repo }}/dl.k8s.io/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubeadm"
+# kubectl_download_url: "{{ files_repo }}/dl.k8s.io/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubectl"
+# kubelet_download_url: "{{ files_repo }}/dl.k8s.io/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubelet"
 
 
 ## Two options - Override entire repository or override only a single binary.
@@ -33,24 +33,24 @@
 
 ## [Optional] 2 - Override a specific binary
 ## CNI Plugins
-# cni_download_url: "{{ files_repo }}/github.com/containernetworking/plugins/releases/download/v{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-v{{ cni_version }}.tgz"
+# cni_download_url: "{{ files_repo }}/github.com/containernetworking/plugins/releases/download/{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
 
 ## cri-tools
-# crictl_download_url: "{{ files_repo }}/github.com/kubernetes-sigs/cri-tools/releases/download/v{{ crictl_version }}/crictl-v{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
+# crictl_download_url: "{{ files_repo }}/github.com/kubernetes-sigs/cri-tools/releases/download/{{ crictl_version }}/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
 
 ## [Optional] etcd: only if you use etcd_deployment=host
-# etcd_download_url: "{{ files_repo }}/github.com/etcd-io/etcd/releases/download/v{{ etcd_version }}/etcd-v{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
+# etcd_download_url: "{{ files_repo }}/github.com/etcd-io/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
 
 # [Optional] Calico: If using Calico network plugin
-# calicoctl_download_url: "{{ files_repo }}/github.com/projectcalico/calico/releases/download/v{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
+# calicoctl_download_url: "{{ files_repo }}/github.com/projectcalico/calico/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
 # [Optional] Calico with kdd: If using Calico network plugin with kdd datastore
-# calico_crds_download_url: "{{ files_repo }}/github.com/projectcalico/calico/archive/v{{ calico_version }}.tar.gz"
+# calico_crds_download_url: "{{ files_repo }}/github.com/projectcalico/calico/archive/{{ calico_version }}.tar.gz"
 
 # [Optional] Cilium: If using Cilium network plugin
-# ciliumcli_download_url: "{{ files_repo }}/github.com/cilium/cilium-cli/releases/download/v{{ cilium_cli_version }}/cilium-linux-{{ image_arch }}.tar.gz"
+# ciliumcli_download_url: "{{ files_repo }}/github.com/cilium/cilium-cli/releases/download/{{ cilium_cli_version }}/cilium-linux-{{ image_arch }}.tar.gz"
 
 # [Optional] helm: only if you set helm_enabled: true
-# helm_download_url: "{{ files_repo }}/get.helm.sh/helm-v{{ helm_version }}-linux-{{ image_arch }}.tar.gz"
+# helm_download_url: "{{ files_repo }}/get.helm.sh/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz"
 
 # [Optional] crun: only if you set crun_enabled: true
 # crun_download_url: "{{ files_repo }}/github.com/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}"
@@ -62,13 +62,13 @@
 # cri_dockerd_download_url: "{{ files_repo }}/github.com/Mirantis/cri-dockerd/releases/download/v{{ cri_dockerd_version }}/cri-dockerd-{{ cri_dockerd_version }}.{{ image_arch }}.tgz"
 
 # [Optional] runc: if you set container_manager to containerd or crio
-# runc_download_url: "{{ files_repo }}/github.com/opencontainers/runc/releases/download/v{{ runc_version }}/runc.{{ image_arch }}"
+# runc_download_url: "{{ files_repo }}/github.com/opencontainers/runc/releases/download/{{ runc_version }}/runc.{{ image_arch }}"
 
 # [Optional] cri-o: only if you set container_manager: crio
 # crio_download_base: "download.opensuse.org/repositories/devel:kubic:libcontainers:stable"
 # crio_download_crio: "http://{{ crio_download_base }}:/cri-o:/"
-# crio_download_url: "{{ files_repo }}/storage.googleapis.com/cri-o/artifacts/cri-o.{{ image_arch }}.v{{ crio_version }}.tar.gz"
-# skopeo_download_url: "{{ files_repo }}/github.com/lework/skopeo-binary/releases/download/v{{ skopeo_version }}/skopeo-linux-{{ image_arch }}"
+# crio_download_url: "{{ files_repo }}/storage.googleapis.com/cri-o/artifacts/cri-o.{{ image_arch }}.{{ crio_version }}.tar.gz"
+# skopeo_download_url: "{{ files_repo }}/github.com/lework/skopeo-binary/releases/download/{{ skopeo_version }}/skopeo-linux-{{ image_arch }}"
 
 # [Optional] containerd: only if you set container_runtime: containerd
 # containerd_download_url: "{{ files_repo }}/github.com/containerd/containerd/releases/download/v{{ containerd_version }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"

inventory/sample/group_vars/all/openstack.yml

@@ -1,4 +1,5 @@
 ## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461)
+# openstack_blockstorage_version: "v1/v2/auto (default)"
 # openstack_blockstorage_ignore_volume_az: yes
 ## When OpenStack is used, if LBaaSv2 is available you can enable it with the following 2 variables.
 # openstack_lbaas_enabled: True

inventory/sample/group_vars/all/vsphere.yml

@@ -7,6 +7,26 @@
 # external_vsphere_datacenter: "DATACENTER_name"
 # external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id"
 
+## Vsphere version where located VMs
+# external_vsphere_version: "6.7u3"
+
+## Tags for the external vSphere Cloud Provider images
+## registry.k8s.io/cloud-pv-vsphere/cloud-provider-vsphere
+# external_vsphere_cloud_controller_image_tag: "v1.31.0"
+## registry.k8s.io/csi-vsphere/syncer
+# vsphere_syncer_image_tag: "v3.3.1"
+## registry.k8s.io/sig-storage/csi-attacher
+# vsphere_csi_attacher_image_tag: "v3.4.0"
+## registry.k8s.io/csi-vsphere/driver
+# vsphere_csi_controller: "v3.3.1"
+## registry.k8s.io/sig-storage/livenessprobe
+# vsphere_csi_liveness_probe_image_tag: "v2.6.0"
+## registry.k8s.io/sig-storage/csi-provisioner
+# vsphere_csi_provisioner_image_tag: "v3.1.0"
+## registry.k8s.io/sig-storage/csi-resizer
+## makes sense only for vSphere version >=7.0
+# vsphere_csi_resizer_tag: "v1.3.0"
+
 ## To use vSphere CSI plugin to provision volumes set this value to true
 # vsphere_csi_enabled: true
 # vsphere_csi_controller_replicas: 1

inventory/sample/group_vars/k8s_cluster/addons.yml

@@ -67,6 +67,7 @@ local_volume_provisioner_enabled: false
 
 # Gateway API CRDs
 gateway_api_enabled: false
+# gateway_api_experimental_channel: false
 
 # Nginx ingress controller deployment
 ingress_nginx_enabled: false
@@ -148,6 +149,7 @@ cert_manager_enabled: false
 metallb_enabled: false
 metallb_speaker_enabled: "{{ metallb_enabled }}"
 metallb_namespace: "metallb-system"
+# metallb_version: 0.13.9
 # metallb_protocol: "layer2"
 # metallb_port: "7472"
 # metallb_memberlist_port: "7946"
@@ -209,6 +211,7 @@ metallb_namespace: "metallb-system"
 #             - pool2
 
 argocd_enabled: false
+# argocd_version: 2.14.5
 # argocd_namespace: argocd
 # Default password:
 #   - https://argo-cd.readthedocs.io/en/stable/getting_started/#4-login-using-the-cli
@@ -236,7 +239,6 @@ kube_vip_enabled: false
 # kube_vip_cp_detect: false
 # kube_vip_leasename: plndr-cp-lock
 # kube_vip_enable_node_labeling: false
-# kube_vip_lb_fwdmethod: local
 
 # Node Feature Discovery
 node_feature_discovery_enabled: false

inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml

@@ -16,6 +16,9 @@ kube_token_dir: "{{ kube_config_dir }}/tokens"
 
 kube_api_anonymous_auth: true
 
+## Change this to use another Kubernetes version, e.g. a current beta release
+kube_version: 1.32.2
+
 # Where the binaries will be downloaded.
 # Note: ensure that you've enough disk space (about 1G)
 local_release_dir: "/tmp/releases"

inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml

@@ -1,4 +1,6 @@
 ---
+# cilium_version: "1.15.9"
+
 # Log-level
 # cilium_debug: false
 
@@ -253,10 +255,6 @@ cilium_l2announcements: false
 #   - name: "blue-pool"
 #     cidrs:
 #       - "10.0.10.0/24"
-#     ranges:
-#       - start: "20.0.20.100"
-#         stop: "20.0.20.200"
-#       - start: "1.2.3.4"
 
 # -- Configure BGP Instances (New bgpv2 API v1.16+)
 # cilium_bgp_cluster_configs:

inventory/sample/group_vars/k8s_cluster/k8s-net-custom-cni.yml

@@ -45,7 +45,7 @@
 # custom_cni_chart_repository_name: cilium
 # custom_cni_chart_repository_url: https://helm.cilium.io
 # custom_cni_chart_ref: cilium/cilium
-# custom_cni_chart_version: <chart version> (e.g.: 1.14.3)
+# custom_cni_chart_version: 1.14.3
 # custom_cni_chart_values:
 #   cluster:
 #     name: "cilium-demo"

inventory/sample/group_vars/k8s_cluster/k8s-net-kube-router.yml

@@ -1,5 +1,11 @@
 # See roles/network_plugin/kube-router/defaults/main.yml
 
+# Kube router version
+# Default to v2
+# kube_router_version: "2.0.0"
+# Uncomment to use v1 (Deprecated)
+# kube_router_version: "1.6.0"
+
 # Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP
 # kube_router_run_router: true
 

pipeline.Dockerfile

@@ -47,8 +47,8 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
     && pip install --no-compile --no-cache-dir pip -U \
     && pip install --no-compile --no-cache-dir -r tests/requirements.txt \
     && pip install --no-compile --no-cache-dir -r requirements.txt \
-    && curl -L https://dl.k8s.io/release/v1.32.4/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
-    && echo $(curl -L https://dl.k8s.io/release/v1.32.4/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
+    && curl -L https://dl.k8s.io/release/v1.32.3/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
+    && echo $(curl -L https://dl.k8s.io/release/v1.32.3/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
     && chmod a+x /usr/local/bin/kubectl \
     # Install Vagrant
     && curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \

playbooks/boilerplate.yml

@@ -30,17 +30,10 @@
         key: "{{ (group_names | intersect(item.value) | length > 0) | ternary(item.key, '_all') }}"
       loop: "{{ group_mappings | dict2items }}"
 
-- name: Check inventory settings
-  hosts: all
-  gather_facts: false
-  tags: always
-  roles:
-    - validate_inventory
-
 - name: Install bastion ssh config
   hosts: bastion[0]
   gather_facts: false
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: bastion-ssh-config, tags: ["localhost", "bastion"] }

playbooks/cluster.yml

@@ -11,15 +11,12 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: "container-engine", tags: "container-engine", when: deploy_container_engine }
     - { role: download, tags: download, when: "not skip_downloads" }
 
 - name: Install etcd
-  vars:
-    etcd_cluster_setup: true
-    etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}"
   import_playbook: install_etcd.yml
 
 - name: Install Kubernetes nodes
@@ -28,7 +25,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/node, tags: node }
 
 - name: Install the control plane
@@ -37,7 +34,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/control-plane, tags: master }
     - { role: kubernetes/client, tags: client }
     - { role: kubernetes-apps/cluster_roles, tags: cluster-roles }
@@ -48,16 +45,12 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/kubeadm, tags: kubeadm}
     - { role: kubernetes/node-label, tags: node-label }
     - { role: kubernetes/node-taint, tags: node-taint }
-    - role: kubernetes-apps/gateway_api
-      when: gateway_api_enabled
-      tags: gateway_api
-      delegate_to: "{{ groups['kube_control_plane'][0] }}"
-      run_once: true
     - { role: network_plugin, tags: network }
+    - { role: kubernetes-apps/kubelet-csr-approver, tags: kubelet-csr-approver }
 
 - name: Install Calico Route Reflector
   hosts: calico_rr
@@ -65,7 +58,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: network_plugin/calico/rr, tags: ['network', 'calico_rr'] }
 
 - name: Patch Kubernetes for Windows
@@ -74,7 +67,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"] }
 
 - name: Install Kubernetes apps
@@ -83,7 +76,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller }
     - { role: kubernetes-apps/network_plugin, tags: network }
     - { role: kubernetes-apps/policy_controller, tags: policy-controller }
@@ -97,5 +90,5 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }

playbooks/facts.yml

@@ -5,17 +5,19 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   gather_facts: false
   environment: "{{ proxy_disable_env }}"
+  vars:
+    # Need to disable pipelining for bootstrap-os as some systems have requiretty in sudoers set, which makes pipelining
+    # fail. bootstrap-os fixes this on these systems, so in later plays it can be enabled.
+    ansible_ssh_pipelining: false
   roles:
-    - { role: bootstrap_os, tags: bootstrap_os}
+    - { role: bootstrap-os, tags: bootstrap-os}
+    - { role: kubespray-defaults }
 
 - name: Gather facts
   hosts: k8s_cluster:etcd:calico_rr
   gather_facts: false
   tags: always
   tasks:
-    - name: Gather and compute network facts
-      import_role:
-        name: network_facts
     - name: Gather minimal facts
       setup:
         gather_subset: '!all'

playbooks/install_etcd.yml

@@ -2,7 +2,7 @@
 - name: Add worker nodes to the etcd play if needed
   hosts: kube_node
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
   tasks:
     - name: Check if nodes needs etcd client certs (depends on network_plugin)
       group_by:
@@ -20,7 +20,10 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - role: etcd
       tags: etcd
+      vars:
+        etcd_cluster_setup: true
+        etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}"
       when: etcd_deployment_type != "kubeadm"

playbooks/recover_control_plane.yml

@@ -6,7 +6,7 @@
   hosts: etcd[0]
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults}
+    - { role: kubespray-defaults}
     - role: recover_control_plane/etcd
       when: etcd_deployment_type != "kubeadm"
 
@@ -14,7 +14,7 @@
   hosts: kube_control_plane[0]
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults}
+    - { role: kubespray-defaults}
     - { role: recover_control_plane/control-plane }
 
 - name: Apply whole cluster install
@@ -24,5 +24,5 @@
   hosts: kube_control_plane
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults}
+    - { role: kubespray-defaults}
     - { role: recover_control_plane/post-recover }

playbooks/remove_node.yml

@@ -42,8 +42,8 @@
       service_facts:
       when: reset_nodes | default(True) | bool
   roles:
-    - { role: kubespray_defaults, when: reset_nodes | default(True) | bool }
-    - { role: remove_node/pre_remove, tags: pre-remove }
+    - { role: kubespray-defaults, when: reset_nodes | default(True) | bool }
+    - { role: remove-node/pre-remove, tags: pre-remove }
     - role: remove-node/remove-etcd-node
       when: "'etcd' in group_names"
     - { role: reset, tags: reset, when: reset_nodes | default(True) | bool }
@@ -54,5 +54,5 @@
   gather_facts: false
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults, when: reset_nodes | default(True) | bool }
+    - { role: kubespray-defaults, when: reset_nodes | default(True) | bool }
     - { role: remove-node/post-remove, tags: post-remove }

playbooks/reset.yml

@@ -30,6 +30,6 @@
 
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults}
+    - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_early: true }
     - { role: reset, tags: reset }

playbooks/scale.yml

@@ -5,11 +5,22 @@
 - name: Gather facts
   import_playbook: facts.yml
 
-- name: Install etcd
-  vars:
-    etcd_cluster_setup: false
-    etcd_events_cluster_setup: false
-  import_playbook: install_etcd.yml
+- name: Generate the etcd certificates beforehand
+  hosts: etcd:kube_control_plane
+  gather_facts: false
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: kubespray-defaults }
+    - role: etcd
+      tags: etcd
+      vars:
+        etcd_cluster_setup: false
+        etcd_events_cluster_setup: false
+      when:
+        - etcd_deployment_type != "kubeadm"
+        - kube_network_plugin in ["calico", "flannel", "canal", "cilium"] or cilium_deploy_additionally | default(false) | bool
+        - kube_network_plugin != "calico" or calico_datastore == "etcd"
 
 - name: Download images to ansible host cache via first kube_control_plane node
   hosts: kube_control_plane[0]
@@ -17,7 +28,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults, when: "not skip_downloads and download_run_once and not download_localhost" }
+    - { role: kubespray-defaults, when: "not skip_downloads and download_run_once and not download_localhost" }
     - { role: kubernetes/preinstall, tags: preinstall, when: "not skip_downloads and download_run_once and not download_localhost" }
     - { role: download, tags: download, when: "not skip_downloads and download_run_once and not download_localhost" }
 
@@ -27,7 +38,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: container-engine, tags: "container-engine", when: deploy_container_engine }
     - { role: download, tags: download, when: "not skip_downloads" }
@@ -46,7 +57,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/node, tags: node }
 
 - name: Upload control plane certs and retrieve encryption key
@@ -55,7 +66,7 @@
   gather_facts: false
   tags: kubeadm
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
   tasks:
     - name: Upload control plane certificates
       command: >-
@@ -77,7 +88,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/kubeadm, tags: kubeadm }
     - { role: kubernetes/node-label, tags: node-label }
     - { role: kubernetes/node-taint, tags: node-taint }
@@ -89,5 +100,5 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }

playbooks/upgrade_cluster.yml

@@ -11,7 +11,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults, when: "not skip_downloads and download_run_once and not download_localhost"}
+    - { role: kubespray-defaults, when: "not skip_downloads and download_run_once and not download_localhost"}
     - { role: kubernetes/preinstall, tags: preinstall, when: "not skip_downloads and download_run_once and not download_localhost" }
     - { role: download, tags: download, when: "not skip_downloads and download_run_once and not download_localhost" }
 
@@ -21,7 +21,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: download, tags: download, when: "not skip_downloads" }
 
@@ -32,13 +32,10 @@
   environment: "{{ proxy_disable_env }}"
   serial: "{{ serial | default('20%') }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: container-engine, tags: "container-engine", when: deploy_container_engine }
 
 - name: Install etcd
-  vars:
-    etcd_cluster_setup: true
-    etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}"
   import_playbook: install_etcd.yml
 
 - name: Handle upgrades to control plane components first to maintain backwards compat.
@@ -48,7 +45,7 @@
   environment: "{{ proxy_disable_env }}"
   serial: 1
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
     - { role: upgrade/system-upgrade, tags: system-upgrade }
     - { role: download, tags: download, when: "system_upgrade and system_upgrade_reboot != 'never' and not skip_downloads" }
@@ -70,7 +67,7 @@
   serial: "{{ serial | default('20%') }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller }
     - { role: network_plugin, tags: network }
     - { role: kubernetes-apps/network_plugin, tags: network }
@@ -83,7 +80,7 @@
   environment: "{{ proxy_disable_env }}"
   serial: "{{ serial | default('20%') }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
     - { role: upgrade/system-upgrade, tags: system-upgrade }
     - { role: download, tags: download, when: "system_upgrade and system_upgrade_reboot != 'never' and not skip_downloads" }
@@ -100,7 +97,7 @@
   any_errors_fatal: true
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"] }
 
 - name: Install Calico Route Reflector
@@ -109,7 +106,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: network_plugin/calico/rr, tags: network }
 
 - name: Install Kubernetes apps
@@ -118,7 +115,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes-apps/ingress_controller, tags: ingress-controller }
     - { role: kubernetes-apps/external_provisioner, tags: external-provisioner }
     - { role: kubernetes-apps, tags: apps }
@@ -129,5 +126,5 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray_defaults }
+    - { role: kubespray-defaults }
     - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }

requirements.txt

@@ -1,6 +1,6 @@
 ansible==9.13.0
 # Needed for community.crypto module
-cryptography==44.0.3
+cryptography==44.0.2
 # Needed for jinja2 json_query templating
 jmespath==1.0.1
 # Needed for ansible.utils.ipaddr

roles/bootstrap-os/defaults/main.yml

@@ -9,9 +9,6 @@ rh_subscription_check_timeout: 180
 # Disable locksmithd or leave it in its current state
 coreos_locksmithd_disable: false
 
-# Install epel repo on Centos/RHEL
-epel_enabled: false
-
 ## Oracle Linux specific variables
 # Install public repo on Oracle Linux
 use_oracle_public_repo: true

roles/bootstrap-os/molecule/default/converge.yml

@@ -4,4 +4,4 @@
   gather_facts: false
   become: true
   roles:
-    - role: bootstrap_os
+    - role: bootstrap-os

roles/bootstrap-os/tasks/amzn.yml

@@ -0,0 +1,27 @@
+---
+- name: Enable selinux-ng repo for Amazon Linux for container-selinux
+  command: amazon-linux-extras enable selinux-ng
+
+- name: Enable EPEL repo for Amazon Linux
+  yum_repository:
+    name: epel
+    file: epel
+    description: Extra Packages for Enterprise Linux 7 - $basearch
+    baseurl: http://download.fedoraproject.org/pub/epel/7/$basearch
+    gpgcheck: true
+    gpgkey: http://download.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7
+    skip_if_unavailable: true
+    enabled: true
+    repo_gpgcheck: false
+  when: epel_enabled
+
+# iproute is required for networking related facts gathering
+# See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_vars_facts.html#package-requirements-for-fact-gathering
+# Note: It is not recommended way, but since the tasks execution order, put it here is the simplest way so far. We can move it to a proper place later.
+# TODO: move this to roles/kubernetes/preinstall/vars/main.yml -> pkgs variables
+# Currently not possible because the collect the network facts before that step, needs reordering of the exec flow.
+- name: Ensure iproute is installed
+  package:
+    name: iproute
+    state: present
+  become: true

roles/bootstrap-os/tasks/centos.yml

@@ -108,3 +108,22 @@
   when:
     - fastestmirror.stat.exists
     - not centos_fastestmirror_enabled
+
+# libselinux-python is required on SELinux enabled hosts
+# See https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#managed-node-requirements
+- name: Install libselinux python package
+  package:
+    name: "{{ ((ansible_distribution_major_version | int) < 8) | ternary('libselinux-python', 'python3-libselinux') }}"
+    state: present
+  become: true
+
+# iproute is required for networking related facts gathering
+# See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_vars_facts.html#package-requirements-for-fact-gathering
+# Note: It is not recommended way, but since the tasks execution order, put it here is the simplest way so far. We can move it to a proper place later.
+# TODO: move this to roles/kubernetes/preinstall/vars/main.yml -> pkgs variables
+# Currently not possible because the collect the network facts before that step, needs reordering of the exec flow.
+- name: Ensure iproute is installed
+  package:
+    name: iproute
+    state: present
+  become: true

roles/bootstrap-os/tasks/clear-linux-os.yml

@@ -0,0 +1,27 @@
+---
+# ClearLinux ships with Python installed
+
+- name: Install basic package to run containers
+  package:
+    name: containers-basic
+    state: present
+
+- name: Make sure docker service is enabled
+  systemd_service:
+    name: docker
+    masked: false
+    enabled: true
+    daemon_reload: true
+    state: started
+  become: true
+
+# iproute2 is required for networking related facts gathering
+# See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_vars_facts.html#package-requirements-for-fact-gathering
+# Note: It is not recommended way, but since the tasks execution order, put it here is the simplest way so far. We can move it to a proper place later.
+# TODO: move this to roles/kubernetes/preinstall/vars/main.yml -> pkgs variables
+# Currently not possible because the collect the network facts before that step, needs reordering of the exec flow.
+- name: Ensure iproute2 is installed
+  package:
+    name: iproute2
+    state: present
+  become: true

roles/bootstrap-os/tasks/debian.yml

@@ -62,3 +62,14 @@
     - '"changed its" in bootstrap_update_apt_result.stdout'
     - '"value from" in bootstrap_update_apt_result.stdout'
   ignore_errors: true
+
+# iproute2 is required for networking related facts gathering
+# See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_vars_facts.html#package-requirements-for-fact-gathering
+# Note: It is not recommended way, but since the tasks execution order, put it here is the simplest way so far. We can move it to a proper place later.
+# TODO: move this to roles/kubernetes/preinstall/vars/main.yml -> pkgs variables
+# Currently not possible because the collect the network facts before that step, needs reordering of the exec flow.
+- name: Ensure iproute2 is installed
+  package:
+    name: iproute2
+    state: present
+  become: true

roles/bootstrap-os/tasks/fedora.yml

@@ -28,3 +28,14 @@
   become: true
   when:
     - need_bootstrap.rc != 0
+
+# iproute is required for networking related facts gathering
+# See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_vars_facts.html#package-requirements-for-fact-gathering
+# Note: It is not recommended way, but since the tasks execution order, put it here is the simplest way so far. We can move it to a proper place later.
+# TODO: move this to roles/kubernetes/preinstall/vars/main.yml -> pkgs variables
+# Currently not possible because the collect the network facts before that step, needs reordering of the exec flow.
+- name: Ensure iproute is installed
+  package:
+    name: iproute
+    state: present
+  become: true

roles/bootstrap-os/tasks/flatcar.yml

@@ -23,7 +23,7 @@
 
 - name: Make interpreter discovery works on Flatcar
   set_fact:
-    ansible_interpreter_python_fallback: "{{ (ansible_interpreter_python_fallback | default([])) + ['/opt/bin/python'] }}"
+    ansible_interpreter_python_fallback: "{{ (ansible_interpreter_python_fallback | default([])) + [ '/opt/bin/python' ] }}"
 
 - name: Disable auto-upgrade
   systemd_service:

roles/bootstrap-os/tasks/main.yml

@@ -1,10 +1,57 @@
 ---
-- name: Warn for usage of deprecated role
-  fail:
-    msg: bootstrap-os is deprecated, switch to bootstrap_os
-  ignore_errors: true # noqa ignore-errors
-  run_once: true
+- name: Fetch /etc/os-release
+  raw: cat /etc/os-release
+  register: os_release
+  changed_when: false
+  # This command should always run, even in check mode
+  check_mode: false
 
-- name: Compat for direct role import
-  import_role:
-    name: bootstrap_os
+- name: Include distro specifics vars and tasks
+  vars:
+    os_release_dict: "{{ os_release.stdout_lines | select('regex', '^.+=.*$') | map('regex_replace', '\"', '') |
+                         map('split', '=') | community.general.dict }}"
+  block:
+  - name: Include vars
+    include_vars: "{{ item }}"
+    tags:
+    - facts
+    with_first_found:
+    - &search
+      files:
+      - "{{ os_release_dict['ID'] }}-{{ os_release_dict['VARIANT_ID'] }}.yml"
+      - "{{ os_release_dict['ID'] }}.yml"
+      paths:
+      - vars/
+      skip: true
+  - name: Include tasks
+    include_tasks: "{{ included_tasks_file }}"
+    with_first_found:
+    - <<: *search
+      paths: []
+    loop_control:
+      loop_var: included_tasks_file
+
+
+- name: Create remote_tmp for it is used by another module
+  file:
+    path: "{{ ansible_remote_tmp | default('~/.ansible/tmp') }}"
+    state: directory
+    mode: "0700"
+
+- name: Gather facts
+  setup:
+    gather_subset: '!all'
+    filter: ansible_*
+
+- name: Assign inventory name to unconfigured hostnames (non-CoreOS, non-Flatcar, Suse and ClearLinux, non-Fedora)
+  hostname:
+    name: "{{ inventory_hostname }}"
+  when: override_system_hostname
+
+- name: Ensure bash_completion.d folder exists
+  file:
+    name: /etc/bash_completion.d/
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"

roles/bootstrap-os/tasks/opensuse.yml

@@ -83,3 +83,15 @@
       - apparmor-parser
     state: present
   become: true
+
+# iproute2 is required for networking related facts gathering
+# See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_vars_facts.html#package-requirements-for-fact-gathering
+# Note: It is not recommended way, but since the tasks execution order, put it here is the simplest way so far. We can move it to a proper place later.
+# TODO: move this to roles/kubernetes/preinstall/vars/main.yml -> pkgs variables
+# Currently not possible because the collect the network facts before that step, needs reordering of the exec flow.
+- name: Ensure iproute2 is installed
+  community.general.zypper:
+    name: iproute2
+    state: present
+    update_cache: true
+  become: true

roles/bootstrap-os/tasks/rhel.yml

@@ -93,3 +93,22 @@
   when:
     - fastestmirror.stat.exists
     - not centos_fastestmirror_enabled
+
+# libselinux-python is required on SELinux enabled hosts
+# See https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#managed-node-requirements
+- name: Install libselinux python package
+  package:
+    name: "{{ ((ansible_distribution_major_version | int) < 8) | ternary('libselinux-python', 'python3-libselinux') }}"
+    state: present
+  become: true
+
+# iproute is required for networking related facts gathering
+# See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_vars_facts.html#package-requirements-for-fact-gathering
+# Note: It is not recommended way, but since the tasks execution order, put it here is the simplest way so far. We can move it to a proper place later.
+# TODO: move this to roles/kubernetes/preinstall/vars/main.yml -> pkgs variables
+# Currently not possible because the collect the network facts before that step, needs reordering of the exec flow.
+- name: Ensure iproute is installed
+  package:
+    name: iproute
+    state: present
+  become: true

roles/bootstrap_os/meta/main.yml

@@ -1,3 +0,0 @@
----
-dependencies:
-  - role: kubespray_defaults

roles/bootstrap_os/tasks/amzn.yml

@@ -1,16 +0,0 @@
----
-- name: Enable selinux-ng repo for Amazon Linux for container-selinux
-  command: amazon-linux-extras enable selinux-ng
-
-- name: Enable EPEL repo for Amazon Linux
-  yum_repository:
-    name: epel
-    file: epel
-    description: Extra Packages for Enterprise Linux 7 - $basearch
-    baseurl: http://download.fedoraproject.org/pub/epel/7/$basearch
-    gpgcheck: true
-    gpgkey: http://download.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7
-    skip_if_unavailable: true
-    enabled: true
-    repo_gpgcheck: false
-  when: epel_enabled

roles/bootstrap_os/tasks/clear-linux-os.yml

@@ -1,16 +0,0 @@
----
-# ClearLinux ships with Python installed
-
-- name: Install basic package to run containers
-  package:
-    name: containers-basic
-    state: present
-
-- name: Make sure docker service is enabled
-  systemd_service:
-    name: docker
-    masked: false
-    enabled: true
-    daemon_reload: true
-    state: started
-  become: true

roles/bootstrap_os/tasks/main.yml

@@ -1,62 +0,0 @@
----
-- name: Fetch /etc/os-release
-  raw: cat /etc/os-release
-  register: os_release
-  changed_when: false
-  # This command should always run, even in check mode
-  check_mode: false
-
-- name: Include distro specifics vars and tasks
-  vars:
-    os_release_dict: "{{ os_release.stdout_lines | select('regex', '^.+=.*$') | map('regex_replace', '\"', '') |
-                         map('split', '=') | community.general.dict }}"
-  block:
-  - name: Include vars
-    include_vars: "{{ item }}"
-    tags:
-    - facts
-    with_first_found:
-    - &search
-      files:
-      - "{{ os_release_dict['ID'] }}-{{ os_release_dict['VARIANT_ID'] }}.yml"
-      - "{{ os_release_dict['ID'] }}.yml"
-      paths:
-      - vars/
-      skip: true
-  - name: Include tasks
-    include_tasks: "{{ included_tasks_file }}"
-    with_first_found:
-    - <<: *search
-      paths: []
-    loop_control:
-      loop_var: included_tasks_file
-
-- name: Install system packages
-  import_role:
-    name: system_packages
-  tags:
-  - system-packages
-
-- name: Create remote_tmp for it is used by another module
-  file:
-    path: "{{ ansible_remote_tmp | default('~/.ansible/tmp') }}"
-    state: directory
-    mode: "0700"
-
-- name: Gather facts
-  setup:
-    gather_subset: '!all'
-    filter: ansible_*
-
-- name: Assign inventory name to unconfigured hostnames (non-CoreOS, non-Flatcar, Suse and ClearLinux, non-Fedora)
-  hostname:
-    name: "{{ inventory_hostname }}"
-  when: override_system_hostname
-
-- name: Ensure bash_completion.d folder exists
-  file:
-    name: /etc/bash_completion.d/
-    state: directory
-    owner: root
-    group: root
-    mode: "0755"

roles/container-engine/containerd-common/defaults/main.yml

@@ -3,3 +3,15 @@
 # manager controlled installs to direct download ones.
 containerd_package: 'containerd.io'
 yum_repo_dir: /etc/yum.repos.d
+
+# Keep minimal repo information around for cleanup
+containerd_repo_info:
+  repos:
+
+# Ubuntu docker-ce repo
+containerd_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu"
+containerd_ubuntu_repo_component: "stable"
+
+# Debian docker-ce repo
+containerd_debian_repo_base_url: "https://download.docker.com/linux/debian"
+containerd_debian_repo_component: "stable"

roles/container-engine/containerd/defaults/main.yml

@@ -17,8 +17,8 @@ containerd_runc_runtime:
   root: ""
   base_runtime_spec: cri-base.json
   options:
-    SystemdCgroup: "{{ containerd_use_systemd_cgroup | ternary('true', 'false') }}"
-    BinaryName: "{{ bin_dir }}/runc"
+    systemdCgroup: "{{ containerd_use_systemd_cgroup | ternary('true', 'false') }}"
+    binaryName: "{{ bin_dir }}/runc"
 
 containerd_additional_runtimes: []
 # Example for Kata Containers as additional runtime:

roles/container-engine/containerd/molecule/default/converge.yml

@@ -5,5 +5,5 @@
   vars:
     container_manager: containerd
   roles:
-    - role: kubespray_defaults
+    - role: kubespray-defaults
     - role: container-engine/containerd

roles/container-engine/containerd/molecule/default/prepare.yml

@@ -6,9 +6,8 @@
   vars:
     ignore_assert_errors: true
   roles:
-    - role: kubespray_defaults
-    - role: bootstrap_os
-    - role: network_facts
+    - role: kubespray-defaults
+    - role: bootstrap-os
     - role: kubernetes/preinstall
     - role: adduser
       user: "{{ addusers.kube }}"
@@ -26,5 +25,5 @@
     ignore_assert_errors: true
     kube_network_plugin: cni
   roles:
-    - role: kubespray_defaults
+    - role: kubespray-defaults
     - role: network_plugin/cni

roles/container-engine/containerd/tasks/main.yml

@@ -1,4 +1,31 @@
 ---
+- name: Fail containerd setup if distribution is not supported
+  fail:
+    msg: "{{ ansible_distribution }} is not supported by containerd."
+  when:
+    - not (allow_unsupported_distribution_setup | default(false)) and (ansible_distribution not in containerd_supported_distributions)
+
+- name: Containerd | Remove any package manager controlled containerd package
+  package:
+    name: "{{ containerd_package }}"
+    state: absent
+  when:
+    - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar"))
+
+- name: Containerd | Remove containerd repository
+  file:
+    path: "{{ yum_repo_dir }}/containerd.repo"
+    state: absent
+  when:
+    - ansible_os_family in ['RedHat']
+
+- name: Containerd | Remove containerd repository
+  apt_repository:
+    repo: "{{ item }}"
+    state: absent
+  with_items: "{{ containerd_repo_info.repos }}"
+  when: ansible_pkg_mgr == 'apt'
+
 - name: Containerd | Download containerd
   include_tasks: "../../../download/tasks/download_file.yml"
   vars:
@@ -14,6 +41,21 @@
       - --strip-components=1
   notify: Restart containerd
 
+- name: Containerd | Remove orphaned binary
+  file:
+    path: "/usr/bin/{{ item }}"
+    state: absent
+  when:
+    - containerd_bin_dir != "/usr/bin"
+    - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar"))
+  ignore_errors: true  # noqa ignore-errors
+  with_items:
+    - containerd
+    - containerd-shim
+    - containerd-shim-runc-v1
+    - containerd-shim-runc-v2
+    - ctr
+
 - name: Containerd | Generate systemd service for containerd
   template:
     src: containerd.service.j2

roles/container-engine/containerd/tasks/reset.yml

@@ -1,4 +1,22 @@
 ---
+- name: Containerd | Remove containerd repository for RedHat os family
+  file:
+    path: "{{ yum_repo_dir }}/containerd.repo"
+    state: absent
+  when:
+    - ansible_os_family in ['RedHat']
+  tags:
+    - reset_containerd
+
+- name: Containerd | Remove containerd repository for Debian os family
+  apt_repository:
+    repo: "{{ item }}"
+    state: absent
+  with_items: "{{ containerd_repo_info.repos }}"
+  when: ansible_pkg_mgr == 'apt'
+  tags:
+    - reset_containerd
+
 - name: Containerd | Stop containerd service
   service:
     name: containerd

roles/container-engine/containerd/templates/config.toml.j2

@@ -76,8 +76,10 @@ oom_score = {{ containerd_oom_score }}
   [plugins."io.containerd.cri.v1.images".registry]
     config_path = "{{ containerd_cfg_dir }}/certs.d"
 
+{% if nri_enabled %}
   [plugins."io.containerd.nri.v1.nri"]
-    disable = {{ 'false' if nri_enabled else 'true' }}
+    disable = false
+{% endif %}
 
 {% if containerd_tracing_enabled %}
   [plugins."io.containerd.tracing.processor.v1.otlp"]

roles/container-engine/containerd/vars/debian.yml

@@ -0,0 +1,7 @@
+---
+containerd_repo_info:
+  repos:
+    - >
+      deb {{ containerd_debian_repo_base_url }}
+      {{ ansible_distribution_release | lower }}
+      {{ containerd_debian_repo_component }}

roles/container-engine/containerd/vars/ubuntu.yml

@@ -0,0 +1,7 @@
+---
+containerd_repo_info:
+  repos:
+    - >
+      deb {{ containerd_ubuntu_repo_base_url }}
+      {{ ansible_distribution_release | lower }}
+      {{ containerd_ubuntu_repo_component }}

roles/container-engine/cri-dockerd/molecule/default/converge.yml

@@ -5,5 +5,5 @@
   vars:
     container_manager: docker
   roles:
-    - role: kubespray_defaults
+    - role: kubespray-defaults
     - role: container-engine/cri-dockerd

roles/container-engine/cri-dockerd/molecule/default/prepare.yml

@@ -3,8 +3,8 @@
   hosts: all
   become: true
   roles:
-    - role: kubespray_defaults
-    - role: bootstrap_os
+    - role: kubespray-defaults
+    - role: bootstrap-os
     - role: adduser
       user: "{{ addusers.kube }}"
   tasks:
@@ -20,7 +20,7 @@
     container_manager: containerd
     kube_network_plugin: cni
   roles:
-    - role: kubespray_defaults
+    - role: kubespray-defaults
     - role: network_plugin/cni
   tasks:
     - name: Copy test container files

roles/container-engine/cri-o/molecule/default/converge.yml

@@ -5,5 +5,5 @@
   vars:
     container_manager: crio
   roles:
-    - role: kubespray_defaults
+    - role: kubespray-defaults
     - role: container-engine/cri-o

roles/container-engine/cri-o/molecule/default/prepare.yml

@@ -6,9 +6,8 @@
   vars:
     ignore_assert_errors: true
   roles:
-    - role: kubespray_defaults
-    - role: bootstrap_os
-    - role: network_facts
+    - role: kubespray-defaults
+    - role: bootstrap-os
     - role: kubernetes/preinstall
     - role: adduser
       user: "{{ addusers.kube }}"
@@ -26,7 +25,7 @@
     ignore_assert_errors: true
     kube_network_plugin: cni
   roles:
-    - role: kubespray_defaults
+    - role: kubespray-defaults
     - role: network_plugin/cni
   tasks:
     - name: Copy test container files

roles/container-engine/cri-o/tasks/main.yaml

@@ -180,7 +180,7 @@
     dest: /etc/containers/storage.conf
     section: storage.options.overlay
     option: mountopt
-    value: '{{ ''"nodev"'' if ansible_kernel is version(("4.18" if ansible_os_family == "RedHat" else "4.19"), "<") else ''"nodev,metacopy=on"'' }}'
+    value: '{{ ''"nodev"'' if ansible_kernel is version_compare(("4.18" if ansible_os_family == "RedHat" else "4.19"), "<") else ''"nodev,metacopy=on"'' }}'
     mode: "0644"
 
 - name: Cri-o | create directory registries configs

roles/container-engine/docker/tasks/main.yml

@@ -50,7 +50,7 @@
   apt_key:
     id: "{{ item }}"
     url: "{{ docker_repo_key_info.url }}"
-    keyring: "{{ docker_repo_key_keyring | default(omit) }}"
+    keyring: "{{ docker_repo_key_keyring|default(omit) }}"
     state: present
   register: keyserver_task_result
   until: keyserver_task_result is succeeded

roles/container-engine/gvisor/molecule/default/converge.yml

@@ -6,6 +6,6 @@
     gvisor_enabled: true
     container_manager: containerd
   roles:
-    - role: kubespray_defaults
+    - role: kubespray-defaults
     - role: container-engine/containerd
     - role: container-engine/gvisor

roles/container-engine/gvisor/molecule/default/prepare.yml

@@ -3,8 +3,8 @@
   hosts: all
   become: true
   roles:
-    - role: kubespray_defaults
-    - role: bootstrap_os
+    - role: kubespray-defaults
+    - role: bootstrap-os
     - role: adduser
       user: "{{ addusers.kube }}"
   tasks:
@@ -20,7 +20,7 @@
     container_manager: containerd
     kube_network_plugin: cni
   roles:
-    - role: kubespray_defaults
+    - role: kubespray-defaults
     - role: network_plugin/cni
     - role: container-engine/crictl
   tasks:

roles/container-engine/kata-containers/molecule/default/converge.yml

@@ -6,6 +6,6 @@
     kata_containers_enabled: true
     container_manager: containerd
   roles:
-    - role: kubespray_defaults
+    - role: kubespray-defaults
     - role: container-engine/containerd
     - role: container-engine/kata-containers

roles/container-engine/kata-containers/molecule/default/prepare.yml

@@ -3,8 +3,8 @@
   hosts: all
   become: true
   roles:
-    - role: kubespray_defaults
-    - role: bootstrap_os
+    - role: kubespray-defaults
+    - role: bootstrap-os
     - role: adduser
       user: "{{ addusers.kube }}"
   tasks:
@@ -20,7 +20,7 @@
     container_manager: containerd
     kube_network_plugin: cni
   roles:
-    - role: kubespray_defaults
+    - role: kubespray-defaults
     - role: network_plugin/cni
     - role: container-engine/crictl
   tasks:

roles/container-engine/validate-container-engine/tasks/main.yml

@@ -84,7 +84,7 @@
   block:
     - name: Drain node
       include_role:
-        name: remove_node/pre_remove
+        name: remove-node/pre-remove
         apply:
           tags:
             - pre-remove
@@ -111,7 +111,7 @@
   block:
     - name: Drain node
       include_role:
-        name: remove_node/pre_remove
+        name: remove-node/pre-remove
         apply:
           tags:
             - pre-remove
@@ -137,7 +137,7 @@
   block:
     - name: Drain node
       include_role:
-        name: remove_node/pre_remove
+        name: remove-node/pre-remove
         apply:
           tags:
             - pre-remove

roles/container-engine/youki/molecule/default/converge.yml

@@ -6,6 +6,6 @@
     youki_enabled: true
     container_manager: crio
   roles:
-    - role: kubespray_defaults
+    - role: kubespray-defaults
     - role: container-engine/cri-o
     - role: container-engine/youki

roles/container-engine/youki/molecule/default/prepare.yml

@@ -3,8 +3,8 @@
   hosts: all
   become: true
   roles:
-    - role: kubespray_defaults
-    - role: bootstrap_os
+    - role: kubespray-defaults
+    - role: bootstrap-os
     - role: adduser
       user: "{{ addusers.kube }}"
   tasks:
@@ -20,7 +20,7 @@
     container_manager: crio
     kube_network_plugin: cni
   roles:
-    - role: kubespray_defaults
+    - role: kubespray-defaults
     - role: network_plugin/cni
     - role: container-engine/crictl
   tasks:

roles/etcd/tasks/check_certs.yml

@@ -84,7 +84,7 @@
         {% if not loop.last %}{{ ',' }}{% endif %}
       {% endfor %}]
   when:
-    - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally
+    - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally | default(false) | bool
     - kube_network_plugin != "calico" or calico_datastore == "etcd"
     - force_etcd_cert_refresh or not item in etcdcert_master.files | map(attribute='path') | list
 

roles/etcd/tasks/gen_certs_script.yml

@@ -54,7 +54,7 @@
   run_once: true
   delegate_to: "{{ groups['etcd'][0] }}"
   when:
-    - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally
+    - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally | default(false) | bool
     - kube_network_plugin != "calico" or calico_datastore == "etcd"
     - gen_certs | default(false)
   notify: Set etcd_secret_changed
@@ -111,7 +111,7 @@
   when:
     - ('etcd' in group_names)
     - inventory_hostname != groups['etcd'][0]
-    - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally
+    - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally | default(false) | bool
     - kube_network_plugin != "calico" or calico_datastore == "etcd"
   notify: Set etcd_secret_changed
 
@@ -126,7 +126,7 @@
   when:
     - ('etcd' in group_names)
     - inventory_hostname != groups['etcd'][0]
-    - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally
+    - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally | default(false) | bool
     - kube_network_plugin != "calico" or calico_datastore == "etcd"
   loop_control:
     label: "{{ item.item }}"
@@ -140,7 +140,7 @@
 - name: Gen_certs | Generate etcd certs on nodes if needed
   include_tasks: gen_nodes_certs_script.yml
   when:
-    - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally
+    - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally | default(false) | bool
     - kube_network_plugin != "calico" or calico_datastore == "etcd"
     - ('k8s_cluster' in group_names) and
         sync_certs | default(false) and inventory_hostname not in groups['etcd']
@@ -153,25 +153,3 @@
     owner: "{{ etcd_owner }}"
     mode: "{{ etcd_cert_dir_mode }}"
     recurse: true
-
-# This is a hack around the fact kubeadm expect the same certs path on all kube_control_plane
-# TODO: fix certs generation to have the same file everywhere
-# OR work with kubeadm on node-specific config
-- name: Gen_certs | Pretend all control plane have all certs (with symlinks)
-  file:
-    state: link
-    src: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}{{ item[0] }}.pem"
-    dest: "{{ etcd_cert_dir }}/node-{{ item[1] }}{{ item[0] }}.pem"
-    mode: "0640"
-  loop: "{{ suffixes | product(groups['kube_control_plane']) }}"
-  vars:
-    suffixes:
-      - ''
-      - '-key'
-  when:
-    - ('kube_control_plane' in group_names)
-    - item[1] != inventory_hostname
-  register: symlink_created
-  failed_when:
-    - symlink_created is failed
-    - ('refusing to convert from file to symlink' not in symlink_created.msg)

roles/etcd/tasks/main.yml

@@ -23,7 +23,7 @@
 - name: Trust etcd CA on nodes if needed
   include_tasks: upd_ca_trust.yml
   when:
-    - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally
+    - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally | default(false) | bool
     - kube_network_plugin != "calico" or calico_datastore == "etcd"
     - ('k8s_cluster' in group_names)
   tags:
@@ -35,7 +35,7 @@
   changed_when: false
   check_mode: false
   when:
-    - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally
+    - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally | default(false) | bool
     - kube_network_plugin != "calico" or calico_datastore == "etcd"
     - ('k8s_cluster' in group_names)
   tags:
@@ -47,7 +47,7 @@
   set_fact:
     etcd_client_cert_serial: "{{ etcd_client_cert_serial_result.stdout.split('=')[1] }}"
   when:
-    - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally
+    - kube_network_plugin in ["calico", "flannel", "cilium"] or cilium_deploy_additionally | default(false) | bool
     - kube_network_plugin != "calico" or calico_datastore == "etcd"
     - ('k8s_cluster' in group_names)
   tags:

roles/etcd/vars/main.yml

@@ -6,5 +6,5 @@ cert_files:
   - "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
   - "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
   node:
-  - "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
-  - "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
+  - "{{ etcd_cert_dir}}/node-{{ inventory_hostname }}.pem"
+  - "{{ etcd_cert_dir}}/node-{{ inventory_hostname }}-key.pem"

roles/kubernetes-apps/ansible/defaults/main.yml

@@ -65,7 +65,6 @@ dns_autoscaler_cpu_requests: 20m
 dns_autoscaler_memory_requests: 10Mi
 dns_autoscaler_deployment_nodeselector: "kubernetes.io/os: linux"
 # dns_autoscaler_extra_tolerations: [{effect: NoSchedule, operator: "Exists"}]
-dns_autoscaler_affinity: {}
 
 # etcd metrics
 # etcd_metrics_service_labels:

roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2

@@ -48,7 +48,21 @@ spec:
         {{ dns_autoscaler_extra_tolerations | list | to_nice_yaml(indent=2) | indent(8) }}
 {% endif %}
       affinity:
-        {{ dns_autoscaler_affinity | to_nice_yaml(indent=2) | indent(8) }}
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - topologyKey: "kubernetes.io/hostname"
+            labelSelector:
+              matchLabels:
+                k8s-app: dns-autoscaler{{ coredns_ordinal_suffix }}
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: In
+                values:
+                - ""
       containers:
       - name: autoscaler
         image: "{{ dnsautoscaler_image_repo }}:{{ dnsautoscaler_image_tag }}"

roles/kubernetes-apps/external_cloud_controller/huaweicloud/defaults/main.yml

@@ -2,12 +2,12 @@
 # The external cloud controller will need credentials to access
 # openstack apis. Per default these values will be
 # read from the environment.
-external_huaweicloud_auth_url: "{{ lookup('env', 'OS_AUTH_URL') }}"
-external_huaweicloud_access_key: "{{ lookup('env', 'OS_ACCESS_KEY') }}"
-external_huaweicloud_secret_key: "{{ lookup('env', 'OS_SECRET_KEY') }}"
-external_huaweicloud_region: "{{ lookup('env', 'OS_REGION_NAME') }}"
-external_huaweicloud_project_id: "{{ lookup('env', 'OS_TENANT_ID') | default(lookup('env', 'OS_PROJECT_ID'), true) }}"
-external_huaweicloud_cloud: "{{ lookup('env', 'OS_CLOUD') }}"
+external_huaweicloud_auth_url: "{{ lookup('env','OS_AUTH_URL')  }}"
+external_huaweicloud_access_key: "{{ lookup('env','OS_ACCESS_KEY')  }}"
+external_huaweicloud_secret_key: "{{ lookup('env','OS_SECRET_KEY')  }}"
+external_huaweicloud_region: "{{ lookup('env','OS_REGION_NAME')  }}"
+external_huaweicloud_project_id: "{{ lookup('env','OS_TENANT_ID')| default(lookup('env','OS_PROJECT_ID'),true) }}"
+external_huaweicloud_cloud: "{{ lookup('env','OS_CLOUD') }}"
 
 ## A dictionary of extra arguments to add to the huawei cloud controller manager deployment
 ## Format:

roles/kubernetes-apps/gateway_api/defaults/main.yml

@@ -1,9 +1,4 @@
 ---
 gateway_api_enabled: false
-gateway_api_version: 1.2.1
-
-# `gateway_api_channel` default is "standard".
-# "standard" release channel includes all resources that have graduated to GA or beta, including GatewayClass, Gateway, HTTPRoute, and ReferenceGrant.
-# "experimental" for some experimental resources and fields. Note that future releases of the API could include breaking changes to experimental resources and fields. For example, any experimental resource or field could be removed in a future release.
-# https://gateway-api.sigs.k8s.io/guides/#install-experimental-channel
-gateway_api_channel: "standard"
+gateway_api_version: 1.1.0
+gateway_api_experimental_channel: false

roles/kubernetes-apps/gateway_api/tasks/main.yml

@@ -1,9 +1,4 @@
 ---
-- name: Gateway API | Download YAML
-  include_tasks: "../../../download/tasks/download_file.yml"
-  vars:
-    download: "{{ download_defaults | combine(downloads.gateway_api_crds) }}"
-
 - name: Gateway API | Create addon dir
   file:
     path: "{{ kube_config_dir }}/addons/gateway_api"
@@ -14,12 +9,17 @@
   when:
     - inventory_hostname == groups['kube_control_plane'][0]
 
-- name: Gateway API | Copy YAML from download dir
-  copy:
-    src: "{{ local_release_dir }}/gateway-api-{{ gateway_api_channel }}-install.yaml"
+- name: Gateway API | Set channel
+  set_fact:
+    gateway_api_channel: "{{ 'experimental' if gateway_api_experimental_channel else 'standard' }}"
+  when:
+    - "inventory_hostname == groups['kube_control_plane'][0]"
+
+- name: Gateway API | Copy Gateway API manifests to remote
+  template:
+    src: "{{ gateway_api_channel }}-install.yaml.j2"
     dest: "{{ kube_config_dir }}/addons/gateway_api/{{ gateway_api_channel }}-install.yaml"
     mode: "0644"
-    remote_src: true
   when:
     - "inventory_hostname == groups['kube_control_plane'][0]"