wait for control plane node to become ready after joining (#12924)

When joining a control plane node and "upgrading" the cluster setup (for
example, to update etcd addresses after adding a new etcd) in the same
playbook run, the node can take a bit of time to become ready after
joining.
This triggers a kubeadm preflight check (ControlPlaneNodesReady) in
kubeadm upgrade, which is run directly after the join tasks.

Add a configurable wait for the control plane node to become Ready to
fix this race condition.

Co-authored-by: Max Gautier <mg@max.gautier.name>

.ansible-lint

@@ -1,5 +1,4 @@
 ---
-parseable: true
 skip_list:
   # see https://docs.ansible.com/ansible-lint/rules/default_rules.html for a list of all default rules
 
@@ -12,10 +11,12 @@ skip_list:
   # (Disabled in June 2021)
   - 'role-name'
 
-  # [var-naming] "defaults/main.yml" File defines variable 'apiVersion' that violates variable naming standards
+  # [var-naming]
   # In Kubespray we use variables that use camelCase to match their k8s counterparts
   # (Disabled in June 2021)
-  - 'var-naming'
+  - 'var-naming[pattern]'
+  # Variables names from within roles in kubespray don't need role name as a prefix
+  - 'var-naming[no-role-prefix]'
 
   # [fqcn-builtins]
   # Roles in kubespray don't need fully qualified collection names

.github/ISSUE_TEMPLATE/bug-report.yaml

@@ -108,7 +108,6 @@ body:
         - meta
         - multus
         - ovn4nfv
-        - weave
     validations:
       required: true
 

.github/workflows/auto-label-os.yml

@@ -13,7 +13,7 @@ jobs:
       issues: write
 
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
 
       - name: Parse issue form
         uses: stefanbuck/github-issue-parser@2ea9b35a8c584529ed00891a8f7e41dc46d0441e
@@ -22,7 +22,7 @@ jobs:
           template-path: .github/ISSUE_TEMPLATE/bug-report.yaml
 
       - name: Set labels based on OS field
-        uses: redhat-plumbers-in-action/advanced-issue-labeler@39087a4b30cb98d57f25f34d617a6af8163c17d9
+        uses: redhat-plumbers-in-action/advanced-issue-labeler@e38e6809c5420d038eed380d49ee9a6ca7c92dbf
         with:
           issue-form: ${{ steps.issue-parser.outputs.jsonString }}
           section: os

.github/workflows/upgrade-patch-versions-schedule.yml

@@ -8,6 +8,7 @@ on:
 permissions: {}
 jobs:
   get-releases-branches:
+    if: github.repository == 'kubernetes-sigs/kubespray'
     runs-on: ubuntu-latest
     outputs:
       branches: ${{ steps.get-branches.outputs.data }}
@@ -19,7 +20,7 @@ jobs:
           query get_release_branches($owner:String!, $name:String!) {
             repository(owner:$owner, name:$name) {
               refs(refPrefix: "refs/heads/",
-                   first: 0, # TODO increment once we have release branch with the new checksums format
+                   first: 1, # TODO increment once we have release branch with the new checksums format
                    query: "release-",
                    orderBy: {
                      field: ALPHABETICAL,

.github/workflows/upgrade-patch-versions.yml

@@ -11,10 +11,10 @@ jobs:
   update-patch-versions:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+    - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
       with:
         ref: ${{ inputs.branch }}
-    - uses: actions/setup-python@v5
+    - uses: actions/setup-python@v6
       with:
         python-version: '3.13'
         cache: 'pip'

.gitlab-ci.yml

@@ -1,9 +1,9 @@
 ---
 stages:
-  - build
-  - test
-  - deploy-part1
-  - deploy-extended
+  - build # build docker image used in most other jobs
+  - test # unit tests
+  - deploy-part1 # kubespray runs - common setup
+  - deploy-extended # kubespray runs - rarer or costlier (to test) setups
 
 variables:
   FAILFASTCI_NAMESPACE: 'kargo-ci'
@@ -24,6 +24,7 @@ variables:
   ANSIBLE_REMOTE_USER: kubespray
   ANSIBLE_PRIVATE_KEY_FILE: /tmp/id_rsa
   ANSIBLE_INVENTORY: /tmp/inventory
+  ANSIBLE_STDOUT_CALLBACK: "default"
   RESET_CHECK: "false"
   REMOVE_NODE_CHECK: "false"
   UPGRADE_TEST: "false"
@@ -31,7 +32,7 @@ variables:
   ANSIBLE_VERBOSITY: 2
   RECOVER_CONTROL_PLANE_TEST: "false"
   RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
-  TF_VERSION: 1.3.7
+  OPENTOFU_VERSION: v1.9.1
   PIPELINE_IMAGE: "$CI_REGISTRY_IMAGE/pipeline:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"
 
 before_script:
@@ -48,44 +49,14 @@ before_script:
       - cluster-dump/
   needs:
     - pipeline-image
-  variables:
-    ANSIBLE_STDOUT_CALLBACK: "debug"
 
 .job-moderated:
   extends: .job
   needs:
     - pipeline-image
-    - ci-not-authorized
     - pre-commit            # lint
     - vagrant-validate      # lint
 
-# For failfast, at least 1 job must be defined in .gitlab-ci.yml
-# Premoderated with manual actions
-ci-not-authorized:
-  stage: build
-  before_script: []
-  after_script: []
-  rules:
-    # LGTM or ok-to-test labels
-    - if: $PR_LABELS =~ /.*,(lgtm|approved|ok-to-test).*|^(lgtm|approved|ok-to-test).*/i
-      variables:
-        CI_OK_TO_TEST: '0'
-      when: always
-    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "trigger"
-      variables:
-        CI_OK_TO_TEST: '0'
-    - if: $CI_COMMIT_BRANCH == "master"
-      variables:
-        CI_OK_TO_TEST: '0'
-    - when: always
-      variables:
-        CI_OK_TO_TEST: '1'
-  script:
-    - exit $CI_OK_TO_TEST
-  tags:
-    - ffci
-  needs: []
-
 include:
   - .gitlab-ci/build.yml
   - .gitlab-ci/lint.yml

.gitlab-ci/build.yml

@@ -7,23 +7,24 @@ pipeline-image:
   tags:
     - ffci
   stage: build
-  image:
-    name: gcr.io/kaniko-project/executor:debug
-    entrypoint: ['']
+  image: moby/buildkit:rootless
   variables:
-    GODEBUG: "http2client=0"
+    BUILDKITD_FLAGS: --oci-worker-no-process-sandbox
+    CACHE_IMAGE: $CI_REGISTRY_IMAGE/pipeline:cache
   # TODO: remove the override
   # currently rebase.sh depends on bash (not available in the kaniko image)
   # once we have a simpler rebase (which should be easy if the target branch ref is available as variable
   # we'll be able to rebase here as well hopefully
-  before_script: []
+  before_script:
+    - mkdir -p ~/.docker
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > ~/.docker/config.json
   script:
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json
-    - /kaniko/executor --cache=true
-                       --cache-dir=image-cache
-                       --context $CI_PROJECT_DIR
-                       --dockerfile $CI_PROJECT_DIR/pipeline.Dockerfile
-                       --label 'git-branch'=$CI_COMMIT_REF_SLUG
-                       --label 'git-tag=$CI_COMMIT_TAG'
-                       --destination $PIPELINE_IMAGE
-                       --log-timestamp=true
+    - |
+      buildctl-daemonless.sh build \
+        --frontend dockerfile.v0 \
+        --local context=$CI_PROJECT_DIR \
+        --local dockerfile=$CI_PROJECT_DIR \
+        --opt filename=pipeline.Dockerfile \
+        --export-cache type=registry,ref=$CACHE_IMAGE \
+        --import-cache type=registry,ref=$CACHE_IMAGE \
+        --output type=image,name=$PIPELINE_IMAGE,push=true

.gitlab-ci/kubevirt.yml

@@ -4,7 +4,7 @@
   interruptible: true
   script:
     - ansible-playbook tests/cloud_playbooks/create-kubevirt.yml
-                      -c local -e @"tests/files/${TESTCASE}.yml"
+                              -e @"tests/files/${TESTCASE}.yml"
     - ./tests/scripts/testcases_run.sh
   variables:
     ANSIBLE_TIMEOUT: "120"
@@ -12,10 +12,9 @@
     - ffci
   needs:
     - pipeline-image
-    - ci-not-authorized
 
 # TODO: generate testcases matrixes from the files in tests/files/
-# this is needed to avoid the need for PR rebasing when a job was added or remvoed in the target branch
+# this is needed to avoid the need for PR rebasing when a job was added or removed in the target branch
 # (currently, a removed job in the target branch breaks the tests, because the
 # pipeline definition is parsed by gitlab before the rebase.sh script)
 # CI template for PRs
@@ -27,42 +26,46 @@ pr:
       allow_failure: true
     - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
       when: on_success
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
+      when: on_success
     - when: manual
       allow_failure: true
   extends: .kubevirt
   parallel:
     matrix:
       - TESTCASE:
-          - almalinux8-calico
           - almalinux9-crio
           - almalinux9-kube-ovn
           - debian11-calico-collection
           - debian11-macvlan
           - debian12-cilium
+          - debian13-cilium
           - fedora39-kube-router
-            # FIXME: this test if broken (perma-failing)
           - openeuler24-calico
-          - opensuse15-6-calico
-          - rockylinux8-calico
           - rockylinux9-cilium
-          - ubuntu20-calico-all-in-one-hardening
-          - ubuntu20-cilium-sep
-          - ubuntu20-flannel-collection
-          - ubuntu20-kube-router-sep
-          - ubuntu20-kube-router-svc-proxy
           - ubuntu22-calico-all-in-one
           - ubuntu22-calico-all-in-one-upgrade
           - ubuntu24-calico-etcd-datastore
+          - ubuntu24-calico-all-in-one-hardening
+          - ubuntu24-cilium-sep
+          - ubuntu24-flannel-collection
+          - ubuntu24-kube-router-sep
+          - ubuntu24-kube-router-svc-proxy
+          - ubuntu24-ha-separate-etcd
+          - flatcar4081-calico
+          - fedora40-flannel-crio-collection-scale
 
-# The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
-ubuntu20-calico-all-in-one:
+# The ubuntu24-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
+ubuntu24-calico-all-in-one:
   stage: deploy-part1
   extends: .kubevirt
   variables:
-    TESTCASE: ubuntu20-calico-all-in-one
+    TESTCASE: ubuntu24-calico-all-in-one
   rules:
     - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
       when: on_success
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
+      when: on_success
     - when: manual
       allow_failure: true
 
@@ -72,6 +75,8 @@ pr_full:
   rules:
     - if: $PR_LABELS =~ /.*ci-full.*/
       when: on_success
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
+      when: on_success
     # Else run as manual
     - when: manual
       allow_failure: true
@@ -85,10 +90,9 @@ pr_full:
           - debian12-custom-cni-helm
           - fedora39-calico-swap-selinux
           - fedora39-crio
-          - ubuntu20-all-in-one-docker
-          - ubuntu20-calico-ha-wireguard
-          - ubuntu20-flannel-ha
-          - ubuntu20-flannel-ha-once
+          - ubuntu24-calico-ha-wireguard
+          - ubuntu24-flannel-ha
+          - ubuntu24-flannel-ha-once
 
 # Need an update of the container image to use schema v2
 # update: quay.io/kubespray/vm-amazon-linux-2:latest
@@ -108,6 +112,8 @@ pr_extended:
   rules:
     - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
       when: on_success
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
+      when: on_success
     - when: manual
       allow_failure: true
   parallel:
@@ -119,21 +125,21 @@ pr_extended:
           - debian11-docker
           - debian12-calico
           - debian12-docker
-          - opensuse15-6-docker-cilium
+          - debian13-calico
           - rockylinux9-calico
-          - ubuntu20-calico-etcd-kubeadm
-          - ubuntu20-flannel
           - ubuntu22-all-in-one-docker
           - ubuntu24-all-in-one-docker
           - ubuntu24-calico-all-in-one
+          - ubuntu24-calico-etcd-kubeadm
+          - ubuntu24-flannel
 
-# Enabled when PERIODIC_CI_ENABLED var is set
+# TODO: migrate to pr-full, fix the broken ones
 periodic:
-  only:
-    variables:
-      - $PERIODIC_CI_ENABLED
   allow_failure: true
   extends: .kubevirt
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
+      when: on_success
   parallel:
     matrix:
       - TESTCASE:
@@ -142,6 +148,6 @@ periodic:
           - debian12-cilium-svc-proxy
           - fedora39-calico-selinux
           - fedora40-docker-calico
-          - ubuntu20-calico-etcd-kubeadm-upgrade-ha
-          - ubuntu20-calico-ha-recover
-          - ubuntu20-calico-ha-recover-noquorum
+          - ubuntu24-calico-etcd-kubeadm-upgrade-ha
+          - ubuntu24-calico-ha-recover
+          - ubuntu24-calico-ha-recover-noquorum

.gitlab-ci/lint.yml

@@ -6,6 +6,7 @@ pre-commit:
   image: 'ghcr.io/pre-commit-ci/runner-image@sha256:fe01a6ec51b298412990b88627c3973b1146c7304f930f469bafa29ba60bcde9'
   variables:
     PRE_COMMIT_HOME: ${CI_PROJECT_DIR}/.cache/pre-commit
+    ANSIBLE_STDOUT_CALLBACK: default
   script:
   - pre-commit run --all-files --show-diff-on-failure
   cache:
@@ -23,4 +24,3 @@ vagrant-validate:
     VAGRANT_VERSION: 2.3.7
   script:
   - ./tests/scripts/vagrant-validate.sh
-  except: ['triggers', 'master']

.gitlab-ci/molecule.yml

@@ -1,17 +1,24 @@
 ---
 .molecule:
   tags: [ffci]
-  only: [/^pr-.*$/]
-  except: ['triggers']
+  rules: # run on ci-short as well
+  - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
+    when: on_success
+  - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
+    when: on_success
+  - when: manual
+    allow_failure: true
   stage: deploy-part1
   image: $PIPELINE_IMAGE
   needs:
   - pipeline-image
-  # - ci-not-authorized
   script:
   - ./tests/scripts/molecule_run.sh
   after_script:
-  - ./tests/scripts/molecule_logs.sh
+  - rm -fr molecule_logs
+  - mkdir -p molecule_logs
+  - find ~/.cache/molecule/ \( -name '*.out' -o -name '*.err' \) -type f | xargs tar -uf molecule_logs/molecule.tar
+  - gzip molecule_logs/molecule.tar
   artifacts:
     when: always
     paths:
@@ -27,28 +34,22 @@ molecule:
       - container-engine/cri-dockerd
       - container-engine/containerd
       - container-engine/cri-o
+      - container-engine/gvisor
+      - container-engine/youki
       - adduser
       - bastion-ssh-config
-      - bootstrap-os
+      - bootstrap_os
 
-# CI template for periodic CI jobs
-# Enabled when PERIODIC_CI_ENABLED var is set
 molecule_full:
-  only:
-    variables:
-    - $PERIODIC_CI_ENABLED
   allow_failure: true
+  rules:
+  - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
+    when: on_success
+  - when: manual
+    allow_failure: true
   extends: molecule
   parallel:
     matrix:
     - ROLE:
-      - container-engine/cri-dockerd
-      - container-engine/containerd
-      - container-engine/cri-o
-      - adduser
-      - bastion-ssh-config
-      - bootstrap-os
       # FIXME : tests below are perma-failing
       - container-engine/kata-containers
-      - container-engine/gvisor
-      - container-engine/youki

.gitlab-ci/terraform.yml

@@ -3,7 +3,6 @@
 .terraform_install:
   extends: .job
   needs:
-    - ci-not-authorized
     - pipeline-image
   variables:
     TF_VAR_public_key_path: "${ANSIBLE_PRIVATE_KEY_FILE}.pub"
@@ -14,18 +13,18 @@
   before_script:
     - ./tests/scripts/rebase.sh
     - mkdir -p cluster-dump $ANSIBLE_INVENTORY
-    - ./tests/scripts/terraform_install.sh
+    - ./tests/scripts/opentofu_install.sh
     - cp contrib/terraform/$PROVIDER/sample-inventory/cluster.tfvars .
     - ln -rs -t $ANSIBLE_INVENTORY contrib/terraform/$PROVIDER/hosts
-    - terraform -chdir="contrib/terraform/$PROVIDER" init
+    - tofu -chdir="contrib/terraform/$PROVIDER" init
 
 terraform_validate:
   extends: .terraform_install
   tags: [ffci]
   only: ['master', /^pr-.*$/]
   script:
-    - terraform -chdir="contrib/terraform/$PROVIDER" validate
-    - terraform -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
+    - tofu -chdir="contrib/terraform/$PROVIDER" validate
+    - tofu -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
   stage: test
   needs:
     - pipeline-image
@@ -33,7 +32,6 @@ terraform_validate:
     matrix:
       - PROVIDER:
           - openstack
-          - equinix
           - aws
           - exoscale
           - hetzner
@@ -60,11 +58,11 @@ terraform_validate:
     - mkdir -p contrib/terraform/$PROVIDER/group_vars
     # Random subnet to avoid routing conflicts
     - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
-    - terraform -chdir="contrib/terraform/$PROVIDER" apply -auto-approve -parallelism=1
+    - tofu -chdir="contrib/terraform/$PROVIDER" apply -auto-approve -parallelism=1
     - tests/scripts/testcases_run.sh
   after_script:
     # Cleanup regardless of exit code
-    - terraform -chdir="contrib/terraform/$PROVIDER" destroy -auto-approve
+    - tofu -chdir="contrib/terraform/$PROVIDER" destroy -auto-approve
 
 # Elastx is generously donating resources for Kubespray on Openstack CI
 # Contacts: @gix @bl0m1

.gitlab-ci/vagrant.yml

@@ -1,8 +1,6 @@
 ---
 vagrant:
   extends: .job-moderated
-  needs:
-    - ci-not-authorized
   variables:
     CI_PLATFORM: "vagrant"
     SSH_USER: "vagrant"
@@ -13,8 +11,6 @@ vagrant:
     VAGRANT_HOME: "$CI_PROJECT_DIR/.vagrant.d"
     PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
   tags: [ffci-vm-large]
-  # only: [/^pr-.*$/]
-  # except: ['triggers']
   image: quay.io/kubespray/vm-kubespray-ci:v13
   services: []
   before_script:
@@ -42,6 +38,10 @@ vagrant:
   rules:
     - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
       when: on_success
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
+      when: on_success
+    - when: manual
+      allow_failure: true
   parallel:
     matrix:
       - TESTCASE:

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -15,13 +15,13 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.35.1
+    rev: v1.37.1
     hooks:
       - id: yamllint
         args: [--strict]
 
   - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: v0.10.0.1
+    rev: v0.11.0.1
     hooks:
       - id: shellcheck
         args: ["--severity=error"]
@@ -29,7 +29,7 @@ repos:
         files: "\\.sh$"
 
   - repo: https://github.com/ansible/ansible-lint
-    rev: v25.1.1
+    rev: v25.11.0
     hooks:
       - id: ansible-lint
         additional_dependencies:
@@ -38,7 +38,7 @@ repos:
           - distlib
 
   - repo: https://github.com/golangci/misspell
-    rev: v0.6.0
+    rev: v0.7.0
     hooks:
       - id: misspell
         exclude: "OWNERS_ALIASES$"

CONTRIBUTING.md

@@ -40,7 +40,7 @@ Vagrant with VirtualBox or libvirt driver helps you to quickly spin test cluster
 2. The [repo owners](OWNERS) will respond to your issue promptly.
 3. Fork the desired repo, develop and test your code changes.
 4. Install [pre-commit](https://pre-commit.com) and install it in your development repo.
-5. Addess any pre-commit validation failures.
+5. Address any pre-commit validation failures.
 6. Sign the CNCF CLA (<https://git.k8s.io/community/CLA.md#the-contributor-license-agreement>)
 7. Submit a pull request.
 8. Work with the reviewers on their suggestions.

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1
 
-# Use imutable image tags rather than mutable tags (like ubuntu:22.04)
+# Use immutable image tags rather than mutable tags (like ubuntu:22.04)
 FROM ubuntu:22.04@sha256:149d67e29f765f4db62aa52161009e99e389544e25a8f43c8c89d4a445a7ca37
 
 # Some tools like yamllint need this
@@ -35,8 +35,8 @@ RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 RUN OS_ARCHITECTURE=$(dpkg --print-architecture) \
-    && curl -L "https://dl.k8s.io/release/v1.32.3/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
-    && echo "$(curl -L "https://dl.k8s.io/release/v1.32.3/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
+    && curl -L "https://dl.k8s.io/release/v1.33.7/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
+    && echo "$(curl -L "https://dl.k8s.io/release/v1.33.7/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
     && chmod a+x /usr/local/bin/kubectl
 
 COPY *.yml ./

OWNERS_ALIASES

@@ -1,17 +1,13 @@
 aliases:
   kubespray-approvers:
-    - cristicalin
-    - floryut
-    - liupeng0518
-    - mzaian
-    - oomichi
-    - yankay
     - ant31
+    - mzaian
+    - tico88612
     - vannten
+    - yankay
   kubespray-reviewers:
     - cyclinder
     - erikjiang
-    - mrfreezeex
     - mzaian
     - tico88612
     - vannten
@@ -19,8 +15,12 @@ aliases:
   kubespray-emeritus_approvers:
     - atoms
     - chadswen
+    - cristicalin
+    - floryut
+    - liupeng0518
     - luckysb
     - mattymo
     - miouge1
+    - oomichi
     - riverzhang
     - woopstar

README.md

@@ -22,7 +22,7 @@ Ensure you have installed Docker then
 ```ShellSession
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
   --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.27.0 bash
+  quay.io/kubespray/kubespray:v2.29.0 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```
@@ -87,8 +87,8 @@ vagrant up
 ## Supported Linux Distributions
 
 - **Flatcar Container Linux by Kinvolk**
-- **Debian** Bookworm, Bullseye
-- **Ubuntu** 20.04, 22.04, 24.04
+- **Debian** Bookworm, Bullseye, Trixie
+- **Ubuntu** 22.04, 24.04
 - **CentOS/RHEL** [8, 9](docs/operating_systems/rhel.md#rhel-8)
 - **Fedora** 39, 40
 - **Fedora CoreOS** (see [fcos Note](docs/operating_systems/fcos.md))
@@ -111,27 +111,26 @@ Note:
 <!-- BEGIN ANSIBLE MANAGED BLOCK -->
 
 - Core
-  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.32.3
-  - [etcd](https://github.com/etcd-io/etcd) 3.5.16
-  - [docker](https://www.docker.com/) 28.0
-  - [containerd](https://containerd.io/) 2.0.3
-  - [cri-o](http://cri-o.io/) 1.32.0 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
+  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.33.7
+  - [etcd](https://github.com/etcd-io/etcd) 3.5.26
+  - [docker](https://www.docker.com/) 28.3
+  - [containerd](https://containerd.io/) 2.1.6
+  - [cri-o](http://cri-o.io/) 1.33.8 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
-  - [cni-plugins](https://github.com/containernetworking/plugins) 1.4.1
-  - [calico](https://github.com/projectcalico/calico) 3.29.2
-  - [cilium](https://github.com/cilium/cilium) 1.15.9
-  - [flannel](https://github.com/flannel-io/flannel) 0.22.0
+  - [cni-plugins](https://github.com/containernetworking/plugins) 1.8.0
+  - [calico](https://github.com/projectcalico/calico) 3.30.6
+  - [cilium](https://github.com/cilium/cilium) 1.18.5
+  - [flannel](https://github.com/flannel-io/flannel) 0.27.3
   - [kube-ovn](https://github.com/alauda/kube-ovn) 1.12.21
   - [kube-router](https://github.com/cloudnativelabs/kube-router) 2.1.1
-  - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) 4.1.0
-  - [weave](https://github.com/rajch/weave) 2.8.7
+  - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) 4.2.2
   - [kube-vip](https://github.com/kube-vip/kube-vip) 0.8.0
 - Application
   - [cert-manager](https://github.com/jetstack/cert-manager) 1.15.3
-  - [coredns](https://github.com/coredns/coredns) 1.11.3
-  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) 1.12.1
+  - [coredns](https://github.com/coredns/coredns) 1.12.0
+  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) 1.13.3
   - [argocd](https://argoproj.github.io/) 2.14.5
-  - [helm](https://helm.sh/) 3.16.4
+  - [helm](https://helm.sh/) 3.18.4
   - [metallb](https://metallb.universe.tf/) 0.13.9
   - [registry](https://github.com/distribution/distribution) 2.8.1
 - Storage Plugin
@@ -139,7 +138,7 @@ Note:
   - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) 1.10.0
   - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) 1.30.0
   - [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) 1.9.2
-  - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) 0.0.24
+  - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) 0.0.32
   - [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) 2.5.0
   - [node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) 0.16.4
 
@@ -183,9 +182,6 @@ You can choose among ten network plugins. (default: `calico`, except Vagrant use
 
 - [cilium](http://docs.cilium.io/en/latest/): layer 3/4 networking (as well as layer 7 to protect and secure application protocols), supports dynamic insertion of BPF bytecode into the Linux kernel to implement security services, networking and visibility logic.
 
-- [weave](docs/CNI/weave.md): Weave is a lightweight container overlay network that doesn't require an external K/V database cluster.
-    (Please refer to `weave` [troubleshooting documentation](https://www.weave.works/docs/net/latest/troubleshooting/)).
-
 - [kube-ovn](docs/CNI/kube-ovn.md): Kube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It offers an advanced Container Network Fabric for Enterprises.
 
 - [kube-router](docs/CNI/kube-router.md): Kube-router is a L3 CNI for Kubernetes networking aiming to provide operational

Vagrantfile

@@ -4,6 +4,8 @@
 # For help on using kubespray with vagrant, check out docs/developers/vagrant.md
 
 require 'fileutils'
+require 'ipaddr'
+require 'socket'
 
 Vagrant.require_version ">= 2.0.0"
 
@@ -99,6 +101,33 @@ $extra_vars ||= {}
 
 host_vars = {}
 
+def collect_networks(subnet, subnet_ipv6)
+  Socket.getifaddrs.filter_map do |iface|
+    next unless iface&.netmask&.ip_address && iface.addr
+
+    is_ipv6 = iface.addr.ipv6?
+    ip      = IPAddr.new(iface.addr.ip_address.split('%').first)
+    ip_test = is_ipv6 ? IPAddr.new("#{subnet_ipv6}::0") : IPAddr.new("#{subnet}.0")
+
+    prefix  = IPAddr.new(iface.netmask.ip_address).to_i.to_s(2).count('1')
+    network = ip.mask(prefix)
+
+    [IPAddr.new("#{network}/#{prefix}"), ip_test]
+  end
+end
+
+def subnet_in_use?(network_ips)
+  network_ips.any? { |net, test_ip| net.include?(test_ip) && test_ip != net }
+end
+
+network_ips = collect_networks($subnet, $subnet_ipv6)
+
+if subnet_in_use?(network_ips)
+  puts "Invalid subnet provided, subnet is already in use: #{$subnet}.0"
+  puts "Subnets in use: #{network_ips.inspect}"
+  exit 1
+end
+
 # throw error if os is not supported
 if ! SUPPORTED_OS.key?($os)
   puts "Unsupported OS: #{$os}"

ansible.cfg

@@ -15,7 +15,7 @@ timeout = 300
 stdout_callback = default
 display_skipped_hosts = no
 library = ./library
-callbacks_enabled = profile_tasks,ara_default
+callbacks_enabled = profile_tasks
 roles_path = roles:$VIRTUAL_ENV/usr/local/share/kubespray/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:/usr/share/kubespray/roles
 deprecation_warnings=False
 inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo, .creds, .gpg

contrib/offline/README.md

@@ -31,7 +31,7 @@ manage-offline-container-images.sh   register
 
 ## generate_list.sh
 
-This script generates the list of downloaded files and the list of container images by `roles/kubespray-defaults/defaults/main/download.yml` file.
+This script generates the list of downloaded files and the list of container images by `roles/kubespray_defaults/defaults/main/download.yml` file.
 
 Run this script will execute `generate_list.yml` playbook in kubespray root directory and generate four files,
 all downloaded files url in files.list, all container images in images.list, jinja2 templates in *.template.

contrib/offline/generate_list.sh

@@ -5,7 +5,7 @@ CURRENT_DIR=$(cd $(dirname $0); pwd)
 TEMP_DIR="${CURRENT_DIR}/temp"
 REPO_ROOT_DIR="${CURRENT_DIR%/contrib/offline}"
 
-: ${DOWNLOAD_YML:="roles/kubespray-defaults/defaults/main/download.yml"}
+: ${DOWNLOAD_YML:="roles/kubespray_defaults/defaults/main/download.yml"}
 
 mkdir -p ${TEMP_DIR}
 
@@ -19,7 +19,7 @@ sed -n '/^downloads:/,/download_defaults:/p' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \
     | sed 'N;s#\n# #g' | tr ' ' ':' | sed 's/\"//g' > ${TEMP_DIR}/images.list.template
 
 # add kube-* images to images list template
-# Those container images are downloaded by kubeadm, then roles/kubespray-defaults/defaults/main/download.yml
+# Those container images are downloaded by kubeadm, then roles/kubespray_defaults/defaults/main/download.yml
 # doesn't contain those images. That is reason why here needs to put those images into the
 # list separately.
 KUBE_IMAGES="kube-apiserver kube-controller-manager kube-scheduler kube-proxy"

contrib/offline/generate_list.yml

@@ -5,7 +5,7 @@
 
   roles:
     # Just load default variables from roles.
-    - role: kubespray-defaults
+    - role: kubespray_defaults
       when: false
     - role: download
       when: false

contrib/offline/manage-offline-container-images.sh

@@ -36,7 +36,7 @@ function create_container_image_tar() {
 	mkdir  ${IMAGE_DIR}
 	cd     ${IMAGE_DIR}
 
-	sudo ${runtime} pull registry:latest
+	sudo --preserve-env=http_proxy,https_proxy,no_proxy ${runtime} pull registry:latest
 	sudo ${runtime} save -o registry-latest.tar registry:latest
 
 	while read -r image
@@ -45,7 +45,7 @@ function create_container_image_tar() {
 		set +e
 		for step in $(seq 1 ${RETRY_COUNT})
 		do
-			sudo ${runtime} pull ${image}
+			sudo --preserve-env=http_proxy,https_proxy,no_proxy ${runtime} pull ${image}
 			if [ $? -eq 0 ]; then
 				break
 			fi
@@ -127,7 +127,7 @@ function register_container_images() {
 
 	tar -zxvf ${IMAGE_TAR_FILE}
 
-	if [ "${create_registry}" ]; then
+	if ${create_registry}; then
 		sudo ${runtime} load -i ${IMAGE_DIR}/registry-latest.tar
 		set +e
 
@@ -148,7 +148,7 @@ function register_container_images() {
 		if [ "${org_image}" == "ID:" ]; then
 		  org_image=$(echo "${load_image}"  | awk '{print $4}')
 		fi
-		image_id=$(sudo ${runtime} image inspect ${org_image} | grep "\"Id\":" | awk -F: '{print $3}'| sed s/'\",'//)
+		image_id=$(sudo ${runtime} image inspect --format "{{.Id}}" "${org_image}")
 		if [ -z "${file_name}" ]; then
 			echo "Failed to get file_name for line ${line}"
 			exit 1

contrib/offline/manage-offline-files.sh

@@ -41,7 +41,7 @@ fi
 
 sudo "${runtime}" container inspect nginx >/dev/null 2>&1
 if [ $? -ne 0 ]; then
-    sudo "${runtime}" run \
+    sudo --preserve-env=http_proxy,https_proxy,no_proxy "${runtime}" run \
         --restart=always -d -p ${NGINX_PORT}:80 \
         --volume "${OFFLINE_FILES_DIR}":/usr/share/nginx/html/download \
         --volume "${CURRENT_DIR}"/nginx.conf:/etc/nginx/nginx.conf \

contrib/terraform/aws/create-infrastructure.tf

@@ -1,5 +1,11 @@
 terraform {
   required_version = ">= 0.12.0"
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.0"
+    }
+  }
 }
 
 provider "aws" {

contrib/terraform/equinix/README.md

@@ -1,246 +0,0 @@
-# Kubernetes on Equinix Metal with Terraform
-
-Provision a Kubernetes cluster with [Terraform](https://www.terraform.io) on
-[Equinix Metal](https://metal.equinix.com) ([formerly Packet](https://blog.equinix.com/blog/2020/10/06/equinix-metal-metal-and-more/)).
-
-## Status
-
-This will install a Kubernetes cluster on Equinix Metal. It should work in all locations and on most server types.
-
-## Approach
-
-The terraform configuration inspects variables found in
-[variables.tf](variables.tf) to create resources in your Equinix Metal project.
-There is a [python script](../terraform.py) that reads the generated`.tfstate`
-file to generate a dynamic inventory that is consumed by [cluster.yml](../../../cluster.yml)
-to actually install Kubernetes with Kubespray.
-
-### Kubernetes Nodes
-
-You can create many different kubernetes topologies by setting the number of
-different classes of hosts.
-
-- Master nodes with etcd: `number_of_k8s_masters` variable
-- Master nodes without etcd: `number_of_k8s_masters_no_etcd` variable
-- Standalone etcd hosts: `number_of_etcd` variable
-- Kubernetes worker nodes: `number_of_k8s_nodes` variable
-
-Note that the Ansible script will report an invalid configuration if you wind up
-with an *even number* of etcd instances since that is not a valid configuration. This
-restriction includes standalone etcd nodes that are deployed in a cluster along with
-master nodes with etcd replicas. As an example, if you have three master nodes with
-etcd replicas and three standalone etcd nodes, the script will fail since there are
-now six total etcd replicas.
-
-## Requirements
-
-- [Install Terraform](https://www.terraform.io/intro/getting-started/install.html)
-- [Install Ansible dependencies](/docs/ansible/ansible.md#installing-ansible)
-- Account with Equinix Metal
-- An SSH key pair
-
-## SSH Key Setup
-
-An SSH keypair is required so Ansible can access the newly provisioned nodes (Equinix Metal hosts). By default, the public SSH key defined in cluster.tfvars will be installed in authorized_key on the newly provisioned nodes (~/.ssh/id_rsa.pub). Terraform will upload this public key and then it will be distributed out to all the nodes. If you have already set this public key in Equinix Metal (i.e. via the portal), then set the public keyfile name in cluster.tfvars to blank to prevent the duplicate key from being uploaded which will cause an error.
-
-If you don't already have a keypair generated (~/.ssh/id_rsa and ~/.ssh/id_rsa.pub), then a new keypair can be generated with the command:
-
-```ShellSession
-ssh-keygen -f ~/.ssh/id_rsa
-```
-
-## Terraform
-
-Terraform will be used to provision all of the Equinix Metal resources with base software as appropriate.
-
-### Configuration
-
-#### Inventory files
-
-Create an inventory directory for your cluster by copying the existing sample and linking the `hosts` script (used to build the inventory based on Terraform state):
-
-```ShellSession
-cp -LRp contrib/terraform/equinix/sample-inventory inventory/$CLUSTER
-cd inventory/$CLUSTER
-ln -s ../../contrib/terraform/equinix/hosts
-```
-
-This will be the base for subsequent Terraform commands.
-
-#### Equinix Metal API access
-
-Your Equinix Metal API key must be available in the `METAL_AUTH_TOKEN` environment variable.
-This key is typically stored outside of the code repo since it is considered secret.
-If someone gets this key, they can startup/shutdown hosts in your project!
-
-For more information on how to generate an API key or find your project ID, please see
-[Accounts Index](https://metal.equinix.com/developers/docs/accounts/).
-
-The Equinix Metal Project ID associated with the key will be set later in `cluster.tfvars`.
-
-For more information about the API, please see [Equinix Metal API](https://metal.equinix.com/developers/api/).
-
-For more information about terraform provider authentication, please see [the equinix provider documentation](https://registry.terraform.io/providers/equinix/equinix/latest/docs).
-
-Example:
-
-```ShellSession
-export METAL_AUTH_TOKEN="Example-API-Token"
-```
-
-Note that to deploy several clusters within the same project you need to use [terraform workspace](https://www.terraform.io/docs/state/workspaces.html#using-workspaces).
-
-#### Cluster variables
-
-The construction of the cluster is driven by values found in
-[variables.tf](variables.tf).
-
-For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
-
-The `cluster_name` is used to set a tag on each server deployed as part of this cluster.
-This helps when identifying which hosts are associated with each cluster.
-
-While the defaults in variables.tf will successfully deploy a cluster, it is recommended to set the following values:
-
-- cluster_name = the name of the inventory directory created above as $CLUSTER
-- equinix_metal_project_id = the Equinix Metal Project ID associated with the Equinix Metal API token above
-
-#### Enable localhost access
-
-Kubespray will pull down a Kubernetes configuration file to access this cluster by enabling the
-`kubeconfig_localhost: true` in the Kubespray configuration.
-
-Edit `inventory/$CLUSTER/group_vars/k8s_cluster/k8s_cluster.yml` and comment back in the following line and change from `false` to `true`:
-`\# kubeconfig_localhost: false`
-becomes:
-`kubeconfig_localhost: true`
-
-Once the Kubespray playbooks are run, a Kubernetes configuration file will be written to the local host at `inventory/$CLUSTER/artifacts/admin.conf`
-
-#### Terraform state files
-
-In the cluster's inventory folder, the following files might be created (either by Terraform
-or manually), to prevent you from pushing them accidentally they are in a
-`.gitignore` file in the `contrib/terraform/equinix` directory :
-
-- `.terraform`
-- `.tfvars`
-- `.tfstate`
-- `.tfstate.backup`
-- `.lock.hcl`
-
-You can still add them manually if you want to.
-
-### Initialization
-
-Before Terraform can operate on your cluster you need to install the required
-plugins. This is accomplished as follows:
-
-```ShellSession
-cd inventory/$CLUSTER
-terraform -chdir=../../contrib/terraform/metal init -var-file=cluster.tfvars
-```
-
-This should finish fairly quickly telling you Terraform has successfully initialized and loaded necessary modules.
-
-### Provisioning cluster
-
-You can apply the Terraform configuration to your cluster with the following command
-issued from your cluster's inventory directory (`inventory/$CLUSTER`):
-
-```ShellSession
-terraform -chdir=../../contrib/terraform/equinix apply -var-file=cluster.tfvars
-export ANSIBLE_HOST_KEY_CHECKING=False
-ansible-playbook -i hosts ../../cluster.yml
-```
-
-### Destroying cluster
-
-You can destroy your new cluster with the following command issued from the cluster's inventory directory:
-
-```ShellSession
-terraform -chdir=../../contrib/terraform/equinix destroy -var-file=cluster.tfvars
-```
-
-If you've started the Ansible run, it may also be a good idea to do some manual cleanup:
-
-- Remove SSH keys from the destroyed cluster from your `~/.ssh/known_hosts` file
-- Clean up any temporary cache files: `rm /tmp/$CLUSTER-*`
-
-### Debugging
-
-You can enable debugging output from Terraform by setting `TF_LOG` to `DEBUG` before running the Terraform command.
-
-## Ansible
-
-### Node access
-
-#### SSH
-
-Ensure your local ssh-agent is running and your ssh key has been added. This
-step is required by the terraform provisioner:
-
-```ShellSession
-eval $(ssh-agent -s)
-ssh-add ~/.ssh/id_rsa
-```
-
-If you have deployed and destroyed a previous iteration of your cluster, you will need to clear out any stale keys from your SSH "known hosts" file ( `~/.ssh/known_hosts`).
-
-#### Test access
-
-Make sure you can connect to the hosts.  Note that Flatcar Container Linux by Kinvolk will have a state `FAILED` due to Python not being present.  This is okay, because Python will be installed during bootstrapping, so long as the hosts are not `UNREACHABLE`.
-
-```ShellSession
-$ ansible -i inventory/$CLUSTER/hosts -m ping all
-example-k8s_node-1 | SUCCESS => {
-    "changed": false,
-    "ping": "pong"
-}
-example-etcd-1 | SUCCESS => {
-    "changed": false,
-    "ping": "pong"
-}
-example-k8s-master-1 | SUCCESS => {
-    "changed": false,
-    "ping": "pong"
-}
-```
-
-If it fails try to connect manually via SSH.  It could be something as simple as a stale host key.
-
-### Deploy Kubernetes
-
-```ShellSession
-ansible-playbook --become -i inventory/$CLUSTER/hosts cluster.yml
-```
-
-This will take some time as there are many tasks to run.
-
-## Kubernetes
-
-### Set up kubectl
-
-- [Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on the localhost.
-- Verify that Kubectl runs correctly
-
-```ShellSession
-kubectl version
-```
-
-- Verify that the Kubernetes configuration file has been copied over
-
-```ShellSession
-cat inventory/alpha/$CLUSTER/admin.conf
-```
-
-- Verify that all the nodes are running correctly.
-
-```ShellSession
-kubectl version
-kubectl --kubeconfig=inventory/$CLUSTER/artifacts/admin.conf  get nodes
-```
-
-## What's next
-
-Try out your new Kubernetes cluster with the [Hello Kubernetes service](https://kubernetes.io/docs/tasks/access-application-cluster/service-access-application-cluster/).

contrib/terraform/equinix/hosts

@@ -1 +0,0 @@
-../terraform.py

contrib/terraform/equinix/kubespray.tf

@@ -1,57 +0,0 @@
-resource "equinix_metal_ssh_key" "k8s" {
-  count      = var.public_key_path != "" ? 1 : 0
-  name       = "kubernetes-${var.cluster_name}"
-  public_key = chomp(file(var.public_key_path))
-}
-
-resource "equinix_metal_device" "k8s_master" {
-  depends_on = [equinix_metal_ssh_key.k8s]
-
-  count            = var.number_of_k8s_masters
-  hostname         = "${var.cluster_name}-k8s-master-${count.index + 1}"
-  plan             = var.plan_k8s_masters
-  metro            = var.metro
-  operating_system = var.operating_system
-  billing_cycle    = var.billing_cycle
-  project_id       = var.equinix_metal_project_id
-  tags             = ["cluster-${var.cluster_name}", "k8s_cluster", "kube_control_plane", "etcd", "kube_node"]
-}
-
-resource "equinix_metal_device" "k8s_master_no_etcd" {
-  depends_on = [equinix_metal_ssh_key.k8s]
-
-  count            = var.number_of_k8s_masters_no_etcd
-  hostname         = "${var.cluster_name}-k8s-master-${count.index + 1}"
-  plan             = var.plan_k8s_masters_no_etcd
-  metro            = var.metro
-  operating_system = var.operating_system
-  billing_cycle    = var.billing_cycle
-  project_id       = var.equinix_metal_project_id
-  tags             = ["cluster-${var.cluster_name}", "k8s_cluster", "kube_control_plane"]
-}
-
-resource "equinix_metal_device" "k8s_etcd" {
-  depends_on = [equinix_metal_ssh_key.k8s]
-
-  count            = var.number_of_etcd
-  hostname         = "${var.cluster_name}-etcd-${count.index + 1}"
-  plan             = var.plan_etcd
-  metro            = var.metro
-  operating_system = var.operating_system
-  billing_cycle    = var.billing_cycle
-  project_id       = var.equinix_metal_project_id
-  tags             = ["cluster-${var.cluster_name}", "etcd"]
-}
-
-resource "equinix_metal_device" "k8s_node" {
-  depends_on = [equinix_metal_ssh_key.k8s]
-
-  count            = var.number_of_k8s_nodes
-  hostname         = "${var.cluster_name}-k8s-node-${count.index + 1}"
-  plan             = var.plan_k8s_nodes
-  metro            = var.metro
-  operating_system = var.operating_system
-  billing_cycle    = var.billing_cycle
-  project_id       = var.equinix_metal_project_id
-  tags             = ["cluster-${var.cluster_name}", "k8s_cluster", "kube_node"]
-}

contrib/terraform/equinix/output.tf

@@ -1,15 +0,0 @@
-output "k8s_masters" {
-  value = equinix_metal_device.k8s_master.*.access_public_ipv4
-}
-
-output "k8s_masters_no_etc" {
-  value = equinix_metal_device.k8s_master_no_etcd.*.access_public_ipv4
-}
-
-output "k8s_etcds" {
-  value = equinix_metal_device.k8s_etcd.*.access_public_ipv4
-}
-
-output "k8s_nodes" {
-  value = equinix_metal_device.k8s_node.*.access_public_ipv4
-}

contrib/terraform/equinix/provider.tf

@@ -1,17 +0,0 @@
-terraform {
-  required_version = ">= 1.0.0"
-
-  provider_meta "equinix" {
-    module_name = "kubespray"
-  }
-  required_providers {
-    equinix = {
-      source  = "equinix/equinix"
-      version = "1.24.0"
-    }
-  }
-}
-
-# Configure the Equinix Metal Provider
-provider "equinix" {
-}

contrib/terraform/equinix/sample-inventory/cluster.tfvars

@@ -1,35 +0,0 @@
-# your Kubernetes cluster name here
-cluster_name = "mycluster"
-
-# Your Equinix Metal project ID. See https://metal.equinix.com/developers/docs/accounts/
-equinix_metal_project_id = "Example-Project-Id"
-
-# The public SSH key to be uploaded into authorized_keys in bare metal Equinix Metal nodes provisioned
-# leave this value blank if the public key is already setup in the Equinix Metal project
-# Terraform will complain if the public key is setup in Equinix Metal
-public_key_path = "~/.ssh/id_rsa.pub"
-
-# Equinix interconnected bare metal across our global metros.
-metro = "da"
-
-# operating_system
-operating_system = "ubuntu_22_04"
-
-# standalone etcds
-number_of_etcd = 0
-
-plan_etcd = "t1.small.x86"
-
-# masters
-number_of_k8s_masters = 1
-
-number_of_k8s_masters_no_etcd = 0
-
-plan_k8s_masters = "t1.small.x86"
-
-plan_k8s_masters_no_etcd = "t1.small.x86"
-
-# nodes
-number_of_k8s_nodes = 2
-
-plan_k8s_nodes = "t1.small.x86"

contrib/terraform/equinix/sample-inventory/group_vars

@@ -1 +0,0 @@
-../../../../inventory/sample/group_vars

contrib/terraform/equinix/variables.tf

@@ -1,56 +0,0 @@
-variable "cluster_name" {
-  default = "kubespray"
-}
-
-variable "equinix_metal_project_id" {
-  description = "Your Equinix Metal project ID. See https://metal.equinix.com/developers/docs/accounts/"
-}
-
-variable "operating_system" {
-  default = "ubuntu_22_04"
-}
-
-variable "public_key_path" {
-  description = "The path of the ssh pub key"
-  default     = "~/.ssh/id_rsa.pub"
-}
-
-variable "billing_cycle" {
-  default = "hourly"
-}
-
-variable "metro" {
-  default = "da"
-}
-
-variable "plan_k8s_masters" {
-  default = "c3.small.x86"
-}
-
-variable "plan_k8s_masters_no_etcd" {
-  default = "c3.small.x86"
-}
-
-variable "plan_etcd" {
-  default = "c3.small.x86"
-}
-
-variable "plan_k8s_nodes" {
-  default = "c3.medium.x86"
-}
-
-variable "number_of_k8s_masters" {
-  default = 1
-}
-
-variable "number_of_k8s_masters_no_etcd" {
-  default = 0
-}
-
-variable "number_of_etcd" {
-  default = 0
-}
-
-variable "number_of_k8s_nodes" {
-  default = 1
-}

contrib/terraform/openstack/README.md

@@ -624,7 +624,7 @@ Edit `inventory/$CLUSTER/group_vars/k8s_cluster/k8s_cluster.yml`:
   - **calico** requires [configuring OpenStack Neutron ports](/docs/cloud_controllers/openstack.md) to allow service and pod subnets
 
 ```yml
-# Choose network plugin (calico, weave or flannel)
+# Choose network plugin (calico or flannel)
 # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
 kube_network_plugin: flannel
 ```

docs/CNI/calico.md

@@ -180,7 +180,7 @@ calico_group_id=rr1
 The inventory above will deploy the following topology assuming that calico's
 `global_as_num` is set to `65400`:
 
-![Image](figures/kubespray-calico-rr.png?raw=true)
+![Image](../figures/kubespray-calico-rr.png?raw=true)
 
 ### Optional : Define default endpoint to host action
 

docs/CNI/cilium.md

@@ -54,6 +54,10 @@ cilium_loadbalancer_ip_pools:
   - name: "blue-pool"
     cidrs:
       - "10.0.10.0/24"
+    ranges:
+      - start: "20.0.20.100"
+        stop: "20.0.20.200"
+      - start: "1.2.3.4"
 ```
 
 For further information, check [LB IPAM documentation](https://docs.cilium.io/en/stable/network/lb-ipam/)
@@ -233,7 +237,7 @@ cilium_operator_extra_volume_mounts:
 ## Choose Cilium version
 
 ```yml
-cilium_version: "1.15.9"
+cilium_version: "1.18.5"
 ```
 
 ## Add variable to config

docs/CNI/weave.md

@@ -1,79 +0,0 @@
-# Weave
-
-Weave 2.0.1 is supported by kubespray
-
-Weave uses [**consensus**](https://www.weave.works/docs/net/latest/ipam/##consensus) mode (default mode) and [**seed**](https://www.weave.works/docs/net/latest/ipam/#seed) mode.
-
-`Consensus` mode is best to use on static size cluster and `seed` mode is best to use on dynamic size cluster
-
-Weave encryption is supported for all communication
-
-* To use Weave encryption, specify a strong password (if no password, no encryption)
-
-```ShellSession
-# In file ./inventory/sample/group_vars/k8s_cluster.yml
-weave_password: EnterPasswordHere
-```
-
-This password is used to set an environment variable inside weave container.
-
-Weave is deployed by kubespray using a daemonSet
-
-* Check the status of Weave containers
-
-```ShellSession
-# From client
-kubectl -n kube-system get pods | grep weave
-# output
-weave-net-50wd2                       2/2       Running   0          2m
-weave-net-js9rb                       2/2       Running   0          2m
-```
-
-There must be as many pods as nodes (here kubernetes have 2 nodes so there are 2 weave pods).
-
-* Check status of weave (connection,encryption ...) for each node
-
-```ShellSession
-# On nodes
-curl http://127.0.0.1:6784/status
-# output on node1
-Version: 2.0.1 (up to date; next check at 2017/08/01 13:51:34)
-
-        Service: router
-       Protocol: weave 1..2
-           Name: fa:16:3e:b3:d6:b2(node1)
-     Encryption: enabled
-  PeerDiscovery: enabled
-        Targets: 2
-    Connections: 2 (1 established, 1 failed)
-          Peers: 2 (with 2 established connections)
- TrustedSubnets: none
-
-        Service: ipam
-         Status: ready
-          Range: 10.233.64.0/18
-  DefaultSubnet: 10.233.64.0/18
-```
-
-* Check parameters of weave for each node
-
-```ShellSession
-# On nodes
-ps -aux | grep weaver
-# output on node1 (here its use seed mode)
-root      8559  0.2  3.0 365280 62700 ?        Sl   08:25   0:00 /home/weave/weaver --name=fa:16:3e:b3:d6:b2 --port=6783 --datapath=datapath --host-root=/host --http-addr=127.0.0.1:6784 --status-addr=0.0.0.0:6782 --docker-api= --no-dns --db-prefix=/weavedb/weave-net --ipalloc-range=10.233.64.0/18 --nickname=node1 --ipalloc-init seed=fa:16:3e:b3:d6:b2,fa:16:3e:f0:50:53 --conn-limit=30 --expect-npc 192.168.208.28 192.168.208.19
-```
-
-## Consensus mode (default mode)
-
-This mode is best to use on static size cluster
-
-### Seed mode
-
-This mode is best to use on dynamic size cluster
-
-The seed mode also allows multi-clouds and hybrid on-premise/cloud clusters deployment.
-
-* Switch from consensus mode to seed/Observation mode
-
-See [weave ipam documentation](https://www.weave.works/docs/net/latest/tasks/ipam/ipam/) and use `weave_extra_args` to enable.

docs/CRI/containerd.md

@@ -68,8 +68,8 @@ containerd_runc_runtime:
   engine: ""
   root: ""
   options:
-    systemdCgroup: "false"
-    binaryName: /usr/local/bin/my-runc
+    SystemdCgroup: "false"
+    BinaryName: /usr/local/bin/my-runc
   base_runtime_spec: cri-base.json
 ```
 
@@ -149,3 +149,11 @@ following configuration:
 ```yaml
 nri_enabled: true
 ```
+
+### Optional : Static Binary
+
+To ensure compatibility with older distributions (such as Debian 11), you can use a static containerd binary. By default, this is static binary if the system's glibc version is less than 2.34; otherwise, it is the default binary.
+
+```yaml
+containerd_static_binary: true
+```

docs/_sidebar.md

@@ -23,7 +23,6 @@
   * [Aws](/docs/cloud_providers/aws.md)
   * [Azure](/docs/cloud_providers/azure.md)
   * [Cloud](/docs/cloud_providers/cloud.md)
-  * [Equinix-metal](/docs/cloud_providers/equinix-metal.md)
 * CNI
   * [Calico](/docs/CNI/calico.md)
   * [Cilium](/docs/CNI/cilium.md)
@@ -33,7 +32,6 @@
   * [Kube-router](/docs/CNI/kube-router.md)
   * [Macvlan](/docs/CNI/macvlan.md)
   * [Multus](/docs/CNI/multus.md)
-  * [Weave](/docs/CNI/weave.md)
 * CRI
   * [Containerd](/docs/CRI/containerd.md)
   * [Cri-o](/docs/CRI/cri-o.md)

docs/advanced/arch.md

@@ -9,7 +9,6 @@ The following table shows the impact of the CPU architecture on compatible featu
 | kube_network_plugin | amd64 | arm64 | amd64 + arm64 |
 |---------------------|-------|-------|---------------|
 | Calico              | Y     | Y     | Y             |
-| Weave               | Y     | Y     | Y             |
 | Flannel             | Y     | N     | N             |
 | Canal               | Y     | N     | N             |
 | Cilium              | Y     | Y     | N             |

docs/advanced/proxy.md

@@ -1,6 +1,6 @@
 # Setting up Environment Proxy
 
-If you set http and https proxy, all nodes and loadbalancer will be excluded from proxy with generating no_proxy variable in `roles/kubespray-defaults/tasks/no_proxy.yml`, if you have additional resources for exclude add them to `additional_no_proxy` variable. If you want fully override your `no_proxy` setting, then fill in just `no_proxy` and no nodes or loadbalancer addresses will be added to no_proxy.
+If you set http and https proxy, all nodes and loadbalancer will be excluded from proxy with generating no_proxy variable in `roles/kubespray_defaults/tasks/no_proxy.yml`, if you have additional resources for exclude add them to `additional_no_proxy` variable. If you want fully override your `no_proxy` setting, then fill in just `no_proxy` and no nodes or loadbalancer addresses will be added to no_proxy.
 
 ## Set proxy for http and https
 

docs/ansible/ansible.md

@@ -13,7 +13,7 @@ KUBESPRAYDIR=kubespray
 python3 -m venv $VENVDIR
 source $VENVDIR/bin/activate
 cd $KUBESPRAYDIR
-pip install -U -r requirements.txt
+pip install -r requirements.txt
 ```
 
 In case you have a similar message when installing the requirements:
@@ -32,7 +32,7 @@ Based on the table below and the available python version for your ansible host
 
 | Ansible Version | Python Version |
 |-----------------|----------------|
-| >= 2.16.4       | 3.10-3.12      |
+| >= 2.17.3       | 3.10-3.12      |
 
 ## Customize Ansible vars
 
@@ -62,7 +62,7 @@ The following tags are defined in playbooks:
 | aws-ebs-csi-driver             | Configuring csi driver: aws-ebs                       |
 | azure-csi-driver               | Configuring csi driver: azure                         |
 | bastion                        | Setup ssh config for bastion                          |
-| bootstrap-os                   | Anything related to host OS configuration             |
+| bootstrap_os                   | Anything related to host OS configuration             |
 | calico                         | Network plugin Calico                                 |
 | calico_rr                      | Configuring Calico route reflector                    |
 | cert-manager                   | Configuring certificate manager for K8s               |
@@ -118,7 +118,6 @@ The following tags are defined in playbooks:
 | local-path-provisioner         | Configure External provisioner: local-path            |
 | local-volume-provisioner       | Configure External provisioner: local-volume          |
 | macvlan                        | Network plugin macvlan                                |
-| master (DEPRECATED)            | Deprecated - see `control-plane`                      |
 | metallb                        | Installing and configuring metallb                    |
 | metrics_server                 | Configuring metrics_server                            |
 | netchecker                     | Installing netchecker K8s app                         |
@@ -153,21 +152,16 @@ The following tags are defined in playbooks:
 | upgrade                        | Upgrading, f.e. container images/binaries             |
 | upload                         | Distributing images/binaries across hosts             |
 | vsphere-csi-driver             | Configuring csi driver: vsphere                       |
-| weave                          | Network plugin Weave                                  |
 | win_nodes                      | Running windows specific tasks                        |
 | youki                          | Configuring youki runtime                             |
 
-Note: Use the ``bash scripts/gen_tags.sh`` command to generate a list of all
-tags found in the codebase. New tags will be listed with the empty "Used for"
-field.
-
 ## Example commands
 
 Example command to filter and apply only DNS configuration tasks and skip
 everything else related to host OS configuration and downloading images of containers:
 
 ```ShellSession
-ansible-playbook -i inventory/sample/hosts.ini cluster.yml --tags preinstall,facts --skip-tags=download,bootstrap-os
+ansible-playbook -i inventory/sample/hosts.ini cluster.yml --tags preinstall,facts --skip-tags=download,bootstrap_os
 ```
 
 And this play only removes the K8s cluster DNS resolver IP from hosts' /etc/resolv.conf files:
@@ -206,11 +200,11 @@ You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mou
 to access the inventory and SSH key in the container, like this:
 
 ```ShellSession
-git checkout v2.27.0
-docker pull quay.io/kubespray/kubespray:v2.27.0
+git checkout v2.29.0
+docker pull quay.io/kubespray/kubespray:v2.29.0
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
   --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.27.0 bash
+  quay.io/kubespray/kubespray:v2.29.0 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```

docs/ansible/ansible_collection.md

@@ -2,14 +2,13 @@
 
 Kubespray can be installed as an [Ansible collection](https://docs.ansible.com/ansible/latest/user_guide/collections_using.html).
 
-## Requirements
-
-- An inventory file with the appropriate host groups. See the [README](../README.md#usage).
-- A `group_vars` directory. These group variables **need** to match the appropriate variable names under `inventory/local/group_vars`. See the [README](../README.md#usage).
-
 ## Usage
 
-1. Add Kubespray to your requirements.yml file
+1. Set up an inventory with the appropriate host groups and required group vars.
+   See also the documentation on [kubespray inventories](./inventory.md) and the
+   general ["Getting started" documentation](../getting_started/getting-started.md#building-your-own-inventory).
+
+2. Add Kubespray to your requirements.yml file
 
    ```yaml
    collections:
@@ -18,20 +17,20 @@ Kubespray can be installed as an [Ansible collection](https://docs.ansible.com/a
      version: master # use the appropriate tag or branch for the version you need
    ```
 
-2. Install your collection
+3. Install your collection
 
    ```ShellSession
    ansible-galaxy install -r requirements.yml
    ```
 
-3. Create a playbook to install your Kubernetes cluster
+4. Create a playbook to install your Kubernetes cluster
 
    ```yaml
    - name: Install Kubernetes
      ansible.builtin.import_playbook: kubernetes_sigs.kubespray.cluster
    ```
 
-4. Update INVENTORY and PLAYBOOK so that they point to your inventory file and the playbook you created above, and then install Kubespray
+5. Update INVENTORY and PLAYBOOK so that they point to your inventory file and the playbook you created above, and then install Kubespray
 
    ```ShellSession
    ansible-playbook -i INVENTORY --become --become-user=root PLAYBOOK

docs/ansible/vars.md

@@ -103,13 +103,13 @@ following default cluster parameters:
 
 * *kube_service_addresses_ipv6* - Subnet for cluster IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1000/116``). Must not overlap with ``kube_pods_subnet_ipv6``.
 
-* *kube_service_subnets* - All service subnets separated by commas (default is a mix of ``kube_service_addresses`` and ``kube_service_addresses_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stacke`` options),
+* *kube_service_subnets* - All service subnets separated by commas (default is a mix of ``kube_service_addresses`` and ``kube_service_addresses_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stack`` options),
   for example ``10.233.0.0/18,fd85:ee78:d8a6:8607::1000/116`` for dual stack(ipv4_stack/ipv6_stack set to `true`).
   It is not recommended to change this variable directly.
 
 * *kube_pods_subnet_ipv6* - Subnet for Pod IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1:0000/112``). Must not overlap with ``kube_service_addresses_ipv6``.
 
-* *kube_pods_subnets* - All pods subnets separated by commas (default is a mix of ``kube_pods_subnet`` and ``kube_pod_subnet_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stacke`` options),
+* *kube_pods_subnets* - All pods subnets separated by commas (default is a mix of ``kube_pods_subnet`` and ``kube_pod_subnet_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stack`` options),
   for example ``10.233.64.0/18,fd85:ee78:d8a6:8607::1:0000/112`` for dual stack(ipv4_stack/ipv6_stack set to `true`).
   It is not recommended to change this variable directly.
 
@@ -180,7 +180,7 @@ and ``kube_pods_subnet``, for example from the ``172.18.0.0/16``.
 
 IPv4 stack enable by *ipv4_stack* is set to ``true``, by default.
 IPv6 stack enable by *ipv6_stack* is set to ``false`` by default.
-This will use the default IPv4 and IPv6 subnets specified in the defaults file in the ``kubespray-defaults`` role, unless overridden of course. The default config will give you room for up to 256 nodes with 126 pods per node, and up to 4096 services.
+This will use the default IPv4 and IPv6 subnets specified in the defaults file in the ``kubespray_defaults`` role, unless overridden of course. The default config will give you room for up to 256 nodes with 126 pods per node, and up to 4096 services.
 Set both variables to ``true`` for Dual Stack mode.
 IPv4 has higher priority in Dual Stack mode(e.g. in variables `main_ip`, `main_access_ip` and other).
 You can also make IPv6 only clusters with ``false`` in *ipv4_stack*.

docs/cloud_controllers/openstack.md

@@ -73,6 +73,7 @@ The cloud provider is configured to have Octavia by default in Kubespray.
   external_openstack_lbaas_method: ROUND_ROBIN
   external_openstack_lbaas_provider: amphora
   external_openstack_lbaas_subnet_id: "Neutron subnet ID to create LBaaS VIP"
+  external_openstack_lbaas_member_subnet_id: "Neutron subnet ID on which to create the members of the load balancer"
   external_openstack_lbaas_network_id: "Neutron network ID to create LBaaS VIP"
   external_openstack_lbaas_manage_security_groups: false
   external_openstack_lbaas_create_monitor: false

docs/cloud_providers/equinix-metal.md

@@ -1,100 +0,0 @@
-# Equinix Metal
-
-Kubespray provides support for bare metal deployments using the [Equinix Metal](http://metal.equinix.com).
-Deploying upon bare metal allows Kubernetes to run at locations where an existing public or private cloud might not exist such
-as cell tower, edge collocated installations. The deployment mechanism used by Kubespray for Equinix Metal is similar to that used for
-AWS and OpenStack clouds (notably using Terraform to deploy the infrastructure). Terraform uses the Equinix Metal provider plugin
-to provision and configure hosts which are then used by the Kubespray Ansible playbooks. The Ansible inventory is generated
-dynamically from the Terraform state file.
-
-## Local Host Configuration
-
-To perform this installation, you will need a localhost to run Terraform/Ansible (laptop, VM, etc) and an account with Equinix Metal.
-In this example, we are provisioning a m1.large CentOS7 OpenStack VM as the localhost for the Kubernetes installation.
-You'll need Ansible, Git, and PIP.
-
-```bash
-sudo yum install epel-release
-sudo yum install ansible
-sudo yum install git
-sudo yum install python-pip
-```
-
-## Playbook SSH Key
-
-An SSH key is needed by Kubespray/Ansible to run the playbooks.
-This key is installed into the bare metal hosts during the Terraform deployment.
-You can generate a key new key or use an existing one.
-
-```bash
-ssh-keygen -f ~/.ssh/id_rsa
-```
-
-## Install Terraform
-
-Terraform is required to deploy the bare metal infrastructure. The steps below are for installing on CentOS 7.
-[More terraform installation options are available.](https://learn.hashicorp.com/terraform/getting-started/install.html)
-
-Grab the latest version of Terraform and install it.
-
-```bash
-echo "https://releases.hashicorp.com/terraform/$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version')/terraform_$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version')_linux_amd64.zip"
-sudo yum install unzip
-sudo unzip terraform_0.14.10_linux_amd64.zip -d /usr/local/bin/
-```
-
-## Download Kubespray
-
-Pull over Kubespray and setup any required libraries.
-
-```bash
-git clone https://github.com/kubernetes-sigs/kubespray
-cd kubespray
-```
-
-## Install Ansible
-
-Install Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible)
-
-## Cluster Definition
-
-In this example, a new cluster called "alpha" will be created.
-
-```bash
-cp -LRp contrib/terraform/packet/sample-inventory inventory/alpha
-cd inventory/alpha/
-ln -s ../../contrib/terraform/packet/hosts
-```
-
-Details about the cluster, such as the name, as well as the authentication tokens and project ID
-for Equinix Metal need to be defined. To find these values see [Equinix Metal API Accounts](https://metal.equinix.com/developers/docs/accounts/).
-
-```bash
-vi cluster.tfvars
-```
-
-* cluster_name = alpha
-* packet_project_id = ABCDEFGHIJKLMNOPQRSTUVWXYZ123456
-* public_key_path = 12345678-90AB-CDEF-GHIJ-KLMNOPQRSTUV
-
-## Deploy Bare Metal Hosts
-
-Initializing Terraform will pull down any necessary plugins/providers.
-
-```bash
-terraform init ../../contrib/terraform/packet/
-```
-
-Run Terraform to deploy the hardware.
-
-```bash
-terraform apply -var-file=cluster.tfvars ../../contrib/terraform/packet
-```
-
-## Run Kubespray Playbooks
-
-With the bare metal infrastructure deployed, Kubespray can now install Kubernetes and setup the cluster.
-
-```bash
-ansible-playbook --become -i inventory/alpha/hosts cluster.yml
-```

docs/developers/ci-setup.md

@@ -2,19 +2,14 @@
 
 ## Pipeline
 
-1. build: build a docker image to be used in the pipeline
-2. unit-tests: fast jobs for fast feedback (linting, etc...)
-3. deploy-part1: small number of jobs to test if the PR works with default settings
-4. deploy-extended: slow jobs testing different platforms, OS, settings, CNI, etc...
-5. deploy-extended: very slow jobs (upgrades, etc...)
+See [.gitlab-ci.yml](/.gitlab-ci.yml) and the included files for an overview.
 
 ## Runners
 
-Kubespray has 3 types of GitLab runners:
+Kubespray has 2 types of GitLab runners, both deployed  on the Kubespray CI cluster (hosted on Oracle Cloud Infrastucture):
 
-- packet runners: used for E2E jobs (usually long), running on Equinix Metal (ex-packet), on kubevirt managed VMs
-- light runners: used for short lived jobs, running on Equinix Metal (ex-packet), as managed pods
-- auto scaling runners (managed via docker-machine on Equinix Metal): used for on-demand resources, see [GitLab docs](https://docs.gitlab.com/runner/configuration/autoscale.html) for more info
+- pods: use the [gitlab-ci kubernetes executor](https://docs.gitlab.com/runner/executors/kubernetes/)
+- vagrant: custom executor running in pods with access to the libvirt socket on the nodes
 
 ## Vagrant
 
@@ -22,18 +17,17 @@ Vagrant jobs are using the [quay.io/kubespray/vagrant](/test-infra/vagrant-docke
 
 ## CI Variables
 
-In CI we have a set of overrides we use to ensure greater success of our CI jobs and avoid throttling by various APIs we depend on. See:
+In CI we have a [set of extra vars](/test/common_vars.yml) we use to ensure greater success of our CI jobs and avoid throttling by various APIs we depend on.
 
-- [Docker mirrors](/tests/common/_docker_hub_registry_mirror.yml)
-- [Test settings](/tests/common/_kubespray_test_settings.yml)
+## CI clusters
 
-## CI Environment
+DISCLAIMER: The following information is not fully up to date, in particular, the CI cluster is now on Oracle Cloud Infrastcture, not Equinix.
 
-The CI packet and light runners are deployed on a kubernetes cluster on Equinix Metal. The cluster is deployed with kubespray itself and maintained by the kubespray maintainers.
+The cluster is deployed with kubespray itself and maintained by the kubespray maintainers.
 
 The following files are used for that inventory:
 
-### cluster.tfvars
+### cluster.tfvars (OBSOLETE: this section is no longer accurate)
 
 ```ini
 # your Kubernetes cluster name here
@@ -166,18 +160,6 @@ kube_feature_gates:
 
 This section documents additional files used to complete a deployment of the kubespray CI, these files sit on the control-plane node and assume a working kubernetes cluster.
 
-### /root/nscleanup.sh
-
-```bash
-#!/bin/bash
-
-kubectl=/usr/local/bin/kubectl
-
-$kubectl get ns | grep -P "(\d.+-\d.+)" | awk 'match($3,/[0-9]+d/) {print $1}' | xargs -r $kubectl delete ns
-$kubectl get ns | grep -P "(\d.+-\d.+)" | awk 'match($3,/[3-9]+h/) {print $1}' | xargs -r $kubectl delete ns
-$kubectl get ns | grep Terminating | awk '{print $1}' | xargs -i $kubectl delete vmi/instance-1 vmi/instance-0 vmi/instance-2 -n {} --force --grace-period=0 &
-```
-
 ### /root/path-calico.sh
 
 ```bash

docs/developers/ci.md

@@ -6,55 +6,52 @@ To generate this Matrix run `./tests/scripts/md-table/main.py`
 
 | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |
 |---| --- | --- | --- | --- | --- | --- | --- |
-almalinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 almalinux9 |  :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
 amazon |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: |
 debian12 |  :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: |
+debian13 |  :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
 fedora40 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+flatcar4081 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-opensuse15 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-rockylinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
-ubuntu20 |  :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: |
+ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu22 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu24 |  :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: |
 
 ## crio
 
 | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |
 |---| --- | --- | --- | --- | --- | --- | --- |
-almalinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 almalinux9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian13 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora40 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora40 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+flatcar4081 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-opensuse15 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu22 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu20 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu22 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 
 ## docker
 
 | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |
 |---| --- | --- | --- | --- | --- | --- | --- |
-almalinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 almalinux9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian13 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora40 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+flatcar4081 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-opensuse15 |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
-rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu20 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu22 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |

docs/external_storage_provisioners/scheduler_plugins.md

@@ -7,7 +7,7 @@ The kube-scheduler binary includes a list of plugins:
 - [CapacityScheduling](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/capacityscheduling) [Beta]
 - [CoScheduling](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/coscheduling) [Beta]
 - [NodeResources](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/noderesources) [Beta]
-- [NodeResouceTopology](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/noderesourcetopology/README.md) [Beta]
+- [NodeResourceTopology](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/noderesourcetopology/README.md) [Beta]
 - [PreemptionToleration](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/preemptiontoleration/README.md) [Alpha]
 - [Trimaran](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/trimaran/README.md) [Alpha]
 - [NetworkAware](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/networkaware/README.md) [Sample]

docs/getting_started/setting-up-your-first-cluster.md

@@ -61,12 +61,12 @@ gcloud compute networks subnets create kubernetes \
 #### Firewall Rules
 
 Create a firewall rule that allows internal communication across all protocols.
-It is important to note that the vxlan protocol has to be allowed in order for
+It is important to note that the vxlan (udp) protocol has to be allowed in order for
 the calico (see later) networking plugin to work.
 
 ```ShellSession
 gcloud compute firewall-rules create kubernetes-the-kubespray-way-allow-internal \
-  --allow tcp,udp,icmp,vxlan \
+  --allow tcp,udp,icmp \
   --network kubernetes-the-kubespray-way \
   --source-ranges 10.240.0.0/24
 ```
@@ -88,7 +88,7 @@ cluster.
 
 ### Compute Instances
 
-The compute instances in this lab will be provisioned using [Ubuntu Server](https://www.ubuntu.com/server) 18.04.
+The compute instances in this lab will be provisioned using [Ubuntu Server](https://www.ubuntu.com/server) 24.04.
 Each compute instance will be provisioned with a fixed private IP address and
  a public IP address (that can be fixed - see [guide](https://cloud.google.com/compute/docs/ip-addresses/reserve-static-external-ip-address)).
 Using fixed public IP addresses has the advantage that our cluster node
@@ -103,7 +103,7 @@ for i in 0 1 2; do
     --async \
     --boot-disk-size 200GB \
     --can-ip-forward \
-    --image-family ubuntu-1804-lts \
+    --image-family ubuntu-2404-lts-amd64 \
     --image-project ubuntu-os-cloud \
     --machine-type e2-standard-2 \
     --private-network-ip 10.240.0.1${i} \
@@ -124,7 +124,7 @@ for i in 0 1 2; do
     --async \
     --boot-disk-size 200GB \
     --can-ip-forward \
-    --image-family ubuntu-1804-lts \
+    --image-family ubuntu-2404-lts-amd64 \
     --image-project ubuntu-os-cloud \
     --machine-type e2-standard-2 \
     --private-network-ip 10.240.0.2${i} \

docs/ingress/ingress_nginx.md

@@ -35,7 +35,7 @@ kubectl create clusterrolebinding cluster-admin-binding \
 The following **Mandatory Command** is required for all deployments except for AWS. See below for the AWS version.
 
 ```console
-kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.0/deploy/static/provider/cloud/deploy.yaml
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.13.3/deploy/static/provider/cloud/deploy.yaml
 ```
 
 ### Provider Specific Steps

docs/operating_systems/bootstrap-os.md

@@ -1,4 +1,4 @@
-# bootstrap-os
+# bootstrap_os
 
 Bootstrap an Ansible host to be able to run Ansible modules.
 
@@ -48,8 +48,8 @@ Remember to disable fact gathering since Python might not be present on hosts.
 - hosts: all
   gather_facts: false  # not all hosts might be able to run modules yet
   roles:
-    - kubespray-defaults
-    - bootstrap-os
+    - kubespray_defaults
+    - bootstrap_os
 ```
 
 ## License

docs/operations/cgroups.md

@@ -30,11 +30,6 @@ kube_memory_reserved: 256Mi
 kube_cpu_reserved: 100m
 # kube_ephemeral_storage_reserved: 2Gi
 # kube_pid_reserved: "1000"
-# Reservation for master hosts
-kube_master_memory_reserved: 512Mi
-kube_master_cpu_reserved: 200m
-# kube_master_ephemeral_storage_reserved: 2Gi
-# kube_master_pid_reserved: "1000"
 
 # Set to true to reserve resources for system daemons
 system_reserved: true
@@ -44,11 +39,6 @@ system_memory_reserved: 512Mi
 system_cpu_reserved: 500m
 # system_ephemeral_storage_reserved: 2Gi
 # system_pid_reserved: "1000"
-# Reservation for master hosts
-system_master_memory_reserved: 256Mi
-system_master_cpu_reserved: 250m
-# system_master_ephemeral_storage_reserved: 2Gi
-# system_master_pid_reserved: "1000"
 ```
 
 After the setup, the cgroups hierarchy is as follows:

docs/operations/hardening.md

@@ -18,8 +18,6 @@ The **kubernetes** version should be at least `v1.23.6` to have all the most rec
 
 ## kube-apiserver
 authorization_modes: ['Node', 'RBAC']
-# AppArmor-based OS
-# kube_apiserver_feature_gates: ['AppArmor=true']
 kube_apiserver_request_timeout: 120s
 kube_apiserver_service_account_lookup: true
 
@@ -77,17 +75,17 @@ remove_anonymous_access: true
 ## kube-controller-manager
 kube_controller_manager_bind_address: 127.0.0.1
 kube_controller_terminated_pod_gc_threshold: 50
-# AppArmor-based OS
-# kube_controller_feature_gates: ["RotateKubeletServerCertificate=true", "AppArmor=true"]
 kube_controller_feature_gates: ["RotateKubeletServerCertificate=true"]
 
 ## kube-scheduler
 kube_scheduler_bind_address: 127.0.0.1
-# AppArmor-based OS
-# kube_scheduler_feature_gates: ["AppArmor=true"]
 
 ## etcd
-etcd_deployment_type: kubeadm
+# Running etcd (on dedicated hosts) outside the Kubernetes cluster is the most secure deployment option,
+# as it isolates etcd from the cluster's CNI network and removes direct pod-level attack vectors.
+# This approach prevents RBAC misconfigurations that potentially compromise etcd,
+# creating an additional security boundary that protects the cluster's critical state store.
+etcd_deployment_type: host
 
 ## kubelet
 kubelet_authorization_mode_webhook: true
@@ -102,6 +100,8 @@ kubelet_make_iptables_util_chains: true
 kubelet_feature_gates: ["RotateKubeletServerCertificate=true"]
 kubelet_seccomp_default: true
 kubelet_systemd_hardening: true
+# To disable kubelet's staticPodPath (for nodes that don't use static pods like worker nodes)
+kubelet_static_pod_path: ""
 # In case you have multiple interfaces in your
 # control plane nodes and you want to specify the right
 # IP addresses, kubelet_secure_addresses allows you
@@ -126,9 +126,8 @@ Let's take a deep look to the resultant **kubernetes** configuration:
 * The `encryption-provider-config` provide encryption at rest. This means that the `kube-apiserver` encrypt data that is going to be stored before they reach `etcd`. So the data is completely unreadable from `etcd` (in case an attacker is able to exploit this).
 * The `rotateCertificates` in `KubeletConfiguration` is set to `true` along with `serverTLSBootstrap`. This could be used in alternative to `tlsCertFile` and `tlsPrivateKeyFile` parameters. Additionally it automatically generates certificates by itself. By default the CSRs are approved automatically via [kubelet-csr-approver](https://github.com/postfinance/kubelet-csr-approver). You can customize approval configuration by modifying Helm values via `kubelet_csr_approver_values`.
   See <https://kubernetes.io/docs/reference/access-authn-authz/kubelet-tls-bootstrapping/> for more information on the subject.
-* If you are installing **kubernetes** in an AppArmor-based OS (eg. Debian/Ubuntu) you can enable the `AppArmor` feature gate uncommenting the lines with the comment `# AppArmor-based OS` on top.
 * The `kubelet_systemd_hardening`, both with `kubelet_secure_addresses` setup a minimal firewall on the system. To better understand how these variables work, here's an explanatory image:
-  ![kubelet hardening](img/kubelet-hardening.png)
+  ![kubelet hardening](../img/kubelet-hardening.png)
 
 Once you have the file properly filled, you can run the **Ansible** command to start the installation:
 

docs/operations/nodes.md

@@ -2,58 +2,6 @@
 
 Modified from [comments in #3471](https://github.com/kubernetes-sigs/kubespray/issues/3471#issuecomment-530036084)
 
-## Limitation: Removal of first kube_control_plane and etcd-master
-
-Currently you can't remove the first node in your kube_control_plane and etcd-master list. If you still want to remove this node you have to:
-
-### 1) Change order of current control planes
-
-Modify the order of your control plane list by pushing your first entry to any other position. E.g. if you want to remove `node-1` of the following example:
-
-```yaml
-  children:
-    kube_control_plane:
-      hosts:
-        node-1:
-        node-2:
-        node-3:
-    kube_node:
-      hosts:
-        node-1:
-        node-2:
-        node-3:
-    etcd:
-      hosts:
-        node-1:
-        node-2:
-        node-3:
-```
-
-change your inventory to:
-
-```yaml
-  children:
-    kube_control_plane:
-      hosts:
-        node-2:
-        node-3:
-        node-1:
-    kube_node:
-      hosts:
-        node-2:
-        node-3:
-        node-1:
-    etcd:
-      hosts:
-        node-2:
-        node-3:
-        node-1:
-```
-
-## 2) Upgrade the cluster
-
-run `upgrade-cluster.yml` or `cluster.yml`. Now you are good to go on with the removal.
-
 ## Adding/replacing a worker node
 
 This should be the easiest.
@@ -100,40 +48,74 @@ crictl ps | grep nginx-proxy | awk '{print $1}' | xargs crictl stop
 With the old node still in the inventory, run `remove-node.yml`. You need to pass `-e node=NODE_NAME` to the playbook to limit the execution to the node being removed.
 If the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars.
 
-## Replacing a first control plane node
+## Adding/Removal of first `kube_control_plane` and etcd-master
 
-### 1) Change control plane nodes order in inventory
+Currently you can't remove the first node in your `kube_control_plane` and etcd-master list. If you still want to remove this node you have to:
 
-from
+### 1) Change order of current control planes
 
-```ini
-[kube_control_plane]
- node-1
- node-2
- node-3
+Modify the order of your control plane list by pushing your first entry to any other position. E.g. if you want to remove `node-1` of the following example:
+
+```yaml
+all:
+  hosts:
+  children:
+    kube_control_plane:
+      hosts:
+        node-1:
+        node-2:
+        node-3:
+    kube_node:
+      hosts:
+        node-1:
+        node-2:
+        node-3:
+    etcd:
+      hosts:
+        node-1:
+        node-2:
+        node-3:
 ```
 
-to
+change your inventory to:
 
-```ini
-[kube_control_plane]
- node-2
- node-3
- node-1
+```yaml
+all:
+  hosts:
+  children:
+    kube_control_plane:
+      hosts:
+        node-2:
+        node-3:
+        node-1:
+    kube_node:
+      hosts:
+        node-2:
+        node-3:
+        node-1:
+    etcd:
+      hosts:
+        node-2:
+        node-3:
+        node-1:
 ```
 
-### 2) Remove old first control plane node from cluster
+### 2) Upgrade the cluster
+
+run `upgrade-cluster.yml` or `cluster.yml`. Now you are good to go on with the removal.
+
+### 3) Remove old first control plane node from cluster
 
 With the old node still in the inventory, run `remove-node.yml`. You need to pass `-e node=node-1` to the playbook to limit the execution to the node being removed.
 If the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars.
 
-### 3) Edit cluster-info configmap in kube-public namespace
+### 4) Edit cluster-info configmap in kube-public namespace
 
 `kubectl  edit cm -n kube-public cluster-info`
 
 Change ip of old kube_control_plane node with ip of live kube_control_plane node (`server` field). Also, update `certificate-authority-data` field if you changed certs.
 
-### 4) Add new control plane node
+### 5) Add new control plane node
 
 Update inventory (if needed)
 

docs/operations/offline-environment.md

@@ -75,17 +75,17 @@ quay_image_repo: "{{ registry_host }}"
 github_image_repo: "{{ registry_host }}"
 
 local_path_provisioner_helper_image_repo: "{{ registry_host }}/busybox"
-kubeadm_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubeadm"
-kubectl_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubectl"
-kubelet_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubelet"
+kubeadm_download_url: "{{ files_repo }}/kubernetes/v{{ kube_version }}/kubeadm"
+kubectl_download_url: "{{ files_repo }}/kubernetes/v{{ kube_version }}/kubectl"
+kubelet_download_url: "{{ files_repo }}/kubernetes/v{{ kube_version }}/kubelet"
 # etcd is optional if you **DON'T** use etcd_deployment=host
-etcd_download_url: "{{ files_repo }}/kubernetes/etcd/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
-cni_download_url: "{{ files_repo }}/kubernetes/cni/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
-crictl_download_url: "{{ files_repo }}/kubernetes/cri-tools/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
+etcd_download_url: "{{ files_repo }}/kubernetes/etcd/etcd-v{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
+cni_download_url: "{{ files_repo }}/kubernetes/cni/cni-plugins-linux-{{ image_arch }}-v{{ cni_version }}.tgz"
+crictl_download_url: "{{ files_repo }}/kubernetes/cri-tools/crictl-v{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
 # If using Calico
-calicoctl_download_url: "{{ files_repo }}/kubernetes/calico/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
+calicoctl_download_url: "{{ files_repo }}/kubernetes/calico/v{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
 # If using Calico with kdd
-calico_crds_download_url: "{{ files_repo }}/kubernetes/calico/{{ calico_version }}.tar.gz"
+calico_crds_download_url: "{{ files_repo }}/kubernetes/calico/v{{ calico_version }}.tar.gz"
 # Containerd
 containerd_download_url: "{{ files_repo }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"
 runc_download_url: "{{ files_repo }}/runc.{{ image_arch }}"
@@ -136,7 +136,7 @@ If you use the settings like the one above, you'll need to define in your invent
 
 * `registry_host`: Container image registry. If you _don't_ use the same repository path for the container images that
   the ones defined
-  in [kubesprays-defaults's role defaults](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/kubespray-defaults/defaults/main/download.yml)
+  in [kubesprays-defaults's role defaults](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/kubespray_defaults/defaults/main/download.yml)
   , you need to override the `*_image_repo` for these container images. If you want to make your life easier, use the
   same repository path, you won't have to override anything else.
 * `registry_addr`: Container image registry, but only have [domain or ip]:[port].

docs/operations/upgrades.md

@@ -13,9 +13,7 @@ versions. Here are all version vars for each component:
 * etcd_version
 * calico_version
 * calico_cni_version
-* weave_version
 * flannel_version
-* kubedns_version
 
 > **Warning**
 > [Attempting to upgrade from an older release straight to the latest release is unsupported and likely to break something](https://github.com/kubernetes-sigs/kubespray/issues/3849#issuecomment-451386515)
@@ -84,7 +82,7 @@ If you don't want to upgrade all nodes in one run, you can use `--limit` [patter
 Before using `--limit` run playbook `facts.yml` without the limit to refresh facts cache for all nodes:
 
 ```ShellSession
-ansible-playbook facts.yml -b -i inventory/sample/hosts.ini
+ansible-playbook playbooks/facts.yml -b -i inventory/sample/hosts.ini
 ```
 
 After this upgrade control plane and etcd groups [#5147](https://github.com/kubernetes-sigs/kubespray/issues/5147):
@@ -357,7 +355,7 @@ follows:
 * Containerd
 * etcd
 * kubelet and kube-proxy
-* network_plugin (such as Calico or Weave)
+* network_plugin (such as Calico)
 * kube-apiserver, kube-scheduler, and kube-controller-manager
 * Add-ons (such as KubeDNS)
 

extra_playbooks/migrate_openstack_provider.yml

@@ -12,7 +12,7 @@
   hosts: kube_control_plane[0]
   tasks:
     - name: Include kubespray-default variables
-      include_vars: ../roles/kubespray-defaults/defaults/main/main.yml
+      include_vars: ../roles/kubespray_defaults/defaults/main/main.yml
     - name: Copy get_cinder_pvs.sh to first control plane node
       copy:
         src: get_cinder_pvs.sh

extra_playbooks/upgrade-only-k8s.yml

@@ -14,7 +14,7 @@
   hosts: localhost
   gather_facts: false
   roles:
-    - { role: kubespray-defaults}
+    - { role: kubespray_defaults}
     - { role: bastion-ssh-config, tags: ["localhost", "bastion"]}
 
 - name: Bootstrap hosts OS for Ansible
@@ -22,18 +22,18 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   gather_facts: false
   vars:
-    # Need to disable pipelining for bootstrap-os as some systems have requiretty in sudoers set, which makes pipelining
-    # fail. bootstrap-os fixes this on these systems, so in later plays it can be enabled.
+    # Need to disable pipelining for bootstrap_os as some systems have requiretty in sudoers set, which makes pipelining
+    # fail. bootstrap_os fixes this on these systems, so in later plays it can be enabled.
     ansible_ssh_pipelining: false
   roles:
-    - { role: kubespray-defaults}
-    - { role: bootstrap-os, tags: bootstrap-os}
+    - { role: kubespray_defaults}
+    - { role: bootstrap_os, tags: bootstrap_os}
 
 - name: Preinstall
   hosts: k8s_cluster:etcd:calico_rr
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
-    - { role: kubespray-defaults}
+    - { role: kubespray_defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
 
 - name: Handle upgrades to control plane components first to maintain backwards compat.
@@ -41,7 +41,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   serial: 1
   roles:
-    - { role: kubespray-defaults}
+    - { role: kubespray_defaults}
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
     - { role: kubernetes/node, tags: node }
     - { role: kubernetes/control-plane, tags: master, upgrade_cluster_setup: true }
@@ -54,8 +54,8 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   serial: "{{ serial | default('20%') }}"
   roles:
-    - { role: kubespray-defaults}
+    - { role: kubespray_defaults}
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
     - { role: kubernetes/node, tags: node }
     - { role: upgrade/post-upgrade, tags: post-upgrade }
-    - { role: kubespray-defaults}
+    - { role: kubespray_defaults}

galaxy.yml

@@ -2,7 +2,7 @@
 namespace: kubernetes_sigs
 description: Deploy a production ready Kubernetes cluster
 name: kubespray
-version: 2.28.0
+version: 2.29.2
 readme: README.md
 authors:
   - The Kubespray maintainers (https://kubernetes.slack.com/channels/kubespray)

inventory/sample/group_vars/all/all.yml

@@ -57,7 +57,7 @@ loadbalancer_apiserver_healthcheck_port: 8081
 # https_proxy: ""
 # https_proxy_cert_file: ""
 
-## Refer to roles/kubespray-defaults/defaults/main/main.yml before modifying no_proxy
+## Refer to roles/kubespray_defaults/defaults/main/main.yml before modifying no_proxy
 # no_proxy: ""
 
 ## Some problems may occur when downloading files over https proxy due to ansible bug
@@ -115,6 +115,9 @@ no_proxy_exclude_workers: false
 # sysctl_file_path to add sysctl conf to
 # sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf"
 
+# ignore sysctl errors about unknown keys
+# sysctl_ignore_unknown_keys: false
+
 ## Variables for webhook token auth https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication
 kube_webhook_token_auth: false
 kube_webhook_token_auth_url_skip_tls_verify: false

inventory/sample/group_vars/all/containerd.yml

@@ -50,6 +50,8 @@
 #     - host: https://registry-1.docker.io
 #       capabilities: ["pull", "resolve"]
 #       skip_verify: false
+#       header:
+#         Authorization: "Basic XXX"
 
 # containerd_max_container_log_line_size: 16384
 

inventory/sample/group_vars/all/oci.yml

@@ -43,7 +43,6 @@
 #   ocid1.subnet.oc1.phx.aaaaaaaahuxrgvs65iwdz7ekwgg3l5gyah7ww5klkwjcso74u3e4i64hvtvq: ocid1.securitylist.oc1.iad.aaaaaaaaqti5jsfvyw6ejahh7r4okb2xbtuiuguswhs746mtahn72r7adt7q
 ## If oci_use_instance_principals is true, you do not need to set the region, tenancy, user, key, passphrase, or fingerprint
 # oci_use_instance_principals: false
-# oci_cloud_controller_version: 0.6.0
 ## If you would like to control OCI query rate limits for the controller
 # oci_rate_limit:
 #   rate_limit_qps_read:

inventory/sample/group_vars/all/offline.yml

@@ -18,9 +18,9 @@
 # quay_image_repo: "{{ registry_host }}"
 
 ## Kubernetes components
-# kubeadm_download_url: "{{ files_repo }}/dl.k8s.io/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubeadm"
-# kubectl_download_url: "{{ files_repo }}/dl.k8s.io/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubectl"
-# kubelet_download_url: "{{ files_repo }}/dl.k8s.io/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubelet"
+# kubeadm_download_url: "{{ files_repo }}/dl.k8s.io/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubeadm"
+# kubectl_download_url: "{{ files_repo }}/dl.k8s.io/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubectl"
+# kubelet_download_url: "{{ files_repo }}/dl.k8s.io/release/v{{ kube_version }}/bin/linux/{{ image_arch }}/kubelet"
 
 
 ## Two options - Override entire repository or override only a single binary.
@@ -33,24 +33,24 @@
 
 ## [Optional] 2 - Override a specific binary
 ## CNI Plugins
-# cni_download_url: "{{ files_repo }}/github.com/containernetworking/plugins/releases/download/{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
+# cni_download_url: "{{ files_repo }}/github.com/containernetworking/plugins/releases/download/v{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-v{{ cni_version }}.tgz"
 
 ## cri-tools
-# crictl_download_url: "{{ files_repo }}/github.com/kubernetes-sigs/cri-tools/releases/download/{{ crictl_version }}/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
+# crictl_download_url: "{{ files_repo }}/github.com/kubernetes-sigs/cri-tools/releases/download/v{{ crictl_version }}/crictl-v{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
 
 ## [Optional] etcd: only if you use etcd_deployment=host
-# etcd_download_url: "{{ files_repo }}/github.com/etcd-io/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
+# etcd_download_url: "{{ files_repo }}/github.com/etcd-io/etcd/releases/download/v{{ etcd_version }}/etcd-v{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
 
 # [Optional] Calico: If using Calico network plugin
-# calicoctl_download_url: "{{ files_repo }}/github.com/projectcalico/calico/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
+# calicoctl_download_url: "{{ files_repo }}/github.com/projectcalico/calico/releases/download/v{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
 # [Optional] Calico with kdd: If using Calico network plugin with kdd datastore
-# calico_crds_download_url: "{{ files_repo }}/github.com/projectcalico/calico/archive/{{ calico_version }}.tar.gz"
+# calico_crds_download_url: "{{ files_repo }}/github.com/projectcalico/calico/archive/v{{ calico_version }}.tar.gz"
 
 # [Optional] Cilium: If using Cilium network plugin
-# ciliumcli_download_url: "{{ files_repo }}/github.com/cilium/cilium-cli/releases/download/{{ cilium_cli_version }}/cilium-linux-{{ image_arch }}.tar.gz"
+# ciliumcli_download_url: "{{ files_repo }}/github.com/cilium/cilium-cli/releases/download/v{{ cilium_cli_version }}/cilium-linux-{{ image_arch }}.tar.gz"
 
 # [Optional] helm: only if you set helm_enabled: true
-# helm_download_url: "{{ files_repo }}/get.helm.sh/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz"
+# helm_download_url: "{{ files_repo }}/get.helm.sh/helm-v{{ helm_version }}-linux-{{ image_arch }}.tar.gz"
 
 # [Optional] crun: only if you set crun_enabled: true
 # crun_download_url: "{{ files_repo }}/github.com/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}"
@@ -62,13 +62,13 @@
 # cri_dockerd_download_url: "{{ files_repo }}/github.com/Mirantis/cri-dockerd/releases/download/v{{ cri_dockerd_version }}/cri-dockerd-{{ cri_dockerd_version }}.{{ image_arch }}.tgz"
 
 # [Optional] runc: if you set container_manager to containerd or crio
-# runc_download_url: "{{ files_repo }}/github.com/opencontainers/runc/releases/download/{{ runc_version }}/runc.{{ image_arch }}"
+# runc_download_url: "{{ files_repo }}/github.com/opencontainers/runc/releases/download/v{{ runc_version }}/runc.{{ image_arch }}"
 
 # [Optional] cri-o: only if you set container_manager: crio
 # crio_download_base: "download.opensuse.org/repositories/devel:kubic:libcontainers:stable"
 # crio_download_crio: "http://{{ crio_download_base }}:/cri-o:/"
-# crio_download_url: "{{ files_repo }}/storage.googleapis.com/cri-o/artifacts/cri-o.{{ image_arch }}.{{ crio_version }}.tar.gz"
-# skopeo_download_url: "{{ files_repo }}/github.com/lework/skopeo-binary/releases/download/{{ skopeo_version }}/skopeo-linux-{{ image_arch }}"
+# crio_download_url: "{{ files_repo }}/storage.googleapis.com/cri-o/artifacts/cri-o.{{ image_arch }}.v{{ crio_version }}.tar.gz"
+# skopeo_download_url: "{{ files_repo }}/github.com/lework/skopeo-binary/releases/download/v{{ skopeo_version }}/skopeo-linux-{{ image_arch }}"
 
 # [Optional] containerd: only if you set container_runtime: containerd
 # containerd_download_url: "{{ files_repo }}/github.com/containerd/containerd/releases/download/v{{ containerd_version }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"

inventory/sample/group_vars/all/openstack.yml

@@ -1,5 +1,4 @@
 ## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461)
-# openstack_blockstorage_version: "v1/v2/auto (default)"
 # openstack_blockstorage_ignore_volume_az: yes
 ## When OpenStack is used, if LBaaSv2 is available you can enable it with the following 2 variables.
 # openstack_lbaas_enabled: True

inventory/sample/group_vars/all/vsphere.yml

@@ -7,26 +7,6 @@
 # external_vsphere_datacenter: "DATACENTER_name"
 # external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id"
 
-## Vsphere version where located VMs
-# external_vsphere_version: "6.7u3"
-
-## Tags for the external vSphere Cloud Provider images
-## registry.k8s.io/cloud-pv-vsphere/cloud-provider-vsphere
-# external_vsphere_cloud_controller_image_tag: "v1.31.0"
-## registry.k8s.io/csi-vsphere/syncer
-# vsphere_syncer_image_tag: "v3.3.1"
-## registry.k8s.io/sig-storage/csi-attacher
-# vsphere_csi_attacher_image_tag: "v3.4.0"
-## registry.k8s.io/csi-vsphere/driver
-# vsphere_csi_controller: "v3.3.1"
-## registry.k8s.io/sig-storage/livenessprobe
-# vsphere_csi_liveness_probe_image_tag: "v2.6.0"
-## registry.k8s.io/sig-storage/csi-provisioner
-# vsphere_csi_provisioner_image_tag: "v3.1.0"
-## registry.k8s.io/sig-storage/csi-resizer
-## makes sense only for vSphere version >=7.0
-# vsphere_csi_resizer_tag: "v1.3.0"
-
 ## To use vSphere CSI plugin to provision volumes set this value to true
 # vsphere_csi_enabled: true
 # vsphere_csi_controller_replicas: 1

inventory/sample/group_vars/etcd.yml

@@ -1,38 +0,0 @@
----
-## Etcd auto compaction retention for mvcc key value store in hour
-# etcd_compaction_retention: 0
-
-## Set level of detail for etcd exported metrics, specify 'extensive' to include histogram metrics.
-# etcd_metrics: basic
-
-## Etcd is restricted by default to 512M on systems under 4GB RAM, 512MB is not enough for much more than testing.
-## Set this if your etcd nodes have less than 4GB but you want more RAM for etcd. Set to 0 for unrestricted RAM.
-## This value is only relevant when deploying etcd with `etcd_deployment_type: docker`
-# etcd_memory_limit: "512M"
-
-## Etcd has a default of 2G for its space quota. If you put a value in etcd_memory_limit which is less than
-## etcd_quota_backend_bytes, you may encounter out of memory terminations of the etcd cluster. Please check
-## etcd documentation for more information.
-# 8G is a suggested maximum size for normal environments and etcd warns at startup if the configured value exceeds it.
-# etcd_quota_backend_bytes: "2147483648"
-
-# Maximum client request size in bytes the server will accept.
-# etcd is designed to handle small key value pairs typical for metadata.
-# Larger requests will work, but may increase the latency of other requests
-# etcd_max_request_bytes: "1572864"
-
-### ETCD: disable peer client cert authentication.
-# This affects ETCD_PEER_CLIENT_CERT_AUTH variable
-# etcd_peer_client_auth: true
-
-## Enable distributed tracing
-## To enable this experimental feature, set the etcd_experimental_enable_distributed_tracing: true, along with the
-## etcd_experimental_distributed_tracing_sample_rate to choose how many samples to collect per million spans,
-## the default sampling rate is 0 https://etcd.io/docs/v3.5/op-guide/monitoring/#distributed-tracing
-# etcd_experimental_enable_distributed_tracing: false
-# etcd_experimental_distributed_tracing_sample_rate: 100
-# etcd_experimental_distributed_tracing_address: "localhost:4317"
-# etcd_experimental_distributed_tracing_service_name: etcd
-
-## The interval for etcd watch progress notify events
-# etcd_experimental_watch_progress_notify_interval: 5s

inventory/sample/group_vars/k8s_cluster/addons.yml

@@ -29,7 +29,6 @@ local_path_provisioner_enabled: false
 # local_path_provisioner_claim_root: /opt/local-path-provisioner/
 # local_path_provisioner_debug: false
 # local_path_provisioner_image_repo: "{{ docker_image_repo }}/rancher/local-path-provisioner"
-# local_path_provisioner_image_tag: "v0.0.24"
 # local_path_provisioner_helper_image_repo: "busybox"
 # local_path_provisioner_helper_image_tag: "latest"
 
@@ -67,7 +66,6 @@ local_volume_provisioner_enabled: false
 
 # Gateway API CRDs
 gateway_api_enabled: false
-# gateway_api_experimental_channel: false
 
 # Nginx ingress controller deployment
 ingress_nginx_enabled: false
@@ -149,7 +147,6 @@ cert_manager_enabled: false
 metallb_enabled: false
 metallb_speaker_enabled: "{{ metallb_enabled }}"
 metallb_namespace: "metallb-system"
-# metallb_version: 0.13.9
 # metallb_protocol: "layer2"
 # metallb_port: "7472"
 # metallb_memberlist_port: "7946"
@@ -211,7 +208,6 @@ metallb_namespace: "metallb-system"
 #             - pool2
 
 argocd_enabled: false
-# argocd_version: 2.14.5
 # argocd_namespace: argocd
 # Default password:
 #   - https://argo-cd.readthedocs.io/en/stable/getting_started/#4-login-using-the-cli
@@ -239,6 +235,7 @@ kube_vip_enabled: false
 # kube_vip_cp_detect: false
 # kube_vip_leasename: plndr-cp-lock
 # kube_vip_enable_node_labeling: false
+# kube_vip_lb_fwdmethod: local
 
 # Node Feature Discovery
 node_feature_discovery_enabled: false

inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml

@@ -16,9 +16,6 @@ kube_token_dir: "{{ kube_config_dir }}/tokens"
 
 kube_api_anonymous_auth: true
 
-## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: 1.32.2
-
 # Where the binaries will be downloaded.
 # Note: ensure that you've enough disk space (about 1G)
 local_release_dir: "/tmp/releases"
@@ -65,7 +62,7 @@ credentials_dir: "{{ inventory_dir }}/credentials"
 # kube_webhook_authorization_url: https://...
 # kube_webhook_authorization_url_skip_tls_verify: false
 
-# Choose network plugin (cilium, calico, kube-ovn, weave or flannel. Use cni for generic cni plugin)
+# Choose network plugin (cilium, calico, kube-ovn or flannel. Use cni for generic cni plugin)
 # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
 kube_network_plugin: calico
 
@@ -349,7 +346,7 @@ event_ttl_duration: "1h0m0s"
 ## Automatically renew K8S control plane certificates on first Monday of each month
 auto_renew_certificates: false
 # First Monday of each month
-# auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:{{ groups['kube_control_plane'].index(inventory_hostname) }}0:00"
+# auto_renew_certificates_systemd_calendar: "Mon *-*-1,2,3,4,5,6,7 03:00:00"
 
 kubeadm_patches_dir: "{{ kube_config_dir }}/patches"
 kubeadm_patches: []

inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml

@@ -25,15 +25,9 @@ calico_pool_blocksize: 26
 # add default ippool CIDR (must be inside kube_pods_subnet, defaults to kube_pods_subnet otherwise)
 # calico_pool_cidr: 1.2.3.4/5
 
-# add default ippool CIDR to CNI config
-# calico_cni_pool: true
-
 # Add default IPV6 IPPool CIDR. Must be inside kube_pods_subnet_ipv6. Defaults to kube_pods_subnet_ipv6 if not set.
 # calico_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112
 
-# Add default IPV6 IPPool CIDR to CNI config
-# calico_cni_pool_ipv6: true
-
 # Global as_num (/calico/bgp/v1/global/as_num)
 # global_as_num: "64512"
 

inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml

@@ -1,6 +1,4 @@
 ---
-# cilium_version: "1.15.9"
-
 # Log-level
 # cilium_debug: false
 
@@ -58,8 +56,8 @@ cilium_l2announcements: false
 #
 # Only effective when monitor aggregation is set to "medium" or higher.
 # cilium_monitor_aggregation_flags: "all"
-# Kube Proxy Replacement mode (strict/partial)
-# cilium_kube_proxy_replacement: partial
+# Kube Proxy Replacement mode (true/false)
+# cilium_kube_proxy_replacement: false
 
 # If upgrading from Cilium < 1.5, you may want to override some of these options
 # to prevent service disruptions. See also:
@@ -177,6 +175,10 @@ cilium_l2announcements: false
 ### Buffer size of the channel to receive monitor events.
 # cilium_hubble_event_queue_size: 50
 
+# Override the DNS suffix that Hubble-Relay uses to resolve its peer service.
+# It defaults to the inventory's `dns_domain`.
+# cilium_hubble_peer_service_cluster_domain: "{{ dns_domain }}"
+
 # IP address management mode for v1.9+.
 # https://docs.cilium.io/en/v1.9/concepts/networking/ipam/
 # cilium_ipam_mode: kubernetes
@@ -255,6 +257,10 @@ cilium_l2announcements: false
 #   - name: "blue-pool"
 #     cidrs:
 #       - "10.0.10.0/24"
+#     ranges:
+#       - start: "20.0.20.100"
+#         stop: "20.0.20.200"
+#       - start: "1.2.3.4"
 
 # -- Configure BGP Instances (New bgpv2 API v1.16+)
 # cilium_bgp_cluster_configs:
@@ -378,3 +384,7 @@ cilium_l2announcements: false
 #       resourceNames:
 #       - toto
 # cilium_clusterrole_rules_operator_extra_vars: []
+
+# Cilium extra values, use any values from cilium Helm Chart
+# ref: https://docs.cilium.io/en/stable/helm-reference/
+# cilium_extra_values: {}

inventory/sample/group_vars/k8s_cluster/k8s-net-custom-cni.yml

@@ -45,7 +45,7 @@
 # custom_cni_chart_repository_name: cilium
 # custom_cni_chart_repository_url: https://helm.cilium.io
 # custom_cni_chart_ref: cilium/cilium
-# custom_cni_chart_version: 1.14.3
+# custom_cni_chart_version: <chart version> (e.g.: 1.14.3)
 # custom_cni_chart_values:
 #   cluster:
 #     name: "cilium-demo"

inventory/sample/group_vars/k8s_cluster/k8s-net-kube-router.yml

@@ -1,11 +1,5 @@
 # See roles/network_plugin/kube-router/defaults/main.yml
 
-# Kube router version
-# Default to v2
-# kube_router_version: "2.0.0"
-# Uncomment to use v1 (Deprecated)
-# kube_router_version: "1.6.0"
-
 # Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP
 # kube_router_run_router: true
 

inventory/sample/group_vars/k8s_cluster/k8s-net-weave.yml

@@ -1,64 +0,0 @@
-# see roles/network_plugin/weave/defaults/main.yml
-
-# Weave's network password for encryption, if null then no network encryption.
-# weave_password: ~
-
-# If set to 1, disable checking for new Weave Net versions (default is blank,
-# i.e. check is enabled)
-# weave_checkpoint_disable: false
-
-# Soft limit on the number of connections between peers. Defaults to 100.
-# weave_conn_limit: 100
-
-# Weave Net defaults to enabling hairpin on the bridge side of the veth pair
-# for containers attached. If you need to disable hairpin, e.g. your kernel is
-# one of those that can panic if hairpin is enabled, then you can disable it by
-# setting `HAIRPIN_MODE=false`.
-# weave_hairpin_mode: true
-
-# The range of IP addresses used by Weave Net and the subnet they are placed in
-# (CIDR format; default 10.32.0.0/12)
-# weave_ipalloc_range: "{{ kube_pods_subnet }}"
-
-# Set to 0 to disable Network Policy Controller (default is on)
-# weave_expect_npc: "{{ enable_network_policy }}"
-
-# List of addresses of peers in the Kubernetes cluster (default is to fetch the
-# list from the api-server)
-# weave_kube_peers: ~
-
-# Set the initialization mode of the IP Address Manager (defaults to consensus
-# amongst the KUBE_PEERS)
-# weave_ipalloc_init: ~
-
-# Set the IP address used as a gateway from the Weave network to the host
-# network - this is useful if you are configuring the addon as a static pod.
-# weave_expose_ip: ~
-
-# Address and port that the Weave Net daemon will serve Prometheus-style
-# metrics on (defaults to 0.0.0.0:6782)
-# weave_metrics_addr: ~
-
-# Address and port that the Weave Net daemon will serve status requests on
-# (defaults to disabled)
-# weave_status_addr: ~
-
-# Weave Net defaults to 1376 bytes, but you can set a smaller size if your
-# underlying network has a tighter limit, or set a larger size for better
-# performance if your network supports jumbo frames (e.g. 8916)
-# weave_mtu: 1376
-
-# Set to 1 to preserve the client source IP address when accessing Service
-# annotated with `service.spec.externalTrafficPolicy=Local`. The feature works
-# only with Weave IPAM (default).
-# weave_no_masq_local: true
-
-# set to nft to use nftables backend for iptables (default is iptables)
-# weave_iptables_backend: iptables
-
-# Extra variables that passing to launch.sh, useful for enabling seed mode, see
-# https://www.weave.works/docs/net/latest/tasks/ipam/ipam/
-# weave_extra_args: ~
-
-# Extra variables for weave_npc that passing to launch.sh, useful for change log level, ex --log-level=error
-# weave_npc_extra_args: ~

meta/runtime.yml

@@ -1,2 +1,2 @@
 ---
-requires_ansible: '>=2.16.4'
+requires_ansible: ">=2.17.3"

pipeline.Dockerfile

@@ -1,4 +1,4 @@
-# Use imutable image tags rather than mutable tags (like ubuntu:22.04)
+# Use immutable image tags rather than mutable tags (like ubuntu:22.04)
 FROM ubuntu:jammy-20230308
 # Some tools like yamllint need this
 # Pip needs this as well at the moment to install ansible
@@ -47,8 +47,8 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 \
     && pip install --no-compile --no-cache-dir pip -U \
     && pip install --no-compile --no-cache-dir -r tests/requirements.txt \
     && pip install --no-compile --no-cache-dir -r requirements.txt \
-    && curl -L https://dl.k8s.io/release/v1.32.3/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
-    && echo $(curl -L https://dl.k8s.io/release/v1.32.3/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
+    && curl -L https://dl.k8s.io/release/v1.33.7/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
+    && echo $(curl -L https://dl.k8s.io/release/v1.33.7/bin/linux/$(dpkg --print-architecture)/kubectl.sha256) /usr/local/bin/kubectl | sha256sum --check \
     && chmod a+x /usr/local/bin/kubectl \
     # Install Vagrant
     && curl -LO https://releases.hashicorp.com/vagrant/${VAGRANT_VERSION}/vagrant_${VAGRANT_VERSION}-1_$(dpkg --print-architecture).deb \

playbooks/ansible_version.yml

@@ -5,8 +5,8 @@
   become: false
   run_once: true
   vars:
-    minimal_ansible_version: 2.16.4
-    maximal_ansible_version: 2.17.0
+    minimal_ansible_version: 2.17.3
+    maximal_ansible_version: 2.18.0
   tags: always
   tasks:
     - name: "Check {{ minimal_ansible_version }} <= Ansible version < {{ maximal_ansible_version }}"

playbooks/boilerplate.yml

@@ -6,34 +6,18 @@
 # - to ensure we keep compatibility with old style group names
 # - to reduce inventory boilerplate (defining parent groups / empty groups)
 
-- name: Define groups for legacy less structured inventories
+- name: Inventory setup and validation
   hosts: all
   gather_facts: false
   tags: always
-  tasks:
-    - name: Match needed groups by their old names or definition
-      vars:
-        group_mappings:
-          kube_control_plane:
-            - kube-master
-          kube_node:
-            - kube-node
-          calico_rr:
-            - calico-rr
-          no_floating:
-            - no-floating
-          k8s_cluster:
-            - kube_node
-            - kube_control_plane
-            - calico_rr
-      group_by:
-        key: "{{ (group_names | intersect(item.value) | length > 0) | ternary(item.key, '_all') }}"
-      loop: "{{ group_mappings | dict2items }}"
+  roles:
+    - dynamic_groups
+    - validate_inventory
 
 - name: Install bastion ssh config
   hosts: bastion[0]
   gather_facts: false
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: bastion-ssh-config, tags: ["localhost", "bastion"] }

playbooks/cluster.yml

@@ -3,7 +3,7 @@
   import_playbook: boilerplate.yml
 
 - name: Gather facts
-  import_playbook: facts.yml
+  import_playbook: internal_facts.yml
 
 - name: Prepare for etcd install
   hosts: k8s_cluster:etcd
@@ -11,12 +11,15 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: "container-engine", tags: "container-engine", when: deploy_container_engine }
     - { role: download, tags: download, when: "not skip_downloads" }
 
 - name: Install etcd
+  vars:
+    etcd_cluster_setup: true
+    etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}"
   import_playbook: install_etcd.yml
 
 - name: Install Kubernetes nodes
@@ -25,7 +28,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes/node, tags: node }
 
 - name: Install the control plane
@@ -34,8 +37,8 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
-    - { role: kubernetes/control-plane, tags: master }
+    - { role: kubespray_defaults }
+    - { role: kubernetes/control-plane, tags: control-plane }
     - { role: kubernetes/client, tags: client }
     - { role: kubernetes-apps/cluster_roles, tags: cluster-roles }
 
@@ -45,12 +48,12 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes/kubeadm, tags: kubeadm}
     - { role: kubernetes/node-label, tags: node-label }
     - { role: kubernetes/node-taint, tags: node-taint }
+    - { role: kubernetes-apps/common_crds }
     - { role: network_plugin, tags: network }
-    - { role: kubernetes-apps/kubelet-csr-approver, tags: kubelet-csr-approver }
 
 - name: Install Calico Route Reflector
   hosts: calico_rr
@@ -58,7 +61,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: network_plugin/calico/rr, tags: ['network', 'calico_rr'] }
 
 - name: Patch Kubernetes for Windows
@@ -67,8 +70,8 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
-    - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"] }
+    - { role: kubespray_defaults }
+    - { role: win_nodes/kubernetes_patch, tags: ["control-plane", "win_nodes"] }
 
 - name: Install Kubernetes apps
   hosts: kube_control_plane
@@ -76,9 +79,8 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller }
-    - { role: kubernetes-apps/network_plugin, tags: network }
     - { role: kubernetes-apps/policy_controller, tags: policy-controller }
     - { role: kubernetes-apps/ingress_controller, tags: ingress-controller }
     - { role: kubernetes-apps/external_provisioner, tags: external-provisioner }
@@ -90,5 +92,5 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }

playbooks/facts.yml

@@ -1,41 +1,6 @@
 ---
-- name: Bootstrap hosts for Ansible
-  hosts: k8s_cluster:etcd:calico_rr
-  strategy: linear
-  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
-  gather_facts: false
-  environment: "{{ proxy_disable_env }}"
-  vars:
-    # Need to disable pipelining for bootstrap-os as some systems have requiretty in sudoers set, which makes pipelining
-    # fail. bootstrap-os fixes this on these systems, so in later plays it can be enabled.
-    ansible_ssh_pipelining: false
-  roles:
-    - { role: bootstrap-os, tags: bootstrap-os}
-    - { role: kubespray-defaults }
+- name: Common tasks for every playbooks
+  import_playbook: boilerplate.yml
 
 - name: Gather facts
-  hosts: k8s_cluster:etcd:calico_rr
-  gather_facts: false
-  tags: always
-  tasks:
-    - name: Gather minimal facts
-      setup:
-        gather_subset: '!all'
-
-    # filter match the following variables:
-    # ansible_default_ipv4
-    # ansible_default_ipv6
-    # ansible_all_ipv4_addresses
-    # ansible_all_ipv6_addresses
-    - name: Gather necessary facts (network)
-      setup:
-        gather_subset: '!all,!min,network'
-        filter: "ansible_*_ipv[46]*"
-
-    # filter match the following variables:
-    # ansible_memtotal_mb
-    # ansible_swaptotal_mb
-    - name: Gather necessary facts (hardware)
-      setup:
-        gather_subset: '!all,!min,hardware'
-        filter: "ansible_*total_mb"
+  import_playbook: internal_facts.yml

playbooks/install_etcd.yml

@@ -2,7 +2,7 @@
 - name: Add worker nodes to the etcd play if needed
   hosts: kube_node
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
   tasks:
     - name: Check if nodes needs etcd client certs (depends on network_plugin)
       group_by:
@@ -20,10 +20,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - role: etcd
       tags: etcd
-      vars:
-        etcd_cluster_setup: true
-        etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}"
       when: etcd_deployment_type != "kubeadm"

playbooks/internal_facts.yml

@@ -0,0 +1,39 @@
+---
+- name: Bootstrap hosts for Ansible
+  hosts: k8s_cluster:etcd:calico_rr
+  strategy: linear
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  gather_facts: false
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: bootstrap_os, tags: bootstrap_os}
+
+- name: Gather facts
+  hosts: k8s_cluster:etcd:calico_rr
+  gather_facts: false
+  tags: always
+  tasks:
+    - name: Gather and compute network facts
+      import_role:
+        name: network_facts
+    - name: Gather minimal facts
+      setup:
+        gather_subset: '!all'
+
+    # filter match the following variables:
+    # ansible_default_ipv4
+    # ansible_default_ipv6
+    # ansible_all_ipv4_addresses
+    # ansible_all_ipv6_addresses
+    - name: Gather necessary facts (network)
+      setup:
+        gather_subset: '!all,!min,network'
+        filter: "ansible_*_ipv[46]*"
+
+    # filter match the following variables:
+    # ansible_memtotal_mb
+    # ansible_swaptotal_mb
+    - name: Gather necessary facts (hardware)
+      setup:
+        gather_subset: '!all,!min,hardware'
+        filter: "ansible_*total_mb"

playbooks/recover_control_plane.yml

@@ -6,7 +6,7 @@
   hosts: etcd[0]
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults}
+    - { role: kubespray_defaults}
     - role: recover_control_plane/etcd
       when: etcd_deployment_type != "kubeadm"
 
@@ -14,7 +14,7 @@
   hosts: kube_control_plane[0]
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults}
+    - { role: kubespray_defaults}
     - { role: recover_control_plane/control-plane }
 
 - name: Apply whole cluster install
@@ -24,5 +24,5 @@
   hosts: kube_control_plane
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults}
+    - { role: kubespray_defaults}
     - { role: recover_control_plane/post-recover }

playbooks/remove_node.yml

@@ -1,6 +1,8 @@
 ---
 - name: Validate nodes for removal
   hosts: localhost
+  gather_facts: false
+  become: false
   tasks:
     - name: Assert that nodes are specified for removal
       assert:
@@ -30,7 +32,7 @@
       when: pause_result.user_input | default('yes') != 'yes'
 
 - name: Gather facts
-  import_playbook: facts.yml
+  import_playbook: internal_facts.yml
   when: reset_nodes | default(True) | bool
 
 - name: Reset node
@@ -42,8 +44,8 @@
       service_facts:
       when: reset_nodes | default(True) | bool
   roles:
-    - { role: kubespray-defaults, when: reset_nodes | default(True) | bool }
-    - { role: remove-node/pre-remove, tags: pre-remove }
+    - { role: kubespray_defaults, when: reset_nodes | default(True) | bool }
+    - { role: remove_node/pre_remove, tags: pre-remove }
     - role: remove-node/remove-etcd-node
       when: "'etcd' in group_names"
     - { role: reset, tags: reset, when: reset_nodes | default(True) | bool }
@@ -54,5 +56,5 @@
   gather_facts: false
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults, when: reset_nodes | default(True) | bool }
+    - { role: kubespray_defaults, when: reset_nodes | default(True) | bool }
     - { role: remove-node/post-remove, tags: post-remove }

playbooks/reset.yml

@@ -3,7 +3,7 @@
   import_playbook: boilerplate.yml
 
 - name: Gather facts
-  import_playbook: facts.yml
+  import_playbook: internal_facts.yml
 
 - name: Reset cluster
   hosts: etcd:k8s_cluster:calico_rr
@@ -30,6 +30,6 @@
 
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults}
+    - { role: kubespray_defaults}
     - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_early: true }
     - { role: reset, tags: reset }

playbooks/scale.yml

@@ -3,24 +3,13 @@
   import_playbook: boilerplate.yml
 
 - name: Gather facts
-  import_playbook: facts.yml
+  import_playbook: internal_facts.yml
 
-- name: Generate the etcd certificates beforehand
-  hosts: etcd:kube_control_plane
-  gather_facts: false
-  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
-  environment: "{{ proxy_disable_env }}"
-  roles:
-    - { role: kubespray-defaults }
-    - role: etcd
-      tags: etcd
-      vars:
-        etcd_cluster_setup: false
-        etcd_events_cluster_setup: false
-      when:
-        - etcd_deployment_type != "kubeadm"
-        - kube_network_plugin in ["calico", "flannel", "canal", "cilium"] or cilium_deploy_additionally | default(false) | bool
-        - kube_network_plugin != "calico" or calico_datastore == "etcd"
+- name: Install etcd
+  vars:
+    etcd_cluster_setup: false
+    etcd_events_cluster_setup: false
+  import_playbook: install_etcd.yml
 
 - name: Download images to ansible host cache via first kube_control_plane node
   hosts: kube_control_plane[0]
@@ -28,7 +17,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults, when: "not skip_downloads and download_run_once and not download_localhost" }
+    - { role: kubespray_defaults, when: "not skip_downloads and download_run_once and not download_localhost" }
     - { role: kubernetes/preinstall, tags: preinstall, when: "not skip_downloads and download_run_once and not download_localhost" }
     - { role: download, tags: download, when: "not skip_downloads and download_run_once and not download_localhost" }
 
@@ -38,7 +27,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: container-engine, tags: "container-engine", when: deploy_container_engine }
     - { role: download, tags: download, when: "not skip_downloads" }
@@ -57,7 +46,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes/node, tags: node }
 
 - name: Upload control plane certs and retrieve encryption key
@@ -66,7 +55,7 @@
   gather_facts: false
   tags: kubeadm
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
   tasks:
     - name: Upload control plane certificates
       command: >-
@@ -88,7 +77,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes/kubeadm, tags: kubeadm }
     - { role: kubernetes/node-label, tags: node-label }
     - { role: kubernetes/node-taint, tags: node-taint }
@@ -100,5 +89,5 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }

playbooks/upgrade_cluster.yml

@@ -3,7 +3,7 @@
   import_playbook: boilerplate.yml
 
 - name: Gather facts
-  import_playbook: facts.yml
+  import_playbook: internal_facts.yml
 
 - name: Download images to ansible host cache via first kube_control_plane node
   hosts: kube_control_plane[0]
@@ -11,7 +11,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults, when: "not skip_downloads and download_run_once and not download_localhost"}
+    - { role: kubespray_defaults, when: "not skip_downloads and download_run_once and not download_localhost"}
     - { role: kubernetes/preinstall, tags: preinstall, when: "not skip_downloads and download_run_once and not download_localhost" }
     - { role: download, tags: download, when: "not skip_downloads and download_run_once and not download_localhost" }
 
@@ -21,7 +21,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: download, tags: download, when: "not skip_downloads" }
 
@@ -32,10 +32,13 @@
   environment: "{{ proxy_disable_env }}"
   serial: "{{ serial | default('20%') }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: container-engine, tags: "container-engine", when: deploy_container_engine }
 
 - name: Install etcd
+  vars:
+    etcd_cluster_setup: true
+    etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}"
   import_playbook: install_etcd.yml
 
 - name: Handle upgrades to control plane components first to maintain backwards compat.
@@ -45,7 +48,7 @@
   environment: "{{ proxy_disable_env }}"
   serial: 1
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
     - { role: upgrade/system-upgrade, tags: system-upgrade }
     - { role: download, tags: download, when: "system_upgrade and system_upgrade_reboot != 'never' and not skip_downloads" }
@@ -67,10 +70,9 @@
   serial: "{{ serial | default('20%') }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller }
     - { role: network_plugin, tags: network }
-    - { role: kubernetes-apps/network_plugin, tags: network }
     - { role: kubernetes-apps/policy_controller, tags: policy-controller }
 
 - name: Finally handle worker upgrades, based on given batch size
@@ -80,7 +82,7 @@
   environment: "{{ proxy_disable_env }}"
   serial: "{{ serial | default('20%') }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
     - { role: upgrade/system-upgrade, tags: system-upgrade }
     - { role: download, tags: download, when: "system_upgrade and system_upgrade_reboot != 'never' and not skip_downloads" }
@@ -97,7 +99,7 @@
   any_errors_fatal: true
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"] }
 
 - name: Install Calico Route Reflector
@@ -106,7 +108,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: network_plugin/calico/rr, tags: network }
 
 - name: Install Kubernetes apps
@@ -115,7 +117,7 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes-apps/ingress_controller, tags: ingress-controller }
     - { role: kubernetes-apps/external_provisioner, tags: external-provisioner }
     - { role: kubernetes-apps, tags: apps }
@@ -126,5 +128,5 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
-    - { role: kubespray-defaults }
+    - { role: kubespray_defaults }
     - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }

remove_node.yml

@@ -0,0 +1,3 @@
+---
+- name: Remove node
+  ansible.builtin.import_playbook: playbooks/remove_node.yml

requirements.txt

@@ -1,6 +1,6 @@
-ansible==9.13.0
+ansible==10.7.0
 # Needed for community.crypto module
-cryptography==44.0.2
+cryptography==46.0.2
 # Needed for jinja2 json_query templating
 jmespath==1.0.1
 # Needed for ansible.utils.ipaddr

roles/adduser/molecule/default/molecule.yml

@@ -3,8 +3,8 @@ role_name_check: 1
 dependency:
   name: galaxy
 platforms:
-  - name: ubuntu20
-    cloud_image: ubuntu-2004
+  - name: ubuntu22
+    cloud_image: ubuntu-2204
     vm_cpu_cores: 1
     vm_memory: 512
 provisioner:

roles/bastion-ssh-config/molecule/default/molecule.yml

@@ -4,7 +4,7 @@ dependency:
   name: galaxy
 platforms:
   - name: bastion-01
-    cloud_image: ubuntu-2004
+    cloud_image: ubuntu-2204
     vm_cpu_cores: 1
     vm_memory: 512
 provisioner:

roles/bootstrap-os/tasks/amzn.yml

@@ -1,27 +0,0 @@
----
-- name: Enable selinux-ng repo for Amazon Linux for container-selinux
-  command: amazon-linux-extras enable selinux-ng
-
-- name: Enable EPEL repo for Amazon Linux
-  yum_repository:
-    name: epel
-    file: epel
-    description: Extra Packages for Enterprise Linux 7 - $basearch
-    baseurl: http://download.fedoraproject.org/pub/epel/7/$basearch
-    gpgcheck: true
-    gpgkey: http://download.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7
-    skip_if_unavailable: true
-    enabled: true
-    repo_gpgcheck: false
-  when: epel_enabled
-
-# iproute is required for networking related facts gathering
-# See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_vars_facts.html#package-requirements-for-fact-gathering
-# Note: It is not recommended way, but since the tasks execution order, put it here is the simplest way so far. We can move it to a proper place later.
-# TODO: move this to roles/kubernetes/preinstall/vars/main.yml -> pkgs variables
-# Currently not possible because the collect the network facts before that step, needs reordering of the exec flow.
-- name: Ensure iproute is installed
-  package:
-    name: iproute
-    state: present
-  become: true

roles/bootstrap-os/tasks/clear-linux-os.yml

@@ -1,27 +0,0 @@
----
-# ClearLinux ships with Python installed
-
-- name: Install basic package to run containers
-  package:
-    name: containers-basic
-    state: present
-
-- name: Make sure docker service is enabled
-  systemd_service:
-    name: docker
-    masked: false
-    enabled: true
-    daemon_reload: true
-    state: started
-  become: true
-
-# iproute2 is required for networking related facts gathering
-# See https://docs.ansible.com/ansible/latest/playbook_guide/playbooks_vars_facts.html#package-requirements-for-fact-gathering
-# Note: It is not recommended way, but since the tasks execution order, put it here is the simplest way so far. We can move it to a proper place later.
-# TODO: move this to roles/kubernetes/preinstall/vars/main.yml -> pkgs variables
-# Currently not possible because the collect the network facts before that step, needs reordering of the exec flow.
-- name: Ensure iproute2 is installed
-  package:
-    name: iproute2
-    state: present
-  become: true

roles/bootstrap-os/tasks/main.yml

@@ -1,57 +1,10 @@
 ---
-- name: Fetch /etc/os-release
-  raw: cat /etc/os-release
-  register: os_release
-  changed_when: false
-  # This command should always run, even in check mode
-  check_mode: false
+- name: Warn for usage of deprecated role
+  fail:
+    msg: bootstrap-os is deprecated, switch to bootstrap_os
+  ignore_errors: true # noqa ignore-errors
+  run_once: true
 
-- name: Include distro specifics vars and tasks
-  vars:
-    os_release_dict: "{{ os_release.stdout_lines | select('regex', '^.+=.*$') | map('regex_replace', '\"', '') |
-                         map('split', '=') | community.general.dict }}"
-  block:
-  - name: Include vars
-    include_vars: "{{ item }}"
-    tags:
-    - facts
-    with_first_found:
-    - &search
-      files:
-      - "{{ os_release_dict['ID'] }}-{{ os_release_dict['VARIANT_ID'] }}.yml"
-      - "{{ os_release_dict['ID'] }}.yml"
-      paths:
-      - vars/
-      skip: true
-  - name: Include tasks
-    include_tasks: "{{ included_tasks_file }}"
-    with_first_found:
-    - <<: *search
-      paths: []
-    loop_control:
-      loop_var: included_tasks_file
-
-
-- name: Create remote_tmp for it is used by another module
-  file:
-    path: "{{ ansible_remote_tmp | default('~/.ansible/tmp') }}"
-    state: directory
-    mode: "0700"
-
-- name: Gather facts
-  setup:
-    gather_subset: '!all'
-    filter: ansible_*
-
-- name: Assign inventory name to unconfigured hostnames (non-CoreOS, non-Flatcar, Suse and ClearLinux, non-Fedora)
-  hostname:
-    name: "{{ inventory_hostname }}"
-  when: override_system_hostname
-
-- name: Ensure bash_completion.d folder exists
-  file:
-    name: /etc/bash_completion.d/
-    state: directory
-    owner: root
-    group: root
-    mode: "0755"
+- name: Compat for direct role import
+  import_role:
+    name: bootstrap_os

roles/bootstrap_os/defaults/main.yml

@@ -9,6 +9,9 @@ rh_subscription_check_timeout: 180
 # Disable locksmithd or leave it in its current state
 coreos_locksmithd_disable: false
 
+# Install epel repo on Centos/RHEL
+epel_enabled: false
+
 ## Oracle Linux specific variables
 # Install public repo on Oracle Linux
 use_oracle_public_repo: true
@@ -16,6 +19,8 @@ use_oracle_public_repo: true
 ## Ubuntu specific variables
 # Disable unattended-upgrades for Linux kernel and all packages start with linux- on Ubuntu
 ubuntu_kernel_unattended_upgrades_disabled: false
+# Stop unattended-upgrades if it is currently running on Ubuntu
+ubuntu_stop_unattended_upgrades: false
 
 fedora_coreos_packages:
   - python
@@ -32,8 +37,3 @@ override_system_hostname: true
 is_fedora_coreos: false
 
 skip_http_proxy_on_os_packages: false
-
-# If this is true, debug information will be displayed but
-# may contain some private data, so it is recommended to set it to false
-# in the production environment.
-unsafe_show_logs: false

roles/bootstrap_os/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: kubespray_defaults

roles/bootstrap_os/molecule/default/converge.yml

@@ -4,4 +4,4 @@
   gather_facts: false
   become: true
   roles:
-    - role: bootstrap-os
+    - role: bootstrap_os

roles/bootstrap_os/molecule/default/molecule.yml

@@ -3,14 +3,14 @@ role_name_check: 1
 dependency:
   name: galaxy
 platforms:
-  - name: ubuntu20
-    cloud_image: ubuntu-2004
-    vm_cpu_cores: 1
-    vm_memory: 512
   - name: ubuntu22
     cloud_image: ubuntu-2204
     vm_cpu_cores: 1
     vm_memory: 512
+  - name: ubuntu24
+    cloud_image: ubuntu-2404
+    vm_cpu_cores: 1
+    vm_memory: 512
   - name: almalinux9
     cloud_image: almalinux-9
     vm_cpu_cores: 1