Patch versions updates

2026-02-01 17:48:12 -03:30 · 2025-03-28 02:47:47 +00:00
546 changed files with 42553 additions and 6688 deletions
--- a/.ansible-lint
+++ b/.ansible-lint
@@ -1,4 +1,5 @@
 ---
+parseable: true
 skip_list:
  # see https://docs.ansible.com/ansible-lint/rules/default_rules.html for a list of all default rules

@@ -11,12 +12,10 @@ skip_list:
  # (Disabled in June 2021)
  - 'role-name'

-  # [var-naming]
+  # [var-naming] "defaults/main.yml" File defines variable 'apiVersion' that violates variable naming standards
  # In Kubespray we use variables that use camelCase to match their k8s counterparts
  # (Disabled in June 2021)
-  - 'var-naming[pattern]'
-  # Variables names from within roles in kubespray don't need role name as a prefix
-  - 'var-naming[no-role-prefix]'
+  - 'var-naming'

  # [fqcn-builtins]
  # Roles in kubespray don't need fully qualified collection names
@@ -33,8 +32,6 @@ skip_list:
  # Disable run-once check with free strategy
  # (Disabled in June 2023 after ansible upgrade; FIXME)
  - 'run-once[task]'
-
-  - 'jinja[spacing]'
 exclude_paths:
  # Generated files
  - tests/files/custom_cni/cilium.yaml
@@ -42,7 +39,5 @@ exclude_paths:
  - .github
  - .ansible
  - .cache
-  - .gitlab-ci.yml
-  - .gitlab-ci
 mock_modules:
  - gluster.gluster.gluster_volume
--- a/.github/ISSUE_TEMPLATE/bug-report.yaml
+++ b/.github/ISSUE_TEMPLATE/bug-report.yaml
@@ -108,6 +108,7 @@ body:
        - meta
        - multus
        - ovn4nfv
+        - weave
    validations:
      required: true

--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -16,6 +16,5 @@ updates:
    directory: "/"
    labels:
      - release-note-none
-      - ci-short
    schedule:
      interval: "weekly"
--- a/.github/workflows/auto-label-os.yml
+++ b/.github/workflows/auto-label-os.yml
@@ -13,16 +13,16 @@ jobs:
      issues: write

    steps:
-      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+      - uses: actions/checkout@v3

      - name: Parse issue form
-        uses: stefanbuck/github-issue-parser@10dcc54158ba4c137713d9d69d70a2da63b6bda3
+        uses: stefanbuck/github-issue-parser@v3
        id: issue-parser
        with:
          template-path: .github/ISSUE_TEMPLATE/bug-report.yaml

      - name: Set labels based on OS field
-        uses: redhat-plumbers-in-action/advanced-issue-labeler@b80ae64e3e156e9c111b075bfa04b295d54e8e2e
+        uses: redhat-plumbers-in-action/advanced-issue-labeler@v2
        with:
          issue-form: ${{ steps.issue-parser.outputs.jsonString }}
          section: os
--- a/.github/workflows/upgrade-patch-versions-schedule.yml
+++ b/.github/workflows/upgrade-patch-versions-schedule.yml
@@ -8,19 +8,18 @@ on:
 permissions: {}
 jobs:
  get-releases-branches:
-    if: github.repository == 'kubernetes-sigs/kubespray'
    runs-on: ubuntu-latest
    outputs:
      branches: ${{ steps.get-branches.outputs.data }}
    steps:
-    - uses: octokit/graphql-action@ddde8ebb2493e79f390e6449c725c21663a67505
+    - uses: octokit/graphql-action@v2.3.2
      id: get-branches
      with:
        query: |
          query get_release_branches($owner:String!, $name:String!) {
            repository(owner:$owner, name:$name) {
              refs(refPrefix: "refs/heads/",
-                   first: 3,
+                   first: 0, # TODO increment once we have release branch with the new checksums format
                   query: "release-",
                   orderBy: {
                     field: ALPHABETICAL,
--- a/.github/workflows/upgrade-patch-versions.yml
+++ b/.github/workflows/upgrade-patch-versions.yml
@@ -11,10 +11,10 @@ jobs:
  update-patch-versions:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+    - uses: actions/checkout@v4
      with:
        ref: ${{ inputs.branch }}
-    - uses: actions/setup-python@v6
+    - uses: actions/setup-python@v5
      with:
        python-version: '3.13'
        cache: 'pip'
@@ -22,19 +22,19 @@ jobs:
    - run: update-hashes
      env:
        API_KEY: ${{ secrets.GITHUB_TOKEN }}
-    - uses: actions/cache@v5
+    - uses: actions/cache@v4
      with:
        key: pre-commit-hook-propagate
        path: |
          ~/.cache/pre-commit
    - run: pre-commit run --all-files propagate-ansible-variables
      continue-on-error: true
-    - uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0
+    - uses: peter-evans/create-pull-request@v7
      with:
        commit-message: Patch versions updates
        title: Patch versions updates - ${{ inputs.branch }}
        labels: bot
-        branch: component_hash_update/${{ inputs.branch }}
+        branch: ${{ inputs.branch }}-patch-updates
        sign-commits: true
        body: |
          /kind feature
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,9 +1,9 @@
 ---
 stages:
-  - build # build docker image used in most other jobs
-  - test # unit tests
-  - deploy-part1 # kubespray runs - common setup
-  - deploy-extended # kubespray runs - rarer or costlier (to test) setups
+  - build
+  - test
+  - deploy-part1
+  - deploy-extended

 variables:
  FAILFASTCI_NAMESPACE: 'kargo-ci'
@@ -24,7 +24,6 @@ variables:
  ANSIBLE_REMOTE_USER: kubespray
  ANSIBLE_PRIVATE_KEY_FILE: /tmp/id_rsa
  ANSIBLE_INVENTORY: /tmp/inventory
-  ANSIBLE_STDOUT_CALLBACK: "default"
  RESET_CHECK: "false"
  REMOVE_NODE_CHECK: "false"
  UPGRADE_TEST: "false"
@@ -32,12 +31,12 @@ variables:
  ANSIBLE_VERBOSITY: 2
  RECOVER_CONTROL_PLANE_TEST: "false"
  RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
-  OPENTOFU_VERSION: v1.9.1
+  TERRAFORM_VERSION: 1.3.7
  PIPELINE_IMAGE: "$CI_REGISTRY_IMAGE/pipeline:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"

 before_script:
  - ./tests/scripts/rebase.sh
-  - mkdir -p cluster-dump $ANSIBLE_INVENTORY
+  - mkdir -p /.ssh

 .job: &job
  tags:
@@ -49,18 +48,60 @@ before_script:
      - cluster-dump/
  needs:
    - pipeline-image
+  variables:
+    ANSIBLE_STDOUT_CALLBACK: "debug"

 .job-moderated:
  extends: .job
  needs:
    - pipeline-image
+    - ci-not-authorized
    - pre-commit            # lint
    - vagrant-validate      # lint

+.testcases: &testcases
+  extends: .job-moderated
+  interruptible: true
+  before_script:
+    - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+    - ./tests/scripts/rebase.sh
+    - ./tests/scripts/testcases_prepare.sh
+  script:
+    - ./tests/scripts/testcases_run.sh
+  after_script:
+    - ./tests/scripts/testcases_cleanup.sh
+
+# For failfast, at least 1 job must be defined in .gitlab-ci.yml
+# Premoderated with manual actions
+ci-not-authorized:
+  stage: build
+  before_script: []
+  after_script: []
+  rules:
+    # LGTM or ok-to-test labels
+    - if: $PR_LABELS =~ /.*,(lgtm|approved|ok-to-test).*|^(lgtm|approved|ok-to-test).*/i
+      variables:
+        CI_OK_TO_TEST: '0'
+      when: always
+    - if: $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "trigger"
+      variables:
+        CI_OK_TO_TEST: '0'
+    - if: $CI_COMMIT_BRANCH == "master"
+      variables:
+        CI_OK_TO_TEST: '0'
+    - when: always
+      variables:
+        CI_OK_TO_TEST: '1'
+  script:
+    - exit $CI_OK_TO_TEST
+  tags:
+    - ffci
+  needs: []
+
 include:
  - .gitlab-ci/build.yml
  - .gitlab-ci/lint.yml
  - .gitlab-ci/terraform.yml
-  - .gitlab-ci/kubevirt.yml
+  - .gitlab-ci/packet.yml
  - .gitlab-ci/vagrant.yml
  - .gitlab-ci/molecule.yml
--- a/.gitlab-ci/build.yml
+++ b/.gitlab-ci/build.yml
@@ -1,5 +1,5 @@
 ---
-pipeline-image:
+.build-container:
  cache:
    key: $CI_COMMIT_REF_SLUG
    paths:
@@ -7,24 +7,27 @@ pipeline-image:
  tags:
    - ffci
  stage: build
-  image: moby/buildkit:rootless
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: ['']
  variables:
-    BUILDKITD_FLAGS: --oci-worker-no-process-sandbox
-    CACHE_IMAGE: $CI_REGISTRY_IMAGE/pipeline:cache
-  # TODO: remove the override
-  # currently rebase.sh depends on bash (not available in the kaniko image)
-  # once we have a simpler rebase (which should be easy if the target branch ref is available as variable
-  # we'll be able to rebase here as well hopefully
+    TAG: $CI_COMMIT_SHORT_SHA
+    PROJECT_DIR: $CI_PROJECT_DIR
+    DOCKERFILE: Dockerfile
+    GODEBUG: "http2client=0"
  before_script:
-    - mkdir -p ~/.docker
-    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > ~/.docker/config.json
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json
  script:
-    - |
-      buildctl-daemonless.sh build \
-        --frontend dockerfile.v0 \
-        --local context=$CI_PROJECT_DIR \
-        --local dockerfile=$CI_PROJECT_DIR \
-        --opt filename=pipeline.Dockerfile \
-        --export-cache type=registry,ref=$CACHE_IMAGE \
-        --import-cache type=registry,ref=$CACHE_IMAGE \
-        --output type=image,name=$PIPELINE_IMAGE,push=true
+    - /kaniko/executor --cache=true
+                       --cache-dir=image-cache
+                       --context $PROJECT_DIR
+                       --dockerfile $PROJECT_DIR/$DOCKERFILE
+                       --label 'git-branch'=$CI_COMMIT_REF_SLUG
+                       --label 'git-tag=$CI_COMMIT_TAG'
+                       --destination $PIPELINE_IMAGE
+                       --log-timestamp=true
+
+pipeline-image:
+  extends: .build-container
+  variables:
+    DOCKERFILE: pipeline.Dockerfile
--- a/.gitlab-ci/kubevirt.yml
+++ b/.gitlab-ci/kubevirt.yml
@@ -1,155 +0,0 @@
---
-.kubevirt:
-  extends: .job-moderated
-  interruptible: true
-  script:
-    - ansible-playbook tests/cloud_playbooks/create-kubevirt.yml
-                              -e @"tests/files/${TESTCASE}.yml"
-    - ./tests/scripts/testcases_run.sh
-  variables:
-    ANSIBLE_TIMEOUT: "120"
-  tags:
-    - ffci
-  needs:
-    - pipeline-image
-
-# TODO: generate testcases matrixes from the files in tests/files/
-# this is needed to avoid the need for PR rebasing when a job was added or removed in the target branch
-# (currently, a removed job in the target branch breaks the tests, because the
-# pipeline definition is parsed by gitlab before the rebase.sh script)
-# CI template for PRs
-pr:
-  stage: deploy-part1
-  rules:
-    - if: $PR_LABELS =~ /.*ci-short.*/
-      when: manual
-      allow_failure: true
-    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    - when: manual
-      allow_failure: true
-  extends: .kubevirt
-  parallel:
-    matrix:
-      - TESTCASE:
-          - almalinux9-crio
-          - almalinux9-kube-ovn
-          - debian11-calico-collection
-          - debian11-macvlan
-          - debian12-cilium
-          - debian13-cilium
-          - fedora39-kube-router
-          - openeuler24-calico
-          - rockylinux9-cilium
-          - rockylinux10-cilium
-          - ubuntu22-calico-all-in-one
-          - ubuntu22-calico-all-in-one-upgrade
-          - ubuntu24-calico-etcd-datastore
-          - ubuntu24-calico-all-in-one-hardening
-          - ubuntu24-cilium-sep
-          - ubuntu24-flannel-collection
-          - ubuntu24-kube-router-sep
-          - ubuntu24-kube-router-svc-proxy
-          - ubuntu24-ha-separate-etcd
-          - flatcar4081-calico
-          - fedora40-flannel-crio-collection-scale
-
-# The ubuntu24-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
-ubuntu24-calico-all-in-one:
-  stage: deploy-part1
-  extends: .kubevirt
-  variables:
-    TESTCASE: ubuntu24-calico-all-in-one
-  rules:
-    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    - when: manual
-      allow_failure: true
-
-pr_full:
-  extends: .kubevirt
-  stage: deploy-extended
-  rules:
-    - if: $PR_LABELS =~ /.*ci-full.*/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    # Else run as manual
-    - when: manual
-      allow_failure: true
-  parallel:
-    matrix:
-      - TESTCASE:
-          - almalinux9-calico-ha-ebpf
-          - almalinux9-calico-nodelocaldns-secondary
-          - debian11-custom-cni
-          - debian11-kubelet-csr-approver
-          - debian12-custom-cni-helm
-          - fedora39-calico-swap-selinux
-          - fedora39-crio
-          - ubuntu24-calico-ha-wireguard
-          - ubuntu24-flannel-ha
-          - ubuntu24-flannel-ha-once
-
-# Need an update of the container image to use schema v2
-# update: quay.io/kubespray/vm-amazon-linux-2:latest
-manual:
-  extends: pr_full
-  parallel:
-    matrix:
-      - TESTCASE:
-          - amazon-linux-2-all-in-one
-  rules:
-    - when: manual
-      allow_failure: true
-
-pr_extended:
-  extends: .kubevirt
-  stage: deploy-extended
-  rules:
-    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
-      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-    - when: manual
-      allow_failure: true
-  parallel:
-    matrix:
-      - TESTCASE:
-          - almalinux9-calico
-          - almalinux9-calico-remove-node
-          - almalinux9-docker
-          - debian11-docker
-          - debian12-calico
-          - debian12-docker
-          - debian13-calico
-          - rockylinux9-calico
-          - rockylinux10-calico
-          - ubuntu22-all-in-one-docker
-          - ubuntu24-all-in-one-docker
-          - ubuntu24-calico-all-in-one
-          - ubuntu24-calico-etcd-kubeadm
-          - ubuntu24-flannel
-
-# TODO: migrate to pr-full, fix the broken ones
-periodic:
-  allow_failure: true
-  extends: .kubevirt
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-      when: on_success
-  parallel:
-    matrix:
-      - TESTCASE:
-          - debian11-calico-upgrade
-          - debian11-calico-upgrade-once
-          - debian12-cilium-svc-proxy
-          - fedora39-calico-selinux
-          - fedora40-docker-calico
-          - ubuntu24-calico-etcd-kubeadm-upgrade-ha
-          - ubuntu24-calico-ha-recover
-          - ubuntu24-calico-ha-recover-noquorum
--- a/.gitlab-ci/lint.yml
+++ b/.gitlab-ci/lint.yml
@@ -6,7 +6,6 @@ pre-commit:
  image: 'ghcr.io/pre-commit-ci/runner-image@sha256:fe01a6ec51b298412990b88627c3973b1146c7304f930f469bafa29ba60bcde9'
  variables:
    PRE_COMMIT_HOME: ${CI_PROJECT_DIR}/.cache/pre-commit
-    ANSIBLE_STDOUT_CALLBACK: default
  script:
  - pre-commit run --all-files --show-diff-on-failure
  cache:
@@ -24,3 +23,4 @@ vagrant-validate:
    VAGRANT_VERSION: 2.3.7
  script:
  - ./tests/scripts/vagrant-validate.sh
+  except: ['triggers', 'master']
--- a/.gitlab-ci/molecule.yml
+++ b/.gitlab-ci/molecule.yml
@@ -1,24 +1,19 @@
 ---
 .molecule:
  tags: [ffci]
-  rules: # run on ci-short as well
-  - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-    when: on_success
-  - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-    when: on_success
-  - when: manual
-    allow_failure: true
+  only: [/^pr-.*$/]
+  except: ['triggers']
  stage: deploy-part1
  image: $PIPELINE_IMAGE
  needs:
  - pipeline-image
+  # - ci-not-authorized
+  before_script:
+  - ./tests/scripts/rebase.sh
  script:
  - ./tests/scripts/molecule_run.sh
  after_script:
-  - rm -fr molecule_logs
-  - mkdir -p molecule_logs
-  - find ~/.cache/molecule/ \( -name '*.out' -o -name '*.err' \) -type f | xargs tar -uf molecule_logs/molecule.tar
-  - gzip molecule_logs/molecule.tar
+  - ./tests/scripts/molecule_logs.sh
  artifacts:
    when: always
    paths:
@@ -34,22 +29,28 @@ molecule:
      - container-engine/cri-dockerd
      - container-engine/containerd
      - container-engine/cri-o
-      - container-engine/gvisor
-      - container-engine/youki
      - adduser
      - bastion-ssh-config
-      - bootstrap_os
+      - bootstrap-os

+# CI template for periodic CI jobs
+# Enabled when PERIODIC_CI_ENABLED var is set
 molecule_full:
-  allow_failure: true
-  rules:
-  - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
-    when: on_success
-  - when: manual
+  only:
+    variables:
+    - $PERIODIC_CI_ENABLED
  allow_failure: true
  extends: molecule
  parallel:
    matrix:
    - ROLE:
+      - container-engine/cri-dockerd
+      - container-engine/containerd
+      - container-engine/cri-o
+      - adduser
+      - bastion-ssh-config
+      - bootstrap-os
      # FIXME : tests below are perma-failing
      - container-engine/kata-containers
+      - container-engine/gvisor
+      - container-engine/youki
--- a/.gitlab-ci/packet.yml
+++ b/.gitlab-ci/packet.yml
@@ -0,0 +1,257 @@
+---
+.packet:
+  extends: .testcases
+  variables:
+    ANSIBLE_TIMEOUT: "120"
+    CI_PLATFORM: packet
+    SSH_USER: kubespray
+  tags:
+    - ffci
+  needs:
+    - pipeline-image
+    - ci-not-authorized
+
+# CI template for PRs
+.packet_pr:
+  stage: deploy-part1
+  rules:
+    - if: $PR_LABELS =~ /.*ci-short.*/
+      when: manual
+      allow_failure: true
+    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
+      when: on_success
+    - when: manual
+      allow_failure: true
+  extends: .packet
+
+  ## Uncomment this to have multiple stages
+  # needs:
+  #   - packet_ubuntu20-calico-all-in-one
+
+.packet_pr_short:
+  stage: deploy-part1
+  extends: .packet
+  rules:
+    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
+      when: on_success
+    - when: manual
+      allow_failure: true
+
+.packet_pr_manual:
+  extends: .packet_pr
+  stage: deploy-extended
+  rules:
+    - if: $PR_LABELS =~ /.*ci-full.*/
+      when: on_success
+    # Else run as manual
+    - when: manual
+      allow_failure: true
+
+.packet_pr_extended:
+  extends: .packet_pr
+  stage: deploy-extended
+  rules:
+    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
+      when: on_success
+    - when: manual
+      allow_failure: true
+
+# CI template for periodic CI jobs
+# Enabled when PERIODIC_CI_ENABLED var is set
+.packet_periodic:
+  only:
+    variables:
+      - $PERIODIC_CI_ENABLED
+  allow_failure: true
+  extends: .packet
+
+# The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
+packet_ubuntu20-calico-all-in-one:
+  stage: deploy-part1
+  extends: .packet_pr_short
+  variables:
+    RESET_CHECK: "true"
+
+# ### PR JOBS PART2
+
+packet_ubuntu20-crio:
+  extends: .packet_pr_manual
+
+packet_ubuntu22-calico-all-in-one:
+  extends: .packet_pr
+
+packet_ubuntu22-calico-all-in-one-upgrade:
+  extends: .packet_pr
+  variables:
+    UPGRADE_TEST: graceful
+
+packet_ubuntu24-calico-etcd-datastore:
+  extends: .packet_pr
+
+packet_almalinux9-crio:
+  extends: .packet_pr
+
+packet_almalinux9-kube-ovn:
+  extends: .packet_pr
+
+packet_debian11-calico-collection:
+  extends: .packet_pr
+
+packet_debian11-macvlan:
+  extends: .packet_pr
+
+packet_debian12-cilium:
+  extends: .packet_pr
+
+packet_almalinux8-calico:
+  extends: .packet_pr
+
+packet_rockylinux8-calico:
+  extends: .packet_pr
+
+packet_rockylinux9-cilium:
+  extends: .packet_pr
+  variables:
+    RESET_CHECK: "true"
+
+# Need an update of the container image to use schema v2
+# update: quay.io/kubespray/vm-amazon-linux-2:latest
+packet_amazon-linux-2-all-in-one:
+  extends: .packet_pr_manual
+  rules:
+    - when: manual
+      allow_failure: true
+
+packet_opensuse15-6-calico:
+  extends: .packet_pr
+
+packet_ubuntu20-cilium-sep:
+  extends: .packet_pr
+
+packet_openeuler24-calico:
+  extends: .packet_pr
+
+packet_ubuntu20-calico-all-in-one-hardening:
+  extends: .packet_pr
+
+## Extended
+packet_debian11-docker:
+  extends: .packet_pr_extended
+
+packet_debian12-docker:
+  extends: .packet_pr_extended
+
+packet_debian12-calico:
+  extends: .packet_pr_extended
+
+packet_almalinux9-calico-remove-node:
+  extends: .packet_pr_extended
+  variables:
+    REMOVE_NODE_CHECK: "true"
+    REMOVE_NODE_NAME: "instance-3"
+
+packet_rockylinux9-calico:
+  extends: .packet_pr_extended
+
+packet_almalinux9-calico:
+  extends: .packet_pr_extended
+
+packet_almalinux9-docker:
+  extends: .packet_pr_extended
+
+packet_opensuse15-6-docker-cilium:
+  extends: .packet_pr_extended
+
+packet_ubuntu24-calico-all-in-one:
+  extends: .packet_pr_extended
+
+packet_ubuntu20-calico-etcd-kubeadm:
+  extends: .packet_pr_extended
+
+packet_ubuntu24-all-in-one-docker:
+  extends: .packet_pr_extended
+
+packet_ubuntu22-all-in-one-docker:
+  extends: .packet_pr_extended
+
+# ### MANUAL JOBS
+packet_fedora39-crio:
+  extends: .packet_pr_manual
+
+packet_ubuntu20-flannel-ha:
+  extends: .packet_pr_manual
+
+packet_ubuntu20-all-in-one-docker:
+  extends: .packet_pr_manual
+
+packet_ubuntu20-flannel-ha-once:
+  extends: .packet_pr_manual
+
+packet_fedora39-calico-swap-selinux:
+  extends: .packet_pr_manual
+
+packet_almalinux9-calico-ha-ebpf:
+  extends: .packet_pr_manual
+
+packet_almalinux9-calico-nodelocaldns-secondary:
+  extends: .packet_pr_manual
+
+packet_debian11-custom-cni:
+  extends: .packet_pr_manual
+
+packet_debian11-kubelet-csr-approver:
+  extends: .packet_pr_manual
+
+packet_debian12-custom-cni-helm:
+  extends: .packet_pr_manual
+
+packet_ubuntu20-calico-ha-wireguard:
+  extends: .packet_pr_manual
+
+# PERIODIC
+packet_fedora40-docker-calico:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    RESET_CHECK: "true"
+
+packet_fedora39-calico-selinux:
+  stage: deploy-extended
+  extends: .packet_periodic
+
+packet_ubuntu20-calico-etcd-kubeadm-upgrade-ha:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    UPGRADE_TEST: basic
+
+
+packet_debian11-calico-upgrade-once:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    UPGRADE_TEST: graceful
+
+packet_ubuntu20-calico-ha-recover:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    RECOVER_CONTROL_PLANE_TEST: "true"
+    RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
+
+packet_ubuntu20-calico-ha-recover-noquorum:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    RECOVER_CONTROL_PLANE_TEST: "true"
+    RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[1:]:kube_control_plane[1:]"
+
+packet_debian11-calico-upgrade:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    UPGRADE_TEST: graceful
+
+packet_debian12-cilium-svc-proxy:
+  stage: deploy-extended
+  extends: .packet_periodic
--- a/.gitlab-ci/terraform.yml
+++ b/.gitlab-ci/terraform.yml
@@ -3,40 +3,39 @@
 .terraform_install:
  extends: .job
  needs:
+    - ci-not-authorized
    - pipeline-image
-  variables:
-    TF_VAR_public_key_path: "${ANSIBLE_PRIVATE_KEY_FILE}.pub"
-    TF_VAR_ssh_private_key_path: $ANSIBLE_PRIVATE_KEY_FILE
-    CLUSTER: $CI_COMMIT_REF_NAME
-    TERRAFORM_STATE_ROOT: $CI_PROJECT_DIR
  stage: deploy-part1
  before_script:
+    - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
    - ./tests/scripts/rebase.sh
-    - mkdir -p cluster-dump $ANSIBLE_INVENTORY
-    - ./tests/scripts/opentofu_install.sh
+    - ./tests/scripts/testcases_prepare.sh
+    - ./tests/scripts/terraform_install.sh
+    # Set Ansible config
+    - cp ansible.cfg ~/.ansible.cfg
+    # Prepare inventory
    - cp contrib/terraform/$PROVIDER/sample-inventory/cluster.tfvars .
-    - ln -rs -t $ANSIBLE_INVENTORY contrib/terraform/$PROVIDER/hosts
-    - tofu -chdir="contrib/terraform/$PROVIDER" init
+    - ln -s contrib/terraform/$PROVIDER/hosts
+    - terraform -chdir="contrib/terraform/$PROVIDER" init
+    # Copy SSH keypair
+    - mkdir -p ~/.ssh
+    - echo "$PACKET_PRIVATE_KEY" | base64 -d > ~/.ssh/id_rsa
+    - chmod 400 ~/.ssh/id_rsa
+    - echo "$PACKET_PUBLIC_KEY" | base64 -d > ~/.ssh/id_rsa.pub
+    - mkdir -p contrib/terraform/$PROVIDER/group_vars
+    # Random subnet to avoid routing conflicts
+    - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"

-terraform_validate:
+.terraform_validate:
  extends: .terraform_install
  tags: [ffci]
  only: ['master', /^pr-.*$/]
  script:
-    - tofu -chdir="contrib/terraform/$PROVIDER" validate
-    - tofu -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
+    - terraform -chdir="contrib/terraform/$PROVIDER" validate
+    - terraform -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
  stage: test
  needs:
    - pipeline-image
-  parallel:
-    matrix:
-      - PROVIDER:
-          - openstack
-          - aws
-          - exoscale
-          - hetzner
-          - vsphere
-          - upcloud

 .terraform_apply:
  extends: .terraform_install
@@ -44,24 +43,99 @@ terraform_validate:
  stage: deploy-extended
  when: manual
  only: [/^pr-.*$/]
+  artifacts:
+    when: always
+    paths:
+      - cluster-dump/
  variables:
    ANSIBLE_INVENTORY_UNPARSED_FAILED: "true"
-    ANSIBLE_REMOTE_USER: ubuntu # the openstack terraform module does not handle custom user correctly
-    ANSIBLE_SSH_RETRIES: 15
-    TF_VAR_ssh_user: $ANSIBLE_REMOTE_USER
+    ANSIBLE_INVENTORY: hosts
+    CI_PLATFORM: tf
+    TF_VAR_ssh_user: $SSH_USER
    TF_VAR_cluster_name: $CI_JOB_ID
  script:
-    # Set Ansible config
-    - cp ansible.cfg ~/.ansible.cfg
-    - ssh-keygen -N '' -f $ANSIBLE_PRIVATE_KEY_FILE -t rsa
-    - mkdir -p contrib/terraform/$PROVIDER/group_vars
-    # Random subnet to avoid routing conflicts
-    - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
-    - tofu -chdir="contrib/terraform/$PROVIDER" apply -auto-approve -parallelism=1
    - tests/scripts/testcases_run.sh
  after_script:
    # Cleanup regardless of exit code
-    - tofu -chdir="contrib/terraform/$PROVIDER" destroy -auto-approve
+    - ./tests/scripts/testcases_cleanup.sh
+
+tf-validate-openstack:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: openstack
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-equinix:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: equinix
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-aws:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: aws
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-exoscale:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: exoscale
+
+tf-validate-hetzner:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: hetzner
+
+tf-validate-vsphere:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: vsphere
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-upcloud:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: upcloud
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-nifcloud:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: nifcloud
+
+# tf-packet-ubuntu20-default:
+#   extends: .terraform_apply
+#   variables:
+#     TF_VERSION: $TERRAFORM_VERSION
+#     PROVIDER: packet
+#     CLUSTER: $CI_COMMIT_REF_NAME
+#     TF_VAR_number_of_k8s_masters: "1"
+#     TF_VAR_number_of_k8s_nodes: "1"
+#     TF_VAR_plan_k8s_masters: t1.small.x86
+#     TF_VAR_plan_k8s_nodes: t1.small.x86
+#     TF_VAR_metro: am
+#     TF_VAR_public_key_path: ""
+#     TF_VAR_operating_system: ubuntu_20_04
+
+.ovh_variables: &ovh_variables
+  OS_AUTH_URL: https://auth.cloud.ovh.net/v3
+  OS_PROJECT_ID: 8d3cd5d737d74227ace462dee0b903fe
+  OS_PROJECT_NAME: "9361447987648822"
+  OS_USER_DOMAIN_NAME: Default
+  OS_PROJECT_DOMAIN_ID: default
+  OS_USERNAME: 8XuhBMfkKVrk
+  OS_REGION_NAME: UK1
+  OS_INTERFACE: public
+  OS_IDENTITY_API_VERSION: "3"

 # Elastx is generously donating resources for Kubespray on Openstack CI
 # Contacts: @gix @bl0m1
@@ -88,14 +162,18 @@ tf-elastx_cleanup:
    - ./scripts/openstack-cleanup/main.py
  allow_failure: true

-tf-elastx_ubuntu24-calico:
+tf-elastx_ubuntu20-calico:
  extends: .terraform_apply
  stage: deploy-part1
  when: on_success
+  allow_failure: true
  variables:
    <<: *elastx_variables
+    TF_VERSION: $TERRAFORM_VERSION
    PROVIDER: openstack
+    CLUSTER: $CI_COMMIT_REF_NAME
    ANSIBLE_TIMEOUT: "60"
+    SSH_USER: ubuntu
    TF_VAR_number_of_k8s_masters: "1"
    TF_VAR_number_of_k8s_masters_no_floating_ip: "0"
    TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
@@ -114,5 +192,48 @@ tf-elastx_ubuntu24-calico:
    TF_VAR_az_list_node: '["sto1"]'
    TF_VAR_flavor_k8s_master: 3f73fc93-ec61-4808-88df-2580d94c1a9b    # v1-standard-2
    TF_VAR_flavor_k8s_node: 3f73fc93-ec61-4808-88df-2580d94c1a9b      # v1-standard-2
-    TF_VAR_image: ubuntu-24.04-server-latest
+    TF_VAR_image: ubuntu-20.04-server-latest
    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
+
+# OVH voucher expired, commenting job until things are sorted  out
+
+# tf-ovh_cleanup:
+#  stage: unit-tests
+#  tags: [light]
+#  image: python
+#  environment: ovh
+#  variables:
+#    <<: *ovh_variables
+#  before_script:
+#    - pip install -r scripts/openstack-cleanup/requirements.txt
+#  script:
+#    - ./scripts/openstack-cleanup/main.py
+
+# tf-ovh_ubuntu20-calico:
+#  extends: .terraform_apply
+#  when: on_success
+#  environment: ovh
+#  variables:
+#    <<: *ovh_variables
+#    TF_VERSION: $TERRAFORM_VERSION
+#    PROVIDER: openstack
+#    CLUSTER: $CI_COMMIT_REF_NAME
+#    ANSIBLE_TIMEOUT: "60"
+#    SSH_USER: ubuntu
+#    TF_VAR_number_of_k8s_masters: "0"
+#    TF_VAR_number_of_k8s_masters_no_floating_ip: "1"
+#    TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
+#    TF_VAR_number_of_etcd: "0"
+#    TF_VAR_number_of_k8s_nodes: "0"
+#    TF_VAR_number_of_k8s_nodes_no_floating_ip: "1"
+#    TF_VAR_number_of_gfs_nodes_no_floating_ip: "0"
+#    TF_VAR_number_of_bastions: "0"
+#    TF_VAR_number_of_k8s_masters_no_etcd: "0"
+#    TF_VAR_use_neutron: "0"
+#    TF_VAR_floatingip_pool: "Ext-Net"
+#    TF_VAR_external_net: "6011fbc9-4cbf-46a4-8452-6890a340b60b"
+#    TF_VAR_network_name: "Ext-Net"
+#    TF_VAR_flavor_k8s_master: "defa64c3-bd46-43b4-858a-d93bbae0a229"    # s1-8
+#    TF_VAR_flavor_k8s_node: "defa64c3-bd46-43b4-858a-d93bbae0a229"      # s1-8
+#    TF_VAR_image: "Ubuntu 20.04"
+#    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
--- a/.gitlab-ci/vagrant.yml
+++ b/.gitlab-ci/vagrant.yml
@@ -1,16 +1,20 @@
 ---
-vagrant:
-  extends: .job-moderated
+.vagrant:
+  extends: .testcases
+  needs:
+    - ci-not-authorized
  variables:
    CI_PLATFORM: "vagrant"
    SSH_USER: "vagrant"
    VAGRANT_DEFAULT_PROVIDER: "libvirt"
-    KUBESPRAY_VAGRANT_CONFIG: tests/files/${TESTCASE}.rb
+    KUBESPRAY_VAGRANT_CONFIG: tests/files/${CI_JOB_NAME}.rb
    DOCKER_NAME: vagrant
    VAGRANT_ANSIBLE_TAGS: facts
    VAGRANT_HOME: "$CI_PROJECT_DIR/.vagrant.d"
    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
  tags: [ffci-vm-large]
+  # only: [/^pr-.*$/]
+  # except: ['triggers']
  image: quay.io/kubespray/vm-kubespray-ci:v13
  services: []
  before_script:
@@ -24,26 +28,54 @@ vagrant:
    - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt
    - ./tests/scripts/vagrant_clean.sh
  script:
-    - vagrant up
    - ./tests/scripts/testcases_run.sh
-  after_script:
-    - vagrant destroy -f
  cache:
    key: $CI_JOB_NAME_SLUG
    paths:
      - .vagrant.d/boxes
      - .cache/pip
    policy: pull-push # TODO: change to "pull" when not on main
+
+vagrant_ubuntu24-calico-dual-stack:
  stage: deploy-extended
+  extends: .vagrant
  rules:
-    - if: $PR_LABELS =~ /.*ci-full.*/
+    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
      when: on_success
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_PIPELINE_SCHEDULE_DESCRIPTION == "daily-ci"
+  allow_failure: false
+
+vagrant_ubuntu24-calico-ipv6only-stack:
+  stage: deploy-extended
+  extends: .vagrant
+  rules:
+    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
      when: on_success
-    - when: manual
-      allow_failure: true
-  parallel:
-    matrix:
-      - TESTCASE:
-          - ubuntu24-calico-dual-stack
-          - ubuntu24-calico-ipv6only-stack
+  allow_failure: false
+
+vagrant_ubuntu20-flannel:
+  stage: deploy-part1
+  extends: .vagrant
+  when: on_success
+  allow_failure: false
+
+vagrant_ubuntu20-flannel-collection:
+  stage: deploy-extended
+  extends: .vagrant
+  when: manual
+
+vagrant_ubuntu20-kube-router-sep:
+  stage: deploy-extended
+  extends: .vagrant
+  when: manual
+
+# Service proxy test fails connectivity testing
+vagrant_ubuntu20-kube-router-svc-proxy:
+  stage: deploy-extended
+  extends: .vagrant
+  when: manual
+
+vagrant_fedora39-kube-router:
+  stage: deploy-extended
+  extends: .vagrant
+  when: manual
+# FIXME: this test if broken (perma-failing)
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,7 +1,7 @@
 ---
 repos:
  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v6.0.0
+    rev: v5.0.0
    hooks:
      - id: check-added-large-files
      - id: check-case-conflict
@@ -15,13 +15,13 @@ repos:
      - id: trailing-whitespace

  - repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.37.1
+    rev: v1.35.1
    hooks:
      - id: yamllint
        args: [--strict]

  - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: v0.11.0.1
+    rev: v0.10.0.1
    hooks:
      - id: shellcheck
        args: ["--severity=error"]
@@ -29,7 +29,7 @@ repos:
        files: "\\.sh$"

  - repo: https://github.com/ansible/ansible-lint
-    rev: v25.11.0
+    rev: v25.1.1
    hooks:
      - id: ansible-lint
        additional_dependencies:
@@ -38,7 +38,7 @@ repos:
          - distlib

  - repo: https://github.com/golangci/misspell
-    rev: v0.7.0
+    rev: v0.6.0
    hooks:
      - id: misspell
        exclude: "OWNERS_ALIASES$"
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -40,7 +40,7 @@ Vagrant with VirtualBox or libvirt driver helps you to quickly spin test cluster
 2. The [repo owners](OWNERS) will respond to your issue promptly.
 3. Fork the desired repo, develop and test your code changes.
 4. Install [pre-commit](https://pre-commit.com) and install it in your development repo.
-5. Address any pre-commit validation failures.
+5. Addess any pre-commit validation failures.
 6. Sign the CNCF CLA (<https://git.k8s.io/community/CLA.md#the-contributor-license-agreement>)
 7. Submit a pull request.
 8. Work with the reviewers on their suggestions.
--- a/10
+++ b/10
@@ -1,7 +1,7 @@
 # syntax=docker/dockerfile:1

-# Use immutable image tags rather than mutable tags (like ubuntu:24.04)
-FROM ubuntu:noble-20260113@sha256:cd1dba651b3080c3686ecf4e3c4220f026b521fb76978881737d24f200828b2b
+# Use imutable image tags rather than mutable tags (like ubuntu:22.04)
+FROM ubuntu:22.04@sha256:149d67e29f765f4db62aa52161009e99e389544e25a8f43c8c89d4a445a7ca37

 # Some tools like yamllint need this
 # Pip needs this as well at the moment to install ansible
@@ -29,14 +29,14 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \

 RUN --mount=type=bind,source=requirements.txt,target=requirements.txt \
    --mount=type=cache,sharing=locked,id=pipcache,mode=0777,target=/root/.cache/pip \
-    pip install --break-system-packages --no-compile --no-cache-dir -r requirements.txt \
+    pip install --no-compile --no-cache-dir -r requirements.txt \
    && find /usr -type d -name '*__pycache__' -prune -exec rm -rf {} \;

 SHELL ["/bin/bash", "-o", "pipefail", "-c"]

 RUN OS_ARCHITECTURE=$(dpkg --print-architecture) \
-    && curl -L "https://dl.k8s.io/release/v1.34.3/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
-    && echo "$(curl -L "https://dl.k8s.io/release/v1.34.3/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
+    && curl -L "https://dl.k8s.io/release/v1.32.3/bin/linux/${OS_ARCHITECTURE}/kubectl" -o /usr/local/bin/kubectl \
+    && echo "$(curl -L "https://dl.k8s.io/release/v1.32.3/bin/linux/${OS_ARCHITECTURE}/kubectl.sha256")" /usr/local/bin/kubectl | sha256sum --check \
    && chmod a+x /usr/local/bin/kubectl

 COPY *.yml ./
--- a/14
+++ b/14
@@ -1,13 +1,17 @@
 aliases:
  kubespray-approvers:
-    - ant31
+    - cristicalin
+    - floryut
+    - liupeng0518
    - mzaian
-    - tico88612
-    - vannten
+    - oomichi
    - yankay
+    - ant31
+    - vannten
  kubespray-reviewers:
    - cyclinder
    - erikjiang
+    - mrfreezeex
    - mzaian
    - tico88612
    - vannten
@@ -15,12 +19,8 @@ aliases:
  kubespray-emeritus_approvers:
    - atoms
    - chadswen
-    - cristicalin
-    - floryut
-    - liupeng0518
    - luckysb
    - mattymo
    - miouge1
-    - oomichi
    - riverzhang
    - woopstar
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@ Ensure you have installed Docker then
 ```ShellSession
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
  --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.30.0 bash
+  quay.io/kubespray/kubespray:v2.27.0 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```
@@ -87,15 +87,15 @@ vagrant up
 ## Supported Linux Distributions

 - **Flatcar Container Linux by Kinvolk**
- **Debian** Bookworm, Bullseye, Trixie
- **Ubuntu** 22.04, 24.04
- **CentOS Stream / RHEL** [9, 10](docs/operating_systems/rhel.md#rhel-8)
+- **Debian** Bookworm, Bullseye
+- **Ubuntu** 20.04, 22.04, 24.04
+- **CentOS/RHEL** [8, 9](docs/operating_systems/rhel.md#rhel-8)
 - **Fedora** 39, 40
 - **Fedora CoreOS** (see [fcos Note](docs/operating_systems/fcos.md))
 - **openSUSE** Leap 15.x/Tumbleweed
- **Oracle Linux** [9, 10](docs/operating_systems/rhel.md#rhel-8)
- **Alma Linux** [9, 10](docs/operating_systems/rhel.md#rhel-8)
- **Rocky Linux** [9, 10](docs/operating_systems/rhel.md#rhel-8) (experimental in 10: see [Rocky Linux 10 notes](docs/operating_systems/rhel.md#rocky-linux-10))
+- **Oracle Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
+- **Alma Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
+- **Rocky Linux** [8, 9](docs/operating_systems/rhel.md#rhel-8)
 - **Kylin Linux Advanced Server V10** (experimental: see [kylin linux notes](docs/operating_systems/kylinlinux.md))
 - **Amazon Linux 2** (experimental: see [amazon linux notes](docs/operating_systems/amazonlinux.md))
 - **UOS Linux** (experimental: see [uos linux notes](docs/operating_systems/uoslinux.md))
@@ -111,34 +111,37 @@ Note:
 <!-- BEGIN ANSIBLE MANAGED BLOCK -->

 - Core
-  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.34.3
-  - [etcd](https://github.com/etcd-io/etcd) 3.5.26
-  - [docker](https://www.docker.com/) 28.3
-  - [containerd](https://containerd.io/) 2.2.1
-  - [cri-o](http://cri-o.io/) 1.34.4 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
+  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.32.3
+  - [etcd](https://github.com/etcd-io/etcd) 3.5.16
+  - [docker](https://www.docker.com/) 28.0
+  - [containerd](https://containerd.io/) 2.0.4
+  - [cri-o](http://cri-o.io/) 1.32.0 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
-  - [cni-plugins](https://github.com/containernetworking/plugins) 1.8.0
-  - [calico](https://github.com/projectcalico/calico) 3.30.6
-  - [cilium](https://github.com/cilium/cilium) 1.18.6
-  - [flannel](https://github.com/flannel-io/flannel) 0.27.3
+  - [cni-plugins](https://github.com/containernetworking/plugins) 1.4.1
+  - [calico](https://github.com/projectcalico/calico) 3.29.3
+  - [cilium](https://github.com/cilium/cilium) 1.15.9
+  - [flannel](https://github.com/flannel-io/flannel) 0.22.0
  - [kube-ovn](https://github.com/alauda/kube-ovn) 1.12.21
  - [kube-router](https://github.com/cloudnativelabs/kube-router) 2.1.1
-  - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) 4.2.2
-  - [kube-vip](https://github.com/kube-vip/kube-vip) 1.0.3
+  - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) 4.1.0
+  - [weave](https://github.com/rajch/weave) 2.8.7
+  - [kube-vip](https://github.com/kube-vip/kube-vip) 0.8.0
 - Application
  - [cert-manager](https://github.com/jetstack/cert-manager) 1.15.3
-  - [coredns](https://github.com/coredns/coredns) 1.12.1
-  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) 1.13.3
+  - [coredns](https://github.com/coredns/coredns) 1.11.3
+  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) 1.12.1
  - [argocd](https://argoproj.github.io/) 2.14.5
-  - [helm](https://helm.sh/) 3.18.4
+  - [helm](https://helm.sh/) 3.16.4
  - [metallb](https://metallb.universe.tf/) 0.13.9
  - [registry](https://github.com/distribution/distribution) 2.8.1
 - Storage Plugin
+  - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) 2.1.0-k8s1.11
+  - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) 2.1.1-k8s1.11
  - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) 0.5.0
  - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) 1.10.0
  - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) 1.30.0
  - [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) 1.9.2
-  - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) 0.0.32
+  - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) 0.0.24
  - [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) 2.5.0
  - [node-feature-discovery](https://github.com/kubernetes-sigs/node-feature-discovery) 0.16.4

@@ -182,6 +185,9 @@ You can choose among ten network plugins. (default: `calico`, except Vagrant use

 - [cilium](http://docs.cilium.io/en/latest/): layer 3/4 networking (as well as layer 7 to protect and secure application protocols), supports dynamic insertion of BPF bytecode into the Linux kernel to implement security services, networking and visibility logic.

+- [weave](docs/CNI/weave.md): Weave is a lightweight container overlay network that doesn't require an external K/V database cluster.
+    (Please refer to `weave` [troubleshooting documentation](https://www.weave.works/docs/net/latest/troubleshooting/)).
+
 - [kube-ovn](docs/CNI/kube-ovn.md): Kube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It offers an advanced Container Network Fabric for Enterprises.

 - [kube-router](docs/CNI/kube-router.md): Kube-router is a L3 CNI for Kubernetes networking aiming to provide operational
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -15,7 +15,7 @@ The Kubespray Project is released on an as-needed basis. The process is as follo
 1. The release issue is closed
 1. An announcement email is sent to `dev@kubernetes.io` with the subject `[ANNOUNCE] Kubespray $VERSION is released`
 1. The topic of the #kubespray channel is updated with `vX.Y.Z is released! | ...`
-1. Create/Update Issue for upgrading kubernetes and [k8s-conformance](https://github.com/cncf/k8s-conformance)
+1. Create/Update Issue for upgradeing kubernetes and [k8s-conformance](https://github.com/cncf/k8s-conformance)

 ## Major/minor releases and milestones

--- a/29
+++ b/29
@@ -4,8 +4,6 @@
 # For help on using kubespray with vagrant, check out docs/developers/vagrant.md

 require 'fileutils'
-require 'ipaddr'
-require 'socket'

 Vagrant.require_version ">= 2.0.0"

@@ -101,33 +99,6 @@ $extra_vars ||= {}

 host_vars = {}

-def collect_networks(subnet, subnet_ipv6)
-  Socket.getifaddrs.filter_map do |iface|
-    next unless iface&.netmask&.ip_address && iface.addr
-
-    is_ipv6 = iface.addr.ipv6?
-    ip      = IPAddr.new(iface.addr.ip_address.split('%').first)
-    ip_test = is_ipv6 ? IPAddr.new("#{subnet_ipv6}::0") : IPAddr.new("#{subnet}.0")
-
-    prefix  = IPAddr.new(iface.netmask.ip_address).to_i.to_s(2).count('1')
-    network = ip.mask(prefix)
-
-    [IPAddr.new("#{network}/#{prefix}"), ip_test]
-  end
-end
-
-def subnet_in_use?(network_ips)
-  network_ips.any? { |net, test_ip| net.include?(test_ip) && test_ip != net }
-end
-
-network_ips = collect_networks($subnet, $subnet_ipv6)
-
-if subnet_in_use?(network_ips)
-  puts "Invalid subnet provided, subnet is already in use: #{$subnet}.0"
-  puts "Subnets in use: #{network_ips.inspect}"
-  exit 1
-end
-
 # throw error if os is not supported
 if ! SUPPORTED_OS.key?($os)
  puts "Unsupported OS: #{$os}"
--- a/ansible.cfg
+++ b/ansible.cfg
@@ -15,7 +15,7 @@ timeout = 300
 stdout_callback = default
 display_skipped_hosts = no
 library = ./library
-callbacks_enabled = profile_tasks
+callbacks_enabled = profile_tasks,ara_default
 roles_path = roles:$VIRTUAL_ENV/usr/local/share/kubespray/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:/usr/share/kubespray/roles
 deprecation_warnings=False
 inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo, .creds, .gpg
--- a/contrib/collection.sh
+++ b/contrib/collection.sh
@@ -1,9 +0,0 @@
-#!/bin/bash -eux
-# Install collection from source assuming dependencies are present.
-# Run in SemaphoreUI this bash script can install Kubespray from the repo
-NAMESPACE=kubernetes_sigs
-COLLECTION=kubespray
-MY_VER=$(grep '^version:' galaxy.yml|cut -d: -f2|sed 's/ //')
-
-ansible-galaxy collection build --force --output-path .
-ansible-galaxy collection install --offline --force $NAMESPACE-$COLLECTION-$MY_VER.tar.gz
--- a/contrib/kvm-setup/README.md
+++ b/contrib/kvm-setup/README.md
@@ -0,0 +1,11 @@
+# Kubespray on KVM Virtual Machines hypervisor preparation
+
+A simple playbook to ensure your system has the right settings to enable Kubespray
+deployment on VMs.
+
+This playbook does not create Virtual Machines, nor does it run Kubespray itself.
+
+## User creation
+
+If you want to create a user for running Kubespray deployment, you should specify
+both `k8s_deployment_user` and `k8s_deployment_user_pkey_path`.
--- a/contrib/kvm-setup/group_vars/all
+++ b/contrib/kvm-setup/group_vars/all
@@ -0,0 +1,2 @@
+#k8s_deployment_user: kubespray
+#k8s_deployment_user_pkey_path: /tmp/ssh_rsa
--- a/contrib/kvm-setup/kvm-setup.yml
+++ b/contrib/kvm-setup/kvm-setup.yml
@@ -0,0 +1,9 @@
+---
+- name: Prepare Hypervisor to later install kubespray VMs
+  hosts: localhost
+  gather_facts: false
+  become: true
+  vars:
+    bootstrap_os: none
+  roles:
+    - { role: kvm-setup }
--- a/contrib/kvm-setup/roles/kvm-setup/tasks/main.yml
+++ b/contrib/kvm-setup/roles/kvm-setup/tasks/main.yml
@@ -0,0 +1,30 @@
+---
+
+- name: Install required packages
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - bind-utils
+    - ntp
+  when: ansible_os_family == "RedHat"
+
+- name: Install required packages
+  apt:
+    upgrade: true
+    update_cache: true
+    cache_valid_time: 3600
+    name: "{{ item }}"
+    state: present
+    install_recommends: false
+  with_items:
+    - dnsutils
+    - ntp
+  when: ansible_os_family == "Debian"
+
+- name: Create deployment user if required
+  include_tasks: user.yml
+  when: k8s_deployment_user is defined
+
+- name: Set proper sysctl values
+  import_tasks: sysctl.yml
--- a/contrib/kvm-setup/roles/kvm-setup/tasks/sysctl.yml
+++ b/contrib/kvm-setup/roles/kvm-setup/tasks/sysctl.yml
@@ -0,0 +1,46 @@
+---
+- name: Load br_netfilter module
+  community.general.modprobe:
+    name: br_netfilter
+    state: present
+  register: br_netfilter
+
+- name: Add br_netfilter into /etc/modules
+  lineinfile:
+    dest: /etc/modules
+    state: present
+    line: 'br_netfilter'
+  when: br_netfilter is defined and ansible_os_family == 'Debian'
+
+- name: Add br_netfilter into /etc/modules-load.d/kubespray.conf
+  copy:
+    dest: /etc/modules-load.d/kubespray.conf
+    content: |-
+      ### This file is managed by Ansible
+      br-netfilter
+    owner: root
+    group: root
+    mode: "0644"
+  when: br_netfilter is defined
+
+
+- name: Enable net.ipv4.ip_forward in sysctl
+  ansible.posix.sysctl:
+    name: net.ipv4.ip_forward
+    value: 1
+    sysctl_file: "{{ sysctl_file_path }}"
+    state: present
+    reload: true
+
+- name: Set bridge-nf-call-{arptables,iptables} to 0
+  ansible.posix.sysctl:
+    name: "{{ item }}"
+    state: present
+    value: 0
+    sysctl_file: "{{ sysctl_file_path }}"
+    reload: true
+  with_items:
+    - net.bridge.bridge-nf-call-arptables
+    - net.bridge.bridge-nf-call-ip6tables
+    - net.bridge.bridge-nf-call-iptables
+  when: br_netfilter is defined
--- a/contrib/kvm-setup/roles/kvm-setup/tasks/user.yml
+++ b/contrib/kvm-setup/roles/kvm-setup/tasks/user.yml
@@ -0,0 +1,47 @@
+---
+- name: Create user {{ k8s_deployment_user }}
+  user:
+    name: "{{ k8s_deployment_user }}"
+    groups: adm
+    shell: /bin/bash
+
+- name: Ensure that .ssh exists
+  file:
+    path: "/home/{{ k8s_deployment_user }}/.ssh"
+    state: directory
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+    mode: "0700"
+
+- name: Configure sudo for deployment user
+  copy:
+    content: |
+      %{{ k8s_deployment_user }} ALL=(ALL) NOPASSWD: ALL
+    dest: "/etc/sudoers.d/55-k8s-deployment"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Write private SSH key
+  copy:
+    src: "{{ k8s_deployment_user_pkey_path }}"
+    dest: "/home/{{ k8s_deployment_user }}/.ssh/id_rsa"
+    mode: "0400"
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+  when: k8s_deployment_user_pkey_path is defined
+
+- name: Write public SSH key
+  shell: "ssh-keygen -y -f /home/{{ k8s_deployment_user }}/.ssh/id_rsa \
+            > /home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+  args:
+    creates: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+  when: k8s_deployment_user_pkey_path is defined
+
+- name: Fix ssh-pub-key permissions
+  file:
+    path: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+    mode: "0600"
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+  when: k8s_deployment_user_pkey_path is defined
--- a/contrib/misc/clusteradmin-rbac.yml
+++ b/contrib/misc/clusteradmin-rbac.yml
@@ -0,0 +1,15 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubernetes-dashboard
+  labels:
+    k8s-app: kubernetes-dashboard
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: kubernetes-dashboard
+  namespace: kube-system
--- a/contrib/mitogen/mitogen.yml
+++ b/contrib/mitogen/mitogen.yml
@@ -0,0 +1,51 @@
+---
+- name: Check ansible version
+  import_playbook: kubernetes_sigs.kubespray.ansible_version
+
+- name: Install mitogen
+  hosts: localhost
+  strategy: linear
+  vars:
+    mitogen_version: 0.3.2
+    mitogen_url: https://github.com/mitogen-hq/mitogen/archive/refs/tags/v{{ mitogen_version }}.tar.gz
+    ansible_connection: local
+  tasks:
+    - name: Create mitogen plugin dir
+      file:
+        path: "{{ item }}"
+        state: directory
+        mode: "0755"
+      become: false
+      loop:
+        - "{{ playbook_dir }}/plugins/mitogen"
+        - "{{ playbook_dir }}/dist"
+
+    - name: Download mitogen release
+      get_url:
+        url: "{{ mitogen_url }}"
+        dest: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.tar.gz"
+        validate_certs: true
+        mode: "0644"
+
+    - name: Extract archive
+      unarchive:
+        src: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.tar.gz"
+        dest: "{{ playbook_dir }}/dist/"
+
+    - name: Copy plugin
+      ansible.posix.synchronize:
+        src: "{{ playbook_dir }}/dist/mitogen-{{ mitogen_version }}/"
+        dest: "{{ playbook_dir }}/plugins/mitogen"
+
+    - name: Add strategy to ansible.cfg
+      community.general.ini_file:
+        path: ansible.cfg
+        mode: "0644"
+        section: "{{ item.section | d('defaults') }}"
+        option: "{{ item.option }}"
+        value: "{{ item.value }}"
+      with_items:
+        - option: strategy
+          value: mitogen_linear
+        - option: strategy_plugins
+          value: plugins/mitogen/ansible_mitogen/plugins/strategy
--- a/contrib/offline/README.md
+++ b/contrib/offline/README.md
@@ -31,7 +31,7 @@ manage-offline-container-images.sh   register

 ## generate_list.sh

-This script generates the list of downloaded files and the list of container images by `roles/kubespray_defaults/defaults/main/download.yml` file.
+This script generates the list of downloaded files and the list of container images by `roles/kubespray-defaults/defaults/main/download.yml` file.

 Run this script will execute `generate_list.yml` playbook in kubespray root directory and generate four files,
 all downloaded files url in files.list, all container images in images.list, jinja2 templates in *.template.
--- a/contrib/offline/generate_list.sh
+++ b/contrib/offline/generate_list.sh
@@ -5,7 +5,7 @@ CURRENT_DIR=$(cd $(dirname $0); pwd)
 TEMP_DIR="${CURRENT_DIR}/temp"
 REPO_ROOT_DIR="${CURRENT_DIR%/contrib/offline}"

-: ${DOWNLOAD_YML:="roles/kubespray_defaults/defaults/main/download.yml"}
+: ${DOWNLOAD_YML:="roles/kubespray-defaults/defaults/main/download.yml"}

 mkdir -p ${TEMP_DIR}

@@ -19,7 +19,7 @@ sed -n '/^downloads:/,/download_defaults:/p' ${REPO_ROOT_DIR}/${DOWNLOAD_YML} \
    | sed 'N;s#\n# #g' | tr ' ' ':' | sed 's/\"//g' > ${TEMP_DIR}/images.list.template

 # add kube-* images to images list template
-# Those container images are downloaded by kubeadm, then roles/kubespray_defaults/defaults/main/download.yml
+# Those container images are downloaded by kubeadm, then roles/kubespray-defaults/defaults/main/download.yml
 # doesn't contain those images. That is reason why here needs to put those images into the
 # list separately.
 KUBE_IMAGES="kube-apiserver kube-controller-manager kube-scheduler kube-proxy"
--- a/contrib/offline/generate_list.yml
+++ b/contrib/offline/generate_list.yml
@@ -5,7 +5,7 @@

  roles:
    # Just load default variables from roles.
-    - role: kubespray_defaults
+    - role: kubespray-defaults
      when: false
    - role: download
      when: false
--- a/contrib/offline/manage-offline-container-images.sh
+++ b/contrib/offline/manage-offline-container-images.sh
@@ -20,6 +20,7 @@ function create_container_image_tar() {

 		kubectl describe cronjobs,jobs,pods --all-namespaces | grep " Image:" | awk '{print $2}' | sort | uniq > "${IMAGES}"
 		# NOTE: etcd and pause cannot be seen as pods.
+		# The pause image is used for --pod-infra-container-image option of kubelet.
 		kubectl cluster-info dump | grep -E "quay.io/coreos/etcd:|registry.k8s.io/pause:" | sed s@\"@@g >> "${IMAGES}"
 	else
 		echo "Getting images from file \"${IMAGES_FROM_FILE}\""
@@ -35,7 +36,7 @@ function create_container_image_tar() {
 	mkdir  ${IMAGE_DIR}
 	cd     ${IMAGE_DIR}

-	sudo --preserve-env=http_proxy,https_proxy,no_proxy ${runtime} pull registry:latest
+	sudo ${runtime} pull registry:latest
 	sudo ${runtime} save -o registry-latest.tar registry:latest

 	while read -r image
@@ -44,7 +45,7 @@ function create_container_image_tar() {
 		set +e
 		for step in $(seq 1 ${RETRY_COUNT})
 		do
-			sudo --preserve-env=http_proxy,https_proxy,no_proxy ${runtime} pull ${image}
+			sudo ${runtime} pull ${image}
 			if [ $? -eq 0 ]; then
 				break
 			fi
@@ -126,7 +127,7 @@ function register_container_images() {

 	tar -zxvf ${IMAGE_TAR_FILE}

-	if ${create_registry}; then
+	if [ "${create_registry}" ]; then
 		sudo ${runtime} load -i ${IMAGE_DIR}/registry-latest.tar
 		set +e

@@ -147,7 +148,7 @@ function register_container_images() {
 		if [ "${org_image}" == "ID:" ]; then
 		  org_image=$(echo "${load_image}"  | awk '{print $4}')
 		fi
-		image_id=$(sudo ${runtime} image inspect --format "{{.Id}}" "${org_image}")
+		image_id=$(sudo ${runtime} image inspect ${org_image} | grep "\"Id\":" | awk -F: '{print $3}'| sed s/'\",'//)
 		if [ -z "${file_name}" ]; then
 			echo "Failed to get file_name for line ${line}"
 			exit 1
--- a/contrib/offline/manage-offline-files.sh
+++ b/contrib/offline/manage-offline-files.sh
@@ -41,7 +41,7 @@ fi

 sudo "${runtime}" container inspect nginx >/dev/null 2>&1
 if [ $? -ne 0 ]; then
-    sudo --preserve-env=http_proxy,https_proxy,no_proxy "${runtime}" run \
+    sudo "${runtime}" run \
        --restart=always -d -p ${NGINX_PORT}:80 \
        --volume "${OFFLINE_FILES_DIR}":/usr/share/nginx/html/download \
        --volume "${CURRENT_DIR}"/nginx.conf:/etc/nginx/nginx.conf \
--- a/contrib/terraform/aws/create-infrastructure.tf
+++ b/contrib/terraform/aws/create-infrastructure.tf
@@ -1,11 +1,5 @@
 terraform {
  required_version = ">= 0.12.0"
-  required_providers {
-    aws = {
-      source  = "hashicorp/aws"
-      version = "~> 5.0"
-    }
-  }
 }

 provider "aws" {
--- a/contrib/terraform/equinix/README.md
+++ b/contrib/terraform/equinix/README.md
@@ -0,0 +1,246 @@
+# Kubernetes on Equinix Metal with Terraform
+
+Provision a Kubernetes cluster with [Terraform](https://www.terraform.io) on
+[Equinix Metal](https://metal.equinix.com) ([formerly Packet](https://blog.equinix.com/blog/2020/10/06/equinix-metal-metal-and-more/)).
+
+## Status
+
+This will install a Kubernetes cluster on Equinix Metal. It should work in all locations and on most server types.
+
+## Approach
+
+The terraform configuration inspects variables found in
+[variables.tf](variables.tf) to create resources in your Equinix Metal project.
+There is a [python script](../terraform.py) that reads the generated`.tfstate`
+file to generate a dynamic inventory that is consumed by [cluster.yml](../../../cluster.yml)
+to actually install Kubernetes with Kubespray.
+
+### Kubernetes Nodes
+
+You can create many different kubernetes topologies by setting the number of
+different classes of hosts.
+
+- Master nodes with etcd: `number_of_k8s_masters` variable
+- Master nodes without etcd: `number_of_k8s_masters_no_etcd` variable
+- Standalone etcd hosts: `number_of_etcd` variable
+- Kubernetes worker nodes: `number_of_k8s_nodes` variable
+
+Note that the Ansible script will report an invalid configuration if you wind up
+with an *even number* of etcd instances since that is not a valid configuration. This
+restriction includes standalone etcd nodes that are deployed in a cluster along with
+master nodes with etcd replicas. As an example, if you have three master nodes with
+etcd replicas and three standalone etcd nodes, the script will fail since there are
+now six total etcd replicas.
+
+## Requirements
+
+- [Install Terraform](https://www.terraform.io/intro/getting-started/install.html)
+- [Install Ansible dependencies](/docs/ansible/ansible.md#installing-ansible)
+- Account with Equinix Metal
+- An SSH key pair
+
+## SSH Key Setup
+
+An SSH keypair is required so Ansible can access the newly provisioned nodes (Equinix Metal hosts). By default, the public SSH key defined in cluster.tfvars will be installed in authorized_key on the newly provisioned nodes (~/.ssh/id_rsa.pub). Terraform will upload this public key and then it will be distributed out to all the nodes. If you have already set this public key in Equinix Metal (i.e. via the portal), then set the public keyfile name in cluster.tfvars to blank to prevent the duplicate key from being uploaded which will cause an error.
+
+If you don't already have a keypair generated (~/.ssh/id_rsa and ~/.ssh/id_rsa.pub), then a new keypair can be generated with the command:
+
+```ShellSession
+ssh-keygen -f ~/.ssh/id_rsa
+```
+
+## Terraform
+
+Terraform will be used to provision all of the Equinix Metal resources with base software as appropriate.
+
+### Configuration
+
+#### Inventory files
+
+Create an inventory directory for your cluster by copying the existing sample and linking the `hosts` script (used to build the inventory based on Terraform state):
+
+```ShellSession
+cp -LRp contrib/terraform/equinix/sample-inventory inventory/$CLUSTER
+cd inventory/$CLUSTER
+ln -s ../../contrib/terraform/equinix/hosts
+```
+
+This will be the base for subsequent Terraform commands.
+
+#### Equinix Metal API access
+
+Your Equinix Metal API key must be available in the `METAL_AUTH_TOKEN` environment variable.
+This key is typically stored outside of the code repo since it is considered secret.
+If someone gets this key, they can startup/shutdown hosts in your project!
+
+For more information on how to generate an API key or find your project ID, please see
+[Accounts Index](https://metal.equinix.com/developers/docs/accounts/).
+
+The Equinix Metal Project ID associated with the key will be set later in `cluster.tfvars`.
+
+For more information about the API, please see [Equinix Metal API](https://metal.equinix.com/developers/api/).
+
+For more information about terraform provider authentication, please see [the equinix provider documentation](https://registry.terraform.io/providers/equinix/equinix/latest/docs).
+
+Example:
+
+```ShellSession
+export METAL_AUTH_TOKEN="Example-API-Token"
+```
+
+Note that to deploy several clusters within the same project you need to use [terraform workspace](https://www.terraform.io/docs/state/workspaces.html#using-workspaces).
+
+#### Cluster variables
+
+The construction of the cluster is driven by values found in
+[variables.tf](variables.tf).
+
+For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
+
+The `cluster_name` is used to set a tag on each server deployed as part of this cluster.
+This helps when identifying which hosts are associated with each cluster.
+
+While the defaults in variables.tf will successfully deploy a cluster, it is recommended to set the following values:
+
+- cluster_name = the name of the inventory directory created above as $CLUSTER
+- equinix_metal_project_id = the Equinix Metal Project ID associated with the Equinix Metal API token above
+
+#### Enable localhost access
+
+Kubespray will pull down a Kubernetes configuration file to access this cluster by enabling the
+`kubeconfig_localhost: true` in the Kubespray configuration.
+
+Edit `inventory/$CLUSTER/group_vars/k8s_cluster/k8s_cluster.yml` and comment back in the following line and change from `false` to `true`:
+`\# kubeconfig_localhost: false`
+becomes:
+`kubeconfig_localhost: true`
+
+Once the Kubespray playbooks are run, a Kubernetes configuration file will be written to the local host at `inventory/$CLUSTER/artifacts/admin.conf`
+
+#### Terraform state files
+
+In the cluster's inventory folder, the following files might be created (either by Terraform
+or manually), to prevent you from pushing them accidentally they are in a
+`.gitignore` file in the `contrib/terraform/equinix` directory :
+
+- `.terraform`
+- `.tfvars`
+- `.tfstate`
+- `.tfstate.backup`
+- `.lock.hcl`
+
+You can still add them manually if you want to.
+
+### Initialization
+
+Before Terraform can operate on your cluster you need to install the required
+plugins. This is accomplished as follows:
+
+```ShellSession
+cd inventory/$CLUSTER
+terraform -chdir=../../contrib/terraform/metal init -var-file=cluster.tfvars
+```
+
+This should finish fairly quickly telling you Terraform has successfully initialized and loaded necessary modules.
+
+### Provisioning cluster
+
+You can apply the Terraform configuration to your cluster with the following command
+issued from your cluster's inventory directory (`inventory/$CLUSTER`):
+
+```ShellSession
+terraform -chdir=../../contrib/terraform/equinix apply -var-file=cluster.tfvars
+export ANSIBLE_HOST_KEY_CHECKING=False
+ansible-playbook -i hosts ../../cluster.yml
+```
+
+### Destroying cluster
+
+You can destroy your new cluster with the following command issued from the cluster's inventory directory:
+
+```ShellSession
+terraform -chdir=../../contrib/terraform/equinix destroy -var-file=cluster.tfvars
+```
+
+If you've started the Ansible run, it may also be a good idea to do some manual cleanup:
+
+- Remove SSH keys from the destroyed cluster from your `~/.ssh/known_hosts` file
+- Clean up any temporary cache files: `rm /tmp/$CLUSTER-*`
+
+### Debugging
+
+You can enable debugging output from Terraform by setting `TF_LOG` to `DEBUG` before running the Terraform command.
+
+## Ansible
+
+### Node access
+
+#### SSH
+
+Ensure your local ssh-agent is running and your ssh key has been added. This
+step is required by the terraform provisioner:
+
+```ShellSession
+eval $(ssh-agent -s)
+ssh-add ~/.ssh/id_rsa
+```
+
+If you have deployed and destroyed a previous iteration of your cluster, you will need to clear out any stale keys from your SSH "known hosts" file ( `~/.ssh/known_hosts`).
+
+#### Test access
+
+Make sure you can connect to the hosts.  Note that Flatcar Container Linux by Kinvolk will have a state `FAILED` due to Python not being present.  This is okay, because Python will be installed during bootstrapping, so long as the hosts are not `UNREACHABLE`.
+
+```ShellSession
+$ ansible -i inventory/$CLUSTER/hosts -m ping all
+example-k8s_node-1 | SUCCESS => {
+    "changed": false,
+    "ping": "pong"
+}
+example-etcd-1 | SUCCESS => {
+    "changed": false,
+    "ping": "pong"
+}
+example-k8s-master-1 | SUCCESS => {
+    "changed": false,
+    "ping": "pong"
+}
+```
+
+If it fails try to connect manually via SSH.  It could be something as simple as a stale host key.
+
+### Deploy Kubernetes
+
+```ShellSession
+ansible-playbook --become -i inventory/$CLUSTER/hosts cluster.yml
+```
+
+This will take some time as there are many tasks to run.
+
+## Kubernetes
+
+### Set up kubectl
+
+- [Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on the localhost.
+- Verify that Kubectl runs correctly
+
+```ShellSession
+kubectl version
+```
+
+- Verify that the Kubernetes configuration file has been copied over
+
+```ShellSession
+cat inventory/alpha/$CLUSTER/admin.conf
+```
+
+- Verify that all the nodes are running correctly.
+
+```ShellSession
+kubectl version
+kubectl --kubeconfig=inventory/$CLUSTER/artifacts/admin.conf  get nodes
+```
+
+## What's next
+
+Try out your new Kubernetes cluster with the [Hello Kubernetes service](https://kubernetes.io/docs/tasks/access-application-cluster/service-access-application-cluster/).
--- a/contrib/terraform/equinix/hosts
+++ b/contrib/terraform/equinix/hosts
@@ -0,0 +1 @@
+../terraform.py
--- a/contrib/terraform/equinix/kubespray.tf
+++ b/contrib/terraform/equinix/kubespray.tf
@@ -0,0 +1,57 @@
+resource "equinix_metal_ssh_key" "k8s" {
+  count      = var.public_key_path != "" ? 1 : 0
+  name       = "kubernetes-${var.cluster_name}"
+  public_key = chomp(file(var.public_key_path))
+}
+
+resource "equinix_metal_device" "k8s_master" {
+  depends_on = [equinix_metal_ssh_key.k8s]
+
+  count            = var.number_of_k8s_masters
+  hostname         = "${var.cluster_name}-k8s-master-${count.index + 1}"
+  plan             = var.plan_k8s_masters
+  metro            = var.metro
+  operating_system = var.operating_system
+  billing_cycle    = var.billing_cycle
+  project_id       = var.equinix_metal_project_id
+  tags             = ["cluster-${var.cluster_name}", "k8s_cluster", "kube_control_plane", "etcd", "kube_node"]
+}
+
+resource "equinix_metal_device" "k8s_master_no_etcd" {
+  depends_on = [equinix_metal_ssh_key.k8s]
+
+  count            = var.number_of_k8s_masters_no_etcd
+  hostname         = "${var.cluster_name}-k8s-master-${count.index + 1}"
+  plan             = var.plan_k8s_masters_no_etcd
+  metro            = var.metro
+  operating_system = var.operating_system
+  billing_cycle    = var.billing_cycle
+  project_id       = var.equinix_metal_project_id
+  tags             = ["cluster-${var.cluster_name}", "k8s_cluster", "kube_control_plane"]
+}
+
+resource "equinix_metal_device" "k8s_etcd" {
+  depends_on = [equinix_metal_ssh_key.k8s]
+
+  count            = var.number_of_etcd
+  hostname         = "${var.cluster_name}-etcd-${count.index + 1}"
+  plan             = var.plan_etcd
+  metro            = var.metro
+  operating_system = var.operating_system
+  billing_cycle    = var.billing_cycle
+  project_id       = var.equinix_metal_project_id
+  tags             = ["cluster-${var.cluster_name}", "etcd"]
+}
+
+resource "equinix_metal_device" "k8s_node" {
+  depends_on = [equinix_metal_ssh_key.k8s]
+
+  count            = var.number_of_k8s_nodes
+  hostname         = "${var.cluster_name}-k8s-node-${count.index + 1}"
+  plan             = var.plan_k8s_nodes
+  metro            = var.metro
+  operating_system = var.operating_system
+  billing_cycle    = var.billing_cycle
+  project_id       = var.equinix_metal_project_id
+  tags             = ["cluster-${var.cluster_name}", "k8s_cluster", "kube_node"]
+}
--- a/contrib/terraform/equinix/output.tf
+++ b/contrib/terraform/equinix/output.tf
@@ -0,0 +1,15 @@
+output "k8s_masters" {
+  value = equinix_metal_device.k8s_master.*.access_public_ipv4
+}
+
+output "k8s_masters_no_etc" {
+  value = equinix_metal_device.k8s_master_no_etcd.*.access_public_ipv4
+}
+
+output "k8s_etcds" {
+  value = equinix_metal_device.k8s_etcd.*.access_public_ipv4
+}
+
+output "k8s_nodes" {
+  value = equinix_metal_device.k8s_node.*.access_public_ipv4
+}
--- a/contrib/terraform/equinix/provider.tf
+++ b/contrib/terraform/equinix/provider.tf
@@ -0,0 +1,17 @@
+terraform {
+  required_version = ">= 1.0.0"
+
+  provider_meta "equinix" {
+    module_name = "kubespray"
+  }
+  required_providers {
+    equinix = {
+      source  = "equinix/equinix"
+      version = "1.24.0"
+    }
+  }
+}
+
+# Configure the Equinix Metal Provider
+provider "equinix" {
+}
--- a/contrib/terraform/equinix/sample-inventory/cluster.tfvars
+++ b/contrib/terraform/equinix/sample-inventory/cluster.tfvars
@@ -0,0 +1,35 @@
+# your Kubernetes cluster name here
+cluster_name = "mycluster"
+
+# Your Equinix Metal project ID. See https://metal.equinix.com/developers/docs/accounts/
+equinix_metal_project_id = "Example-Project-Id"
+
+# The public SSH key to be uploaded into authorized_keys in bare metal Equinix Metal nodes provisioned
+# leave this value blank if the public key is already setup in the Equinix Metal project
+# Terraform will complain if the public key is setup in Equinix Metal
+public_key_path = "~/.ssh/id_rsa.pub"
+
+# Equinix interconnected bare metal across our global metros.
+metro = "da"
+
+# operating_system
+operating_system = "ubuntu_22_04"
+
+# standalone etcds
+number_of_etcd = 0
+
+plan_etcd = "t1.small.x86"
+
+# masters
+number_of_k8s_masters = 1
+
+number_of_k8s_masters_no_etcd = 0
+
+plan_k8s_masters = "t1.small.x86"
+
+plan_k8s_masters_no_etcd = "t1.small.x86"
+
+# nodes
+number_of_k8s_nodes = 2
+
+plan_k8s_nodes = "t1.small.x86"
--- a/contrib/terraform/equinix/sample-inventory/group_vars
+++ b/contrib/terraform/equinix/sample-inventory/group_vars
@@ -0,0 +1 @@
+../../../../inventory/sample/group_vars
--- a/contrib/terraform/equinix/variables.tf
+++ b/contrib/terraform/equinix/variables.tf
@@ -0,0 +1,56 @@
+variable "cluster_name" {
+  default = "kubespray"
+}
+
+variable "equinix_metal_project_id" {
+  description = "Your Equinix Metal project ID. See https://metal.equinix.com/developers/docs/accounts/"
+}
+
+variable "operating_system" {
+  default = "ubuntu_22_04"
+}
+
+variable "public_key_path" {
+  description = "The path of the ssh pub key"
+  default     = "~/.ssh/id_rsa.pub"
+}
+
+variable "billing_cycle" {
+  default = "hourly"
+}
+
+variable "metro" {
+  default = "da"
+}
+
+variable "plan_k8s_masters" {
+  default = "c3.small.x86"
+}
+
+variable "plan_k8s_masters_no_etcd" {
+  default = "c3.small.x86"
+}
+
+variable "plan_etcd" {
+  default = "c3.small.x86"
+}
+
+variable "plan_k8s_nodes" {
+  default = "c3.medium.x86"
+}
+
+variable "number_of_k8s_masters" {
+  default = 1
+}
+
+variable "number_of_k8s_masters_no_etcd" {
+  default = 0
+}
+
+variable "number_of_etcd" {
+  default = 0
+}
+
+variable "number_of_k8s_nodes" {
+  default = 1
+}
--- a/contrib/terraform/hetzner/README.md
+++ b/contrib/terraform/hetzner/README.md
@@ -102,8 +102,7 @@ Please read the instructions in both repos on how to install it.
 You can teardown your infrastructure using the following Terraform command:

 ```bash
-cd ./kubespray
-terraform -chdir=./contrib/terraform/hetzner/ destroy --var-file=../../../inventory/$CLUSTER/default.tfvars
+terraform destroy --var-file default.tfvars ../../contrib/terraform/hetzner
 ```

 ## Variables
--- a/contrib/terraform/nifcloud/.gitignore
+++ b/contrib/terraform/nifcloud/.gitignore
@@ -0,0 +1,5 @@
+*.tfstate*
+.terraform.lock.hcl
+.terraform
+
+sample-inventory/inventory.ini
--- a/contrib/terraform/nifcloud/README.md
+++ b/contrib/terraform/nifcloud/README.md
@@ -0,0 +1,138 @@
+# Kubernetes on NIFCLOUD with Terraform
+
+Provision a Kubernetes cluster on [NIFCLOUD](https://pfs.nifcloud.com/) using Terraform and Kubespray
+
+## Overview
+
+The setup looks like following
+
+```text
+                              Kubernetes cluster
+                        +----------------------------+
+---------------+       |   +--------------------+   |
+|               |       |   | +--------------------+ |
+| API server LB +---------> | |                    | |
+|               |       |   | | Control Plane/etcd | |
+---------------+       |   | | node(s)            | |
+                        |   +-+                    | |
+                        |     +--------------------+ |
+                        |           ^                |
+                        |           |                |
+                        |           v                |
+                        |   +--------------------+   |
+                        |   | +--------------------+ |
+                        |   | |                    | |
+                        |   | |        Worker      | |
+                        |   | |        node(s)     | |
+                        |   +-+                    | |
+                        |     +--------------------+ |
+                        +----------------------------+
+```
+
+## Requirements
+
+* Terraform 1.3.7
+
+## Quickstart
+
+### Export Variables
+
+* Your NIFCLOUD credentials:
+
+  ```bash
+  export NIFCLOUD_ACCESS_KEY_ID=<YOUR ACCESS KEY>
+  export NIFCLOUD_SECRET_ACCESS_KEY=<YOUR SECRET ACCESS KEY>
+  ```
+
+* The SSH KEY used to connect to the instance:
+  * FYI: [Cloud Help(SSH Key)](https://pfs.nifcloud.com/help/ssh.htm)
+
+  ```bash
+  export TF_VAR_SSHKEY_NAME=<YOUR SSHKEY NAME>
+  ```
+
+* The IP address to connect to bastion server:
+
+  ```bash
+  export TF_VAR_working_instance_ip=$(curl ifconfig.me)
+  ```
+
+### Create The Infrastructure
+
+* Run terraform:
+
+  ```bash
+  terraform init
+  terraform apply -var-file ./sample-inventory/cluster.tfvars
+  ```
+
+### Setup The Kubernetes
+
+* Generate cluster configuration file:
+
+  ```bash
+  ./generate-inventory.sh > sample-inventory/inventory.ini
+  ```
+
+* Export Variables:
+
+  ```bash
+  BASTION_IP=$(terraform output -json | jq -r '.kubernetes_cluster.value.bastion_info | to_entries[].value.public_ip')
+  API_LB_IP=$(terraform output -json | jq -r '.kubernetes_cluster.value.control_plane_lb')
+  CP01_IP=$(terraform output -json | jq -r '.kubernetes_cluster.value.control_plane_info | to_entries[0].value.private_ip')
+  export ANSIBLE_SSH_ARGS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ProxyCommand=\"ssh root@${BASTION_IP} -W %h:%p\""
+  ```
+
+* Set ssh-agent"
+
+  ```bash
+  eval `ssh-agent`
+  ssh-add <THE PATH TO YOUR SSH KEY>
+  ```
+
+* Run cluster.yml playbook:
+
+  ```bash
+  cd ./../../../
+  ansible-playbook -i contrib/terraform/nifcloud/inventory/inventory.ini cluster.yml
+  ```
+
+### Connecting to Kubernetes
+
+* [Install kubectl](https://kubernetes.io/docs/tasks/tools/) on the localhost
+* Fetching kubeconfig file:
+
+  ```bash
+  mkdir -p ~/.kube
+  scp -o ProxyCommand="ssh root@${BASTION_IP} -W %h:%p" root@${CP01_IP}:/etc/kubernetes/admin.conf ~/.kube/config
+  ```
+
+* Rewrite /etc/hosts
+
+  ```bash
+  sudo echo "${API_LB_IP} lb-apiserver.kubernetes.local" >> /etc/hosts
+  ```
+
+* Run kubectl
+
+  ```bash
+  kubectl get node
+  ```
+
+## Variables
+
+* `region`: Region where to run the cluster
+* `az`: Availability zone where to run the cluster
+* `private_ip_bn`: Private ip address of bastion server
+* `private_network_cidr`:  Subnet of private network
+* `instances_cp`: Machine to provision as Control Plane. Key of this object will be used as part of the machine' name
+  * `private_ip`: private ip address of machine
+* `instances_wk`: Machine to provision as Worker Node. Key of this object will be used as part of the machine' name
+  * `private_ip`: private ip address of machine
+* `instance_key_name`: The key name of the Key Pair to use for the instance
+* `instance_type_bn`: The instance type of bastion server
+* `instance_type_wk`: The instance type of worker node
+* `instance_type_cp`: The instance type of control plane
+* `image_name`: OS image used for the instance
+* `working_instance_ip`: The IP address to connect to bastion server
+* `accounting_type`: Accounting type. (1: monthly, 2: pay per use)
--- a/contrib/terraform/nifcloud/generate-inventory.sh
+++ b/contrib/terraform/nifcloud/generate-inventory.sh
@@ -0,0 +1,64 @@
+#!/bin/bash
+
+#
+# Generates a inventory file based on the terraform output.
+# After provisioning a cluster, simply run this command and supply the terraform state file
+# Default state file is terraform.tfstate
+#
+
+set -e
+
+TF_OUT=$(terraform output -json)
+
+CONTROL_PLANES=$(jq -r '.kubernetes_cluster.value.control_plane_info | to_entries[]'  <(echo "${TF_OUT}"))
+WORKERS=$(jq -r '.kubernetes_cluster.value.worker_info | to_entries[]'  <(echo "${TF_OUT}"))
+mapfile -t CONTROL_PLANE_NAMES < <(jq -r '.key'  <(echo "${CONTROL_PLANES}"))
+mapfile -t WORKER_NAMES < <(jq -r '.key'  <(echo "${WORKERS}"))
+
+API_LB=$(jq -r '.kubernetes_cluster.value.control_plane_lb' <(echo "${TF_OUT}"))
+
+echo "[all]"
+# Generate control plane hosts
+i=1
+for name in "${CONTROL_PLANE_NAMES[@]}"; do
+  private_ip=$(jq -r '. | select( .key=='"\"${name}\""' ) | .value.private_ip'  <(echo "${CONTROL_PLANES}"))
+  echo "${name} ansible_user=root ansible_host=${private_ip} access_ip=${private_ip} ip=${private_ip} etcd_member_name=etcd${i}"
+  i=$(( i + 1 ))
+done
+
+# Generate worker hosts
+for name in "${WORKER_NAMES[@]}"; do
+  private_ip=$(jq -r '. | select( .key=='"\"${name}\""' ) | .value.private_ip'  <(echo "${WORKERS}"))
+  echo "${name} ansible_user=root ansible_host=${private_ip} access_ip=${private_ip} ip=${private_ip}"
+done
+
+API_LB=$(jq -r '.kubernetes_cluster.value.control_plane_lb'  <(echo "${TF_OUT}"))
+
+echo ""
+echo "[all:vars]"
+echo "upstream_dns_servers=['8.8.8.8','8.8.4.4']"
+echo "loadbalancer_apiserver={'address':'${API_LB}','port':'6443'}"
+
+
+echo ""
+echo "[kube_control_plane]"
+for name in "${CONTROL_PLANE_NAMES[@]}"; do
+  echo "${name}"
+done
+
+echo ""
+echo "[etcd]"
+for name in "${CONTROL_PLANE_NAMES[@]}"; do
+  echo "${name}"
+done
+
+echo ""
+echo "[kube_node]"
+for name in "${WORKER_NAMES[@]}"; do
+  echo "${name}"
+done
+
+echo ""
+echo "[k8s_cluster:children]"
+echo "kube_control_plane"
+echo "kube_node"
--- a/contrib/terraform/nifcloud/main.tf
+++ b/contrib/terraform/nifcloud/main.tf
@@ -0,0 +1,36 @@
+provider "nifcloud" {
+  region = var.region
+}
+
+module "kubernetes_cluster" {
+  source = "./modules/kubernetes-cluster"
+
+  availability_zone = var.az
+  prefix            = "dev"
+
+  private_network_cidr = var.private_network_cidr
+
+  instance_key_name = var.instance_key_name
+  instances_cp      = var.instances_cp
+  instances_wk      = var.instances_wk
+  image_name        = var.image_name
+
+  instance_type_bn = var.instance_type_bn
+  instance_type_cp = var.instance_type_cp
+  instance_type_wk = var.instance_type_wk
+
+  private_ip_bn = var.private_ip_bn
+
+  additional_lb_filter = [var.working_instance_ip]
+}
+
+resource "nifcloud_security_group_rule" "ssh_from_bastion" {
+  security_group_names = [
+    module.kubernetes_cluster.security_group_name.bastion
+  ]
+  type      = "IN"
+  from_port = 22
+  to_port   = 22
+  protocol  = "TCP"
+  cidr_ip   = var.working_instance_ip
+}
--- a/contrib/terraform/nifcloud/modules/kubernetes-cluster/main.tf
+++ b/contrib/terraform/nifcloud/modules/kubernetes-cluster/main.tf
@@ -0,0 +1,301 @@
+#################################################
+##
+## Local variables
+##
+locals {
+  # e.g. east-11 is 11
+  az_num = reverse(split("-", var.availability_zone))[0]
+  # e.g. east-11 is e11
+  az_short_name = "${substr(reverse(split("-", var.availability_zone))[1], 0, 1)}${local.az_num}"
+
+  # Port used by the protocol
+  port_ssh     = 22
+  port_kubectl = 6443
+  port_kubelet = 10250
+
+  # calico: https://docs.tigera.io/calico/latest/getting-started/kubernetes/requirements#network-requirements
+  port_bgp   = 179
+  port_vxlan = 4789
+  port_etcd  = 2379
+}
+
+#################################################
+##
+## General
+##
+
+# data
+data "nifcloud_image" "this" {
+  image_name = var.image_name
+}
+
+# private lan
+resource "nifcloud_private_lan" "this" {
+  private_lan_name  = "${var.prefix}lan"
+  availability_zone = var.availability_zone
+  cidr_block        = var.private_network_cidr
+  accounting_type   = var.accounting_type
+}
+
+#################################################
+##
+## Bastion
+##
+resource "nifcloud_security_group" "bn" {
+  group_name        = "${var.prefix}bn"
+  description       = "${var.prefix} bastion"
+  availability_zone = var.availability_zone
+}
+
+resource "nifcloud_instance" "bn" {
+
+  instance_id    = "${local.az_short_name}${var.prefix}bn01"
+  security_group = nifcloud_security_group.bn.group_name
+  instance_type  = var.instance_type_bn
+
+  user_data = templatefile("${path.module}/templates/userdata.tftpl", {
+    private_ip_address = var.private_ip_bn
+    ssh_port           = local.port_ssh
+    hostname           = "${local.az_short_name}${var.prefix}bn01"
+  })
+
+  availability_zone = var.availability_zone
+  accounting_type   = var.accounting_type
+  image_id          = data.nifcloud_image.this.image_id
+  key_name          = var.instance_key_name
+
+  network_interface {
+    network_id = "net-COMMON_GLOBAL"
+  }
+  network_interface {
+    network_id = nifcloud_private_lan.this.network_id
+    ip_address = "static"
+  }
+
+  # The image_id changes when the OS image type is demoted from standard to public.
+  lifecycle {
+    ignore_changes = [
+      image_id,
+      user_data,
+    ]
+  }
+}
+
+#################################################
+##
+## Control Plane
+##
+resource "nifcloud_security_group" "cp" {
+  group_name        = "${var.prefix}cp"
+  description       = "${var.prefix} control plane"
+  availability_zone = var.availability_zone
+}
+
+resource "nifcloud_instance" "cp" {
+  for_each = var.instances_cp
+
+  instance_id    = "${local.az_short_name}${var.prefix}${each.key}"
+  security_group = nifcloud_security_group.cp.group_name
+  instance_type  = var.instance_type_cp
+  user_data = templatefile("${path.module}/templates/userdata.tftpl", {
+    private_ip_address = each.value.private_ip
+    ssh_port           = local.port_ssh
+    hostname           = "${local.az_short_name}${var.prefix}${each.key}"
+  })
+
+  availability_zone = var.availability_zone
+  accounting_type   = var.accounting_type
+  image_id          = data.nifcloud_image.this.image_id
+  key_name          = var.instance_key_name
+
+  network_interface {
+    network_id = "net-COMMON_GLOBAL"
+  }
+  network_interface {
+    network_id = nifcloud_private_lan.this.network_id
+    ip_address = "static"
+  }
+
+  # The image_id changes when the OS image type is demoted from standard to public.
+  lifecycle {
+    ignore_changes = [
+      image_id,
+      user_data,
+    ]
+  }
+}
+
+resource "nifcloud_load_balancer" "this" {
+  load_balancer_name = "${local.az_short_name}${var.prefix}cp"
+  accounting_type    = var.accounting_type
+  balancing_type     = 1 // Round-Robin
+  load_balancer_port = local.port_kubectl
+  instance_port      = local.port_kubectl
+  instances          = [for v in nifcloud_instance.cp : v.instance_id]
+  filter = concat(
+    [for k, v in nifcloud_instance.cp : v.public_ip],
+    [for k, v in nifcloud_instance.wk : v.public_ip],
+    var.additional_lb_filter,
+  )
+  filter_type = 1 // Allow
+}
+
+#################################################
+##
+## Worker
+##
+resource "nifcloud_security_group" "wk" {
+  group_name        = "${var.prefix}wk"
+  description       = "${var.prefix} worker"
+  availability_zone = var.availability_zone
+}
+
+resource "nifcloud_instance" "wk" {
+  for_each = var.instances_wk
+
+  instance_id    = "${local.az_short_name}${var.prefix}${each.key}"
+  security_group = nifcloud_security_group.wk.group_name
+  instance_type  = var.instance_type_wk
+  user_data = templatefile("${path.module}/templates/userdata.tftpl", {
+    private_ip_address = each.value.private_ip
+    ssh_port           = local.port_ssh
+    hostname           = "${local.az_short_name}${var.prefix}${each.key}"
+  })
+
+  availability_zone = var.availability_zone
+  accounting_type   = var.accounting_type
+  image_id          = data.nifcloud_image.this.image_id
+  key_name          = var.instance_key_name
+
+  network_interface {
+    network_id = "net-COMMON_GLOBAL"
+  }
+  network_interface {
+    network_id = nifcloud_private_lan.this.network_id
+    ip_address = "static"
+  }
+
+  # The image_id changes when the OS image type is demoted from standard to public.
+  lifecycle {
+    ignore_changes = [
+      image_id,
+      user_data,
+    ]
+  }
+}
+
+#################################################
+##
+## Security Group Rule: Kubernetes
+##
+
+# ssh
+resource "nifcloud_security_group_rule" "ssh_from_bastion" {
+  security_group_names = [
+    nifcloud_security_group.wk.group_name,
+    nifcloud_security_group.cp.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_ssh
+  to_port                    = local.port_ssh
+  protocol                   = "TCP"
+  source_security_group_name = nifcloud_security_group.bn.group_name
+}
+
+# kubectl
+resource "nifcloud_security_group_rule" "kubectl_from_worker" {
+  security_group_names = [
+    nifcloud_security_group.cp.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_kubectl
+  to_port                    = local.port_kubectl
+  protocol                   = "TCP"
+  source_security_group_name = nifcloud_security_group.wk.group_name
+}
+
+# kubelet
+resource "nifcloud_security_group_rule" "kubelet_from_worker" {
+  security_group_names = [
+    nifcloud_security_group.cp.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_kubelet
+  to_port                    = local.port_kubelet
+  protocol                   = "TCP"
+  source_security_group_name = nifcloud_security_group.wk.group_name
+}
+
+resource "nifcloud_security_group_rule" "kubelet_from_control_plane" {
+  security_group_names = [
+    nifcloud_security_group.wk.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_kubelet
+  to_port                    = local.port_kubelet
+  protocol                   = "TCP"
+  source_security_group_name = nifcloud_security_group.cp.group_name
+}
+
+#################################################
+##
+## Security Group Rule: calico
+##
+
+# vslan
+resource "nifcloud_security_group_rule" "vxlan_from_control_plane" {
+  security_group_names = [
+    nifcloud_security_group.wk.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_vxlan
+  to_port                    = local.port_vxlan
+  protocol                   = "UDP"
+  source_security_group_name = nifcloud_security_group.cp.group_name
+}
+
+resource "nifcloud_security_group_rule" "vxlan_from_worker" {
+  security_group_names = [
+    nifcloud_security_group.cp.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_vxlan
+  to_port                    = local.port_vxlan
+  protocol                   = "UDP"
+  source_security_group_name = nifcloud_security_group.wk.group_name
+}
+
+# bgp
+resource "nifcloud_security_group_rule" "bgp_from_control_plane" {
+  security_group_names = [
+    nifcloud_security_group.wk.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_bgp
+  to_port                    = local.port_bgp
+  protocol                   = "TCP"
+  source_security_group_name = nifcloud_security_group.cp.group_name
+}
+
+resource "nifcloud_security_group_rule" "bgp_from_worker" {
+  security_group_names = [
+    nifcloud_security_group.cp.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_bgp
+  to_port                    = local.port_bgp
+  protocol                   = "TCP"
+  source_security_group_name = nifcloud_security_group.wk.group_name
+}
+
+# etcd
+resource "nifcloud_security_group_rule" "etcd_from_worker" {
+  security_group_names = [
+    nifcloud_security_group.cp.group_name,
+  ]
+  type                       = "IN"
+  from_port                  = local.port_etcd
+  to_port                    = local.port_etcd
+  protocol                   = "TCP"
+  source_security_group_name = nifcloud_security_group.wk.group_name
+}
--- a/contrib/terraform/nifcloud/modules/kubernetes-cluster/outputs.tf
+++ b/contrib/terraform/nifcloud/modules/kubernetes-cluster/outputs.tf
@@ -0,0 +1,48 @@
+output "control_plane_lb" {
+  description = "The DNS name of LB for control plane"
+  value       = nifcloud_load_balancer.this.dns_name
+}
+
+output "security_group_name" {
+  description = "The security group used in the cluster"
+  value = {
+    bastion       = nifcloud_security_group.bn.group_name,
+    control_plane = nifcloud_security_group.cp.group_name,
+    worker        = nifcloud_security_group.wk.group_name,
+  }
+}
+
+output "private_network_id" {
+  description = "The private network used in the cluster"
+  value       = nifcloud_private_lan.this.id
+}
+
+output "bastion_info" {
+  description = "The basion information in cluster"
+  value = { (nifcloud_instance.bn.instance_id) : {
+    instance_id = nifcloud_instance.bn.instance_id,
+    unique_id   = nifcloud_instance.bn.unique_id,
+    private_ip  = nifcloud_instance.bn.private_ip,
+    public_ip   = nifcloud_instance.bn.public_ip,
+  } }
+}
+
+output "worker_info" {
+  description = "The worker information in cluster"
+  value = { for v in nifcloud_instance.wk : v.instance_id => {
+    instance_id = v.instance_id,
+    unique_id   = v.unique_id,
+    private_ip  = v.private_ip,
+    public_ip   = v.public_ip,
+  } }
+}
+
+output "control_plane_info" {
+  description = "The control plane information in cluster"
+  value = { for v in nifcloud_instance.cp : v.instance_id => {
+    instance_id = v.instance_id,
+    unique_id   = v.unique_id,
+    private_ip  = v.private_ip,
+    public_ip   = v.public_ip,
+  } }
+}
--- a/contrib/terraform/nifcloud/modules/kubernetes-cluster/templates/userdata.tftpl
+++ b/contrib/terraform/nifcloud/modules/kubernetes-cluster/templates/userdata.tftpl
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+#################################################
+##
+## IP Address
+##
+configure_private_ip_address () {
+  cat << EOS > /etc/netplan/01-netcfg.yaml
+network:
+    version: 2
+    renderer: networkd
+    ethernets:
+        ens192:
+            dhcp4: yes
+            dhcp6: yes
+            dhcp-identifier: mac
+        ens224:
+            dhcp4: no
+            dhcp6: no
+            addresses: [${private_ip_address}]
+EOS
+  netplan apply
+}
+configure_private_ip_address
+
+#################################################
+##
+## SSH
+##
+configure_ssh_port () {
+  sed -i 's/^#*Port [0-9]*/Port ${ssh_port}/' /etc/ssh/sshd_config
+}
+configure_ssh_port
+
+#################################################
+##
+## Hostname
+##
+hostnamectl set-hostname ${hostname}
+
+#################################################
+##
+## Disable swap files genereated by systemd-gpt-auto-generator
+##
+systemctl mask "dev-sda3.swap"
--- a/contrib/terraform/nifcloud/modules/kubernetes-cluster/terraform.tf
+++ b/contrib/terraform/nifcloud/modules/kubernetes-cluster/terraform.tf
@@ -0,0 +1,9 @@
+terraform {
+  required_version = ">=1.3.7"
+  required_providers {
+    nifcloud = {
+      source  = "nifcloud/nifcloud"
+      version = ">= 1.8.0, < 2.0.0"
+    }
+  }
+}
--- a/contrib/terraform/nifcloud/modules/kubernetes-cluster/variables.tf
+++ b/contrib/terraform/nifcloud/modules/kubernetes-cluster/variables.tf
@@ -0,0 +1,81 @@
+variable "availability_zone" {
+  description = "The availability zone"
+  type        = string
+}
+
+variable "prefix" {
+  description = "The prefix for the entire cluster"
+  type        = string
+  validation {
+    condition     = length(var.prefix) <= 5
+    error_message = "Must be a less than 5 character long."
+  }
+}
+
+variable "private_network_cidr" {
+  description = "The subnet of private network"
+  type        = string
+  validation {
+    condition     = can(cidrnetmask(var.private_network_cidr))
+    error_message = "Must be a valid IPv4 CIDR block address."
+  }
+}
+
+variable "private_ip_bn" {
+  description = "Private IP of bastion server"
+  type        = string
+}
+
+variable "instances_cp" {
+  type = map(object({
+    private_ip = string
+  }))
+}
+
+variable "instances_wk" {
+  type = map(object({
+    private_ip = string
+  }))
+}
+
+variable "instance_key_name" {
+  description = "The key name of the Key Pair to use for the instance"
+  type        = string
+}
+
+variable "instance_type_bn" {
+  description = "The instance type of bastion server"
+  type        = string
+}
+
+variable "instance_type_wk" {
+  description = "The instance type of worker"
+  type        = string
+}
+
+variable "instance_type_cp" {
+  description = "The instance type of control plane"
+  type        = string
+}
+
+variable "image_name" {
+  description = "The name of image"
+  type        = string
+}
+
+variable "additional_lb_filter" {
+  description = "Additional LB filter"
+  type        = list(string)
+}
+
+variable "accounting_type" {
+  type    = string
+  default = "1"
+  validation {
+    condition = anytrue([
+      var.accounting_type == "1", // Monthly
+      var.accounting_type == "2", // Pay per use
+    ])
+    error_message = "Must be a 1 or 2."
+  }
+}
--- a/contrib/terraform/nifcloud/output.tf
+++ b/contrib/terraform/nifcloud/output.tf
@@ -0,0 +1,3 @@
+output "kubernetes_cluster" {
+  value = module.kubernetes_cluster
+}
--- a/contrib/terraform/nifcloud/sample-inventory/cluster.tfvars
+++ b/contrib/terraform/nifcloud/sample-inventory/cluster.tfvars
@@ -0,0 +1,22 @@
+region = "jp-west-1"
+az     = "west-11"
+
+instance_key_name = "deployerkey"
+
+instance_type_bn = "e-medium"
+instance_type_cp = "e-medium"
+instance_type_wk = "e-medium"
+
+private_network_cidr = "192.168.30.0/24"
+instances_cp = {
+  "cp01" : { private_ip : "192.168.30.11/24" }
+  "cp02" : { private_ip : "192.168.30.12/24" }
+  "cp03" : { private_ip : "192.168.30.13/24" }
+}
+instances_wk = {
+  "wk01" : { private_ip : "192.168.30.21/24" }
+  "wk02" : { private_ip : "192.168.30.22/24" }
+}
+private_ip_bn = "192.168.30.10/24"
+
+image_name = "Ubuntu Server 22.04 LTS"
--- a/contrib/terraform/nifcloud/sample-inventory/group_vars
+++ b/contrib/terraform/nifcloud/sample-inventory/group_vars
@@ -0,0 +1 @@
+../../../../inventory/sample/group_vars
--- a/contrib/terraform/nifcloud/terraform.tf
+++ b/contrib/terraform/nifcloud/terraform.tf
@@ -0,0 +1,9 @@
+terraform {
+  required_version = ">=1.3.7"
+  required_providers {
+    nifcloud = {
+      source  = "nifcloud/nifcloud"
+      version = "1.8.0"
+    }
+  }
+}
--- a/contrib/terraform/nifcloud/variables.tf
+++ b/contrib/terraform/nifcloud/variables.tf
@@ -0,0 +1,77 @@
+variable "region" {
+  description = "The region"
+  type        = string
+}
+
+variable "az" {
+  description = "The availability zone"
+  type        = string
+}
+
+variable "private_ip_bn" {
+  description = "Private IP of bastion server"
+  type        = string
+}
+
+variable "private_network_cidr" {
+  description = "The subnet of private network"
+  type        = string
+  validation {
+    condition     = can(cidrnetmask(var.private_network_cidr))
+    error_message = "Must be a valid IPv4 CIDR block address."
+  }
+}
+
+variable "instances_cp" {
+  type = map(object({
+    private_ip = string
+  }))
+}
+
+variable "instances_wk" {
+  type = map(object({
+    private_ip = string
+  }))
+}
+
+variable "instance_key_name" {
+  description = "The key name of the Key Pair to use for the instance"
+  type        = string
+}
+
+variable "instance_type_bn" {
+  description = "The instance type of bastion server"
+  type        = string
+}
+
+variable "instance_type_wk" {
+  description = "The instance type of worker"
+  type        = string
+}
+
+variable "instance_type_cp" {
+  description = "The instance type of control plane"
+  type        = string
+}
+
+variable "image_name" {
+  description = "The name of image"
+  type        = string
+}
+
+variable "working_instance_ip" {
+  description = "The IP address to connect to bastion server."
+  type        = string
+}
+
+variable "accounting_type" {
+  type    = string
+  default = "2"
+  validation {
+    condition = anytrue([
+      var.accounting_type == "1", // Monthly
+      var.accounting_type == "2", // Pay per use
+    ])
+    error_message = "Must be a 1 or 2."
+  }
+}
--- a/contrib/terraform/openstack/README.md
+++ b/contrib/terraform/openstack/README.md
@@ -281,9 +281,9 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
 |`k8s_allowed_remote_ips_ipv6` | List of IPv6 CIDR allowed to initiate a SSH connection, empty by default |
 |`k8s_allowed_egress_ipv6_ips` | List of IPv6 CIDRs allowed for egress traffic, `["::/0"]` by default |
 |`worker_allowed_ports` | List of ports to open on worker nodes, `[{ "protocol" = "tcp", "port_range_min" = 30000, "port_range_max" = 32767, "remote_ip_prefix" = "0.0.0.0/0"}]` by default |
-|`worker_allowed_ports_ipv6` | List of ports to open on worker nodes for IPv6 CIDR blocks, `[{ "protocol" = "tcp", "port_range_min" = 30000, "port_range_max" = 32767, "remote_ip_prefix" = "::/0"}, { "protocol" = "ipv6-icmp", "port_range_min" = 0, "port_range_max" = 0, "remote_ip_prefix" = "::/0"}]` by default |
+|`worker_allowed_ports_ipv6` | List of ports to open on worker nodes for IPv6 CIDR blocks, `[{ "protocol" = "tcp", "port_range_min" = 30000, "port_range_max" = 32767, "remote_ip_prefix" = "::/0"}]` by default |
 |`master_allowed_ports` | List of ports to open on master nodes, expected format is `[{ "protocol" = "tcp", "port_range_min" = 443, "port_range_max" = 443, "remote_ip_prefix" = "0.0.0.0/0"}]`, empty by default |
-|`master_allowed_ports_ipv6` | List of ports to open on master nodes for IPv6 CIDR blocks, `[{ "protocol" = "ipv6-icmp", "port_range_min" = 0, "port_range_max" = 0, "remote_ip_prefix" = "::/0"}]` by default |
+|`master_allowed_ports_ipv6` | List of ports to open on master nodes for IPv6 CIDR blocks, expected format is `[{ "protocol" = "tcp", "port_range_min" = 443, "port_range_max" = 443, "remote_ip_prefix" = "::/0"}]`, empty by default |
 |`node_root_volume_size_in_gb` | Size of the root volume for nodes, 0 to use ephemeral storage |
 |`master_root_volume_size_in_gb` | Size of the root volume for masters, 0 to use ephemeral storage |
 |`master_volume_type` | Volume type of the root volume for control_plane, 'Default' by default |
@@ -624,7 +624,7 @@ Edit `inventory/$CLUSTER/group_vars/k8s_cluster/k8s_cluster.yml`:
  - **calico** requires [configuring OpenStack Neutron ports](/docs/cloud_controllers/openstack.md) to allow service and pod subnets

 ```yml
-# Choose network plugin (calico or flannel)
+# Choose network plugin (calico, weave or flannel)
 # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
 kube_network_plugin: flannel
 ```
--- a/contrib/terraform/openstack/variables.tf
+++ b/contrib/terraform/openstack/variables.tf
@@ -271,14 +271,7 @@ variable "master_allowed_ports" {
 variable "master_allowed_ports_ipv6" {
  type = list(any)

-  default = [
-    {
-      "protocol"         = "ipv6-icmp"
-      "port_range_min"   = 0
-      "port_range_max"   = 0
-      "remote_ip_prefix" = "::/0"
-    },
-  ]
+  default = []
 }

 variable "worker_allowed_ports" {
@@ -304,12 +297,6 @@ variable "worker_allowed_ports_ipv6" {
      "port_range_max"   = 32767
      "remote_ip_prefix" = "::/0"
    },
-    {
-      "protocol"         = "ipv6-icmp"
-      "port_range_min"   = 0
-      "port_range_max"   = 0
-      "remote_ip_prefix" = "::/0"
-    },
  ]
 }

--- a/contrib/terraform/upcloud/README.md
+++ b/contrib/terraform/upcloud/README.md
@@ -2,6 +2,35 @@

 Provision a Kubernetes cluster on [UpCloud](https://upcloud.com/) using Terraform and Kubespray

+## Overview
+
+The setup looks like following
+
+```text
+   Kubernetes cluster
+--------------------------+
+|      +--------------+    |
+|      | +--------------+  |
+| -->  | |              |  |
+|      | | Master/etcd  |  |
+|      | | node(s)      |  |
+|      +-+              |  |
+|        +--------------+  |
+|              ^           |
+|              |           |
+|              v           |
+|      +--------------+    |
+|      | +--------------+  |
+| -->  | |              |  |
+|      | |    Worker    |  |
+|      | |    node(s)   |  |
+|      +-+              |  |
+|        +--------------+  |
+--------------------------+
+```
+
+The nodes uses a private network for node to node communication and a public interface for all external communication.
+
 ## Requirements

 * Terraform 0.13.0 or newer
@@ -71,8 +100,6 @@ terraform destroy --var-file cluster-settings.tfvars \
 * `template_name`: The name or UUID  of a base image
 * `username`: a user to access the nodes, defaults to "ubuntu"
 * `private_network_cidr`: CIDR to use for the private network, defaults to "172.16.0.0/24"
-* `dns_servers`: DNS servers that will be used by the nodes. Until [this is solved](https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562) this is done using user_data to reconfigure resolved. Defaults to `[]`
-* `use_public_ips`: If a NIC connencted to the Public network should be attached to all nodes by default. Can be overridden by `force_public_ip` if this is set to `false`. Defaults to `true`
 * `ssh_public_keys`: List of public SSH keys to install on all machines
 * `zone`: The zone where to run the cluster
 * `machines`: Machines to provision. Key of this object will be used as the name of the machine
@@ -81,8 +108,6 @@ terraform destroy --var-file cluster-settings.tfvars \
  * `cpu`: number of cpu cores
  * `mem`: memory size in MB
  * `disk_size`: The size of the storage in GB
-  * `force_public_ip`: If `use_public_ips` is set to `false`, this forces a public NIC onto the machine anyway when set to `true`. Useful if you're migrating from public nodes to only private. Defaults to `false`
-  * `dns_servers`: This works the same way as the global `dns_severs` but only applies to a single node. If set to `[]` while the global `dns_servers` is set to something else, then it will not add the user_data and thus will not be recreated. Useful if you're migrating from public nodes to only private. Defaults to `null`
  * `additional_disks`: Additional disks to attach to the node.
    * `size`: The size of the additional disk in GB
    * `tier`: The tier of disk to use (`maxiops` is the only one you can choose atm)
@@ -114,7 +139,6 @@ terraform destroy --var-file cluster-settings.tfvars \
  * `port`: Port to load balance.
  * `target_port`: Port to the backend servers.
  * `backend_servers`: List of servers that traffic to the port should be forwarded to.
-  * `proxy_protocol`: If the loadbalancer should set up the backend using proxy protocol.
 * `router_enable`: If a router should be connected to the private network or not
 * `gateways`: Gateways that should be connected to the router, requires router_enable is set to true
  * `features`: List of features for the gateway
@@ -147,27 +171,3 @@ terraform destroy --var-file cluster-settings.tfvars \
 * `server_groups`: Group servers together
  * `servers`: The servers that should be included in the group.
  * `anti_affinity_policy`: Defines if a server group is an anti-affinity group. Setting this to "strict" or yes" will result in all servers in the group being placed on separate compute hosts. The value can be "strict", "yes" or "no". "strict" refers to strict policy doesn't allow servers in the same server group to be on the same host. "yes" refers to best-effort policy and tries to put servers on different hosts, but this is not guaranteed.
-
-## Migration
-
-When `null_resource.inventories` and `data.template_file.inventory` was changed to `local_file.inventory` the old state file needs to be cleaned of the old state.
-The error messages you'll see if you encounter this is:
-
-```text
-Error: failed to read schema for null_resource.inventories in registry.terraform.io/hashicorp/null: failed to instantiate provider "registry.terraform.io/hashicorp/null" to obtain schema: unavailable provider "registry.terraform.io/hashicorp/null"
-Error: failed to read schema for data.template_file.inventory in registry.terraform.io/hashicorp/template: failed to instantiate provider "registry.terraform.io/hashicorp/template" to obtain schema: unavailable provider "registry.terraform.io/hashicorp/template"
-```
-
-This can be fixed with the following lines
-
-```bash
-terraform state rm -state=terraform.tfstate null_resource.inventories
-terraform state rm -state=terraform.tfstate data.template_file.inventory
-```
-
-### Public to Private only migration
-
-Since there's no way to remove the public NIC on a machine without recreating its private NIC it's not possible to inplace change a cluster to only use private IPs.
-The way to migrate is to first set `use_public_ips` to `false`, `dns_servers` to some DNS servers and then update all existing servers to have `force_public_ip` set to `true` and `dns_severs` set to `[]`.
-After that you can add new nodes without `force_public_ip` and `dns_servers` set and create them.
-Add the new nodes into the cluster and when all of them are added, remove the old nodes.
--- a/contrib/terraform/upcloud/cluster-settings.tfvars
+++ b/contrib/terraform/upcloud/cluster-settings.tfvars
@@ -124,9 +124,9 @@ worker_allowed_ports = []

 loadbalancer_enabled        = false
 loadbalancer_plan           = "development"
+loadbalancer_proxy_protocol = false
 loadbalancers = {
  # "http" : {
-  #   "proxy_protocol" : false
  #   "port" : 80,
  #   "target_port" : 80,
  #   "backend_servers" : [
--- a/contrib/terraform/upcloud/main.tf
+++ b/contrib/terraform/upcloud/main.tf
@@ -20,8 +20,6 @@ module "kubernetes" {
  username      = var.username

  private_network_cidr = var.private_network_cidr
-  dns_servers          = var.dns_servers
-  use_public_ips       = var.use_public_ips

  machines = var.machines

@@ -32,12 +30,12 @@ module "kubernetes" {
  firewall_default_deny_out = var.firewall_default_deny_out
  master_allowed_remote_ips = var.master_allowed_remote_ips
  k8s_allowed_remote_ips    = var.k8s_allowed_remote_ips
-  bastion_allowed_remote_ips = var.bastion_allowed_remote_ips
  master_allowed_ports      = var.master_allowed_ports
  worker_allowed_ports      = var.worker_allowed_ports

  loadbalancer_enabled                 = var.loadbalancer_enabled
  loadbalancer_plan                    = var.loadbalancer_plan
+  loadbalancer_outbound_proxy_protocol = var.loadbalancer_proxy_protocol ? "v2" : ""
  loadbalancer_legacy_network          = var.loadbalancer_legacy_network
  loadbalancers                        = var.loadbalancers

@@ -54,12 +52,32 @@ module "kubernetes" {
 # Generate ansible inventory
 #

-resource "local_file" "inventory" {
-  content = templatefile("${path.module}/templates/inventory.tpl", {
-    master_ip  = module.kubernetes.master_ip
-    worker_ip  = module.kubernetes.worker_ip
-    bastion_ip = module.kubernetes.bastion_ip
-    username   = var.username
-  })
-  filename = var.inventory_file
+data "template_file" "inventory" {
+  template = file("${path.module}/templates/inventory.tpl")
+
+  vars = {
+    connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d",
+      keys(module.kubernetes.master_ip),
+      values(module.kubernetes.master_ip).*.public_ip,
+      values(module.kubernetes.master_ip).*.private_ip,
+    range(1, length(module.kubernetes.master_ip) + 1)))
+    connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s",
+      keys(module.kubernetes.worker_ip),
+      values(module.kubernetes.worker_ip).*.public_ip,
+    values(module.kubernetes.worker_ip).*.private_ip))
+    list_master = join("\n", formatlist("%s",
+    keys(module.kubernetes.master_ip)))
+    list_worker = join("\n", formatlist("%s",
+    keys(module.kubernetes.worker_ip)))
+  }
+}
+
+resource "null_resource" "inventories" {
+  provisioner "local-exec" {
+    command = "echo '${data.template_file.inventory.rendered}' > ${var.inventory_file}"
+  }
+
+  triggers = {
+    template = data.template_file.inventory.rendered
+  }
 }
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
@@ -53,44 +53,6 @@ locals {
  # If prefix is set, all resources will be prefixed with "${var.prefix}-"
  # Else don't prefix with anything
  resource-prefix = "%{if var.prefix != ""}${var.prefix}-%{endif}"
-
-  master_ip = {
-    for instance in upcloud_server.master :
-      instance.hostname => {
-        for nic in instance.network_interface :
-          nic.type => nic.ip_address
-        if nic.ip_address != null
-      }
-  }
-  worker_ip = {
-    for instance in upcloud_server.worker :
-      instance.hostname => {
-        for nic in instance.network_interface :
-          nic.type => nic.ip_address
-        if nic.ip_address != null
-      }
-  }
-
-  bastion_ip = {
-    for instance in upcloud_server.bastion :
-      instance.hostname => {
-        for nic in instance.network_interface :
-          nic.type => nic.ip_address
-        if nic.ip_address != null
-      }
-  }
-
-  node_user_data = {
-    for name, machine in var.machines :
-      name => <<EOF
-%{ if ( length(machine.dns_servers != null ? machine.dns_servers : [] ) > 0 ) || ( length(var.dns_servers) > 0 && machine.dns_servers == null ) ~}
-#!/bin/bash
-echo -e "[Resolve]\nDNS=${ join(" ", length(machine.dns_servers != null ? machine.dns_servers : []) > 0 ? machine.dns_servers : var.dns_servers) }" > /etc/systemd/resolved.conf
-
-systemctl restart systemd-resolved
-%{ endif ~}
-EOF
-  }
 }

 resource "upcloud_network" "private" {
@@ -100,9 +62,6 @@ resource "upcloud_network" "private" {
  ip_network {
    address            = var.private_network_cidr
    dhcp_default_route = var.router_enable
-    # TODO: When support for dhcp_dns for private networks are in, remove the user_data and enable it here.
-    #       See more here https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562
-    # dhcp_dns           = length(var.private_network_dns) > 0 ? var.private_network_dns : null
    dhcp               = true
    family             = "IPv4"
  }
@@ -130,8 +89,8 @@ resource "upcloud_server" "master" {

  hostname     = "${local.resource-prefix}${each.key}"
  plan         = each.value.plan
-  cpu          = each.value.cpu
-  mem          = each.value.mem
+  cpu          = each.value.plan == null ? null : each.value.cpu
+  mem          = each.value.plan == null ? null : each.value.mem
  zone         = var.zone
  server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id

@@ -140,13 +99,10 @@ resource "upcloud_server" "master" {
    size    = each.value.disk_size
  }

-  dynamic "network_interface" {
-    for_each = each.value.force_public_ip || var.use_public_ips ? [1] : []
-
-    content {
+  # Public network interface
+  network_interface {
    type = "public"
  }
-  }

  # Private network interface
  network_interface {
@@ -180,9 +136,6 @@ resource "upcloud_server" "master" {
    keys            = var.ssh_public_keys
    create_password = false
  }
-
-  metadata = local.node_user_data[each.key] != "" ? true : null
-  user_data = local.node_user_data[each.key] != "" ? local.node_user_data[each.key] : null
 }

 resource "upcloud_server" "worker" {
@@ -194,8 +147,8 @@ resource "upcloud_server" "worker" {

  hostname     = "${local.resource-prefix}${each.key}"
  plan         = each.value.plan
-  cpu          = each.value.cpu
-  mem          = each.value.mem
+  cpu          = each.value.plan == null ? null : each.value.cpu
+  mem          = each.value.plan == null ? null : each.value.mem
  zone         = var.zone
  server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id

@@ -205,13 +158,10 @@ resource "upcloud_server" "worker" {
    size    = each.value.disk_size
  }

-  dynamic "network_interface" {
-    for_each = each.value.force_public_ip || var.use_public_ips ? [1] : []
-
-    content {
+  # Public network interface
+  network_interface {
    type = "public"
  }
-  }

  # Private network interface
  network_interface {
@@ -245,63 +195,6 @@ resource "upcloud_server" "worker" {
    keys            = var.ssh_public_keys
    create_password = false
  }
-
-  metadata = local.node_user_data[each.key] != "" ? true : null
-  user_data = local.node_user_data[each.key] != "" ? local.node_user_data[each.key] : null
-}
-
-resource "upcloud_server" "bastion" {
-  for_each = {
-    for name, machine in var.machines :
-    name => machine
-    if machine.node_type == "bastion"
-  }
-
-  hostname     = "${local.resource-prefix}${each.key}"
-  plan         = each.value.plan
-  cpu          = each.value.cpu
-  mem          = each.value.mem
-  zone         = var.zone
-  server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id
-
-
-  template {
-    storage = var.template_name
-    size    = each.value.disk_size
-  }
-
-  # Private network interface
-  network_interface {
-    type    = "private"
-    network = upcloud_network.private.id
-  }
-
-  # Private network interface
-  network_interface {
-    type    = "public"
-  }
-
-  firewall = var.firewall_enabled
-
-  dynamic "storage_devices" {
-    for_each = {
-      for disk_key_name, disk in upcloud_storage.additional_disks :
-      disk_key_name => disk
-      # Only add the disk if it matches the node name in the start of its name
-      if length(regexall("^${each.key}_.+", disk_key_name)) > 0
-    }
-
-    content {
-      storage = storage_devices.value.id
-    }
-  }
-
-  # Include at least one public SSH key
-  login {
-    user            = var.username
-    keys            = var.ssh_public_keys
-    create_password = false
-  }
 }

 resource "upcloud_firewall_rules" "master" {
@@ -650,53 +543,6 @@ resource "upcloud_firewall_rules" "k8s" {
  }
 }

-resource "upcloud_firewall_rules" "bastion" {
-  for_each  = upcloud_server.bastion
-  server_id = each.value.id
-
-  dynamic "firewall_rule" {
-    for_each = var.bastion_allowed_remote_ips
-
-    content {
-      action                 = "accept"
-      comment                = "Allow bastion SSH access from this network"
-      destination_port_end   = "22"
-      destination_port_start = "22"
-      direction              = "in"
-      family                 = "IPv4"
-      protocol               = "tcp"
-      source_address_end     = firewall_rule.value.end_address
-      source_address_start   = firewall_rule.value.start_address
-    }
-  }
-
-  dynamic "firewall_rule" {
-    for_each = length(var.bastion_allowed_remote_ips) > 0 ? [1] : []
-
-    content {
-      action                 = "drop"
-      comment                = "Drop bastion SSH access from other networks"
-      destination_port_end   = "22"
-      destination_port_start = "22"
-      direction              = "in"
-      family                 = "IPv4"
-      protocol               = "tcp"
-      source_address_end     = "255.255.255.255"
-      source_address_start   = "0.0.0.0"
-    }
-  }
-
-  firewall_rule {
-    action    = var.firewall_default_deny_in ? "drop" : "accept"
-    direction = "in"
-  }
-
-  firewall_rule {
-    action    = var.firewall_default_deny_out ? "drop" : "accept"
-    direction = "out"
-  }
-}
-
 resource "upcloud_loadbalancer" "lb" {
  count             = var.loadbalancer_enabled ? 1 : 0
  configured_status = "started"
@@ -737,7 +583,7 @@ resource "upcloud_loadbalancer_backend" "lb_backend" {
  loadbalancer = upcloud_loadbalancer.lb[0].id
  name         = "lb-backend-${each.key}"
  properties {
-    outbound_proxy_protocol = each.value.proxy_protocol ? "v2" : ""
+    outbound_proxy_protocol = var.loadbalancer_outbound_proxy_protocol
  }
 }

@@ -776,7 +622,7 @@ resource "upcloud_loadbalancer_static_backend_member" "lb_backend_member" {

  backend      = upcloud_loadbalancer_backend.lb_backend[each.value.lb_name].id
  name         = "${local.resource-prefix}${each.key}"
-  ip           = merge(local.master_ip, local.worker_ip)["${local.resource-prefix}${each.value.server_name}"].private
+  ip           = merge(upcloud_server.master, upcloud_server.worker)[each.value.server_name].network_interface[1].ip_address
  port         = each.value.port
  weight       = 100
  max_sessions = var.loadbalancer_plan == "production-small" ? 50000 : 1000
@@ -816,7 +662,7 @@ resource "upcloud_router" "router" {
 resource "upcloud_gateway" "gateway" {
  for_each = var.router_enable ? var.gateways : {}
  name = "${local.resource-prefix}${each.key}-gateway"
-  zone = var.private_cloud ? var.public_zone : var.zone
+  zone = var.zone

  features = each.value.features
  plan = each.value.plan
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf
@@ -1,13 +1,22 @@
+
 output "master_ip" {
-  value = local.master_ip
+  value = {
+    for instance in upcloud_server.master :
+    instance.hostname => {
+      "public_ip" : instance.network_interface[0].ip_address
+      "private_ip" : instance.network_interface[1].ip_address
+    }
+  }
 }

 output "worker_ip" {
-  value = local.worker_ip
-}
-
-output "bastion_ip" {
-  value = local.bastion_ip
+  value = {
+    for instance in upcloud_server.worker :
+    instance.hostname => {
+      "public_ip" : instance.network_interface[0].ip_address
+      "private_ip" : instance.network_interface[1].ip_address
+    }
+  }
 }

 output "loadbalancer_domain" {
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
@@ -20,21 +20,15 @@ variable "username" {}

 variable "private_network_cidr" {}

-variable "dns_servers" {}
-
-variable "use_public_ips" {}
-
 variable "machines" {
  description = "Cluster machines"
  type = map(object({
    node_type = string
    plan      = string
-    cpu       = optional(number)
-    mem       = optional(number)
+    cpu       = string
+    mem       = string
    disk_size = number
    server_group : string
-    force_public_ip : optional(bool, false)
-    dns_servers : optional(set(string))
    additional_disks = map(object({
      size = number
      tier = string
@@ -64,13 +58,6 @@ variable "k8s_allowed_remote_ips" {
  }))
 }

-variable "bastion_allowed_remote_ips" {
-  type = list(object({
-    start_address = string
-    end_address   = string
-  }))
-}
-
 variable "master_allowed_ports" {
  type = list(object({
    protocol       = string
@@ -107,6 +94,10 @@ variable "loadbalancer_plan" {
  type = string
 }

+variable "loadbalancer_outbound_proxy_protocol" {
+  type = string
+}
+
 variable "loadbalancer_legacy_network" {
  type = bool
  default = false
@@ -116,7 +107,6 @@ variable "loadbalancers" {
  description = "Load balancers"

  type = map(object({
-    proxy_protocol          = bool
    port                    = number
    target_port             = number
    allow_internal_frontend = optional(bool)
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf
@@ -3,7 +3,7 @@ terraform {
  required_providers {
    upcloud = {
      source  = "UpCloudLtd/upcloud"
-      version = "~>5.29.1"
+      version = "~>5.9.0"
    }
  }
  required_version = ">= 0.13"
--- a/contrib/terraform/upcloud/output.tf
+++ b/contrib/terraform/upcloud/output.tf
@@ -7,10 +7,6 @@ output "worker_ip" {
  value = module.kubernetes.worker_ip
 }

-output "bastion_ip" {
-  value = module.kubernetes.bastion_ip
-}
-
 output "loadbalancer_domain" {
  value = module.kubernetes.loadbalancer_domain
 }
--- a/contrib/terraform/upcloud/templates/inventory.tpl
+++ b/contrib/terraform/upcloud/templates/inventory.tpl
@@ -1,33 +1,17 @@
+
 [all]
-%{ for name, ips in master_ip ~}
-${name} ansible_user=${username} ansible_host=${lookup(ips, "public", ips.private)} ip=${ips.private}
-%{ endfor ~}
-%{ for name, ips in worker_ip ~}
-${name} ansible_user=${username} ansible_host=${lookup(ips, "public", ips.private)} ip=${ips.private}
-%{ endfor ~}
+${connection_strings_master}
+${connection_strings_worker}

 [kube_control_plane]
-%{ for name, ips in master_ip ~}
-${name}
-%{ endfor ~}
+${list_master}

 [etcd]
-%{ for name, ips in master_ip ~}
-${name}
-%{ endfor ~}
+${list_master}

 [kube_node]
-%{ for name, ips in worker_ip ~}
-${name}
-%{ endfor ~}
+${list_worker}

 [k8s_cluster:children]
 kube_control_plane
 kube_node
-
-%{ if length(bastion_ip) > 0 ~}
-[bastion]
-%{ for name, ips in bastion_ip ~}
-bastion ansible_user=${username} ansible_host=${ips.public}
-%{ endfor ~}
-%{ endif ~}
--- a/contrib/terraform/upcloud/variables.tf
+++ b/contrib/terraform/upcloud/variables.tf
@@ -32,31 +32,16 @@ variable "private_network_cidr" {
  default     = "172.16.0.0/24"
 }

-variable "dns_servers" {
-  description = "DNS servers that will be used by the nodes. Until [this is solved](https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562) this is done using user_data to reconfigure resolved"
-
-  type    = set(string)
-  default = []
-}
-
-variable "use_public_ips" {
-  description = "If all nodes should get a public IP"
-  type        = bool
-  default     = true
-}
-
 variable "machines" {
  description = "Cluster machines"

  type = map(object({
    node_type = string
    plan      = string
-    cpu       = optional(number)
-    mem       = optional(number)
+    cpu       = string
+    mem       = string
    disk_size = number
    server_group : string
-    force_public_ip : optional(bool, false)
-    dns_servers : optional(set(string))
    additional_disks = map(object({
      size = number
      tier = string
@@ -104,15 +89,6 @@ variable "k8s_allowed_remote_ips" {
  default = []
 }

-variable "bastion_allowed_remote_ips" {
-  description = "List of IP start/end addresses allowed to SSH to bastion"
-  type = list(object({
-    start_address = string
-    end_address   = string
-  }))
-  default = []
-}
-
 variable "master_allowed_ports" {
  description = "List of ports to allow on masters"
  type = list(object({
@@ -155,6 +131,11 @@ variable "loadbalancer_plan" {
  default     = "development"
 }

+variable "loadbalancer_proxy_protocol" {
+  type    = bool
+  default = false
+}
+
 variable "loadbalancer_legacy_network" {
  description = "If the loadbalancer should use the deprecated network field instead of networks blocks. You probably want to have this set to false"

@@ -166,7 +147,6 @@ variable "loadbalancers" {
  description = "Load balancers"

  type = map(object({
-    proxy_protocol          = bool
    port                    = number
    target_port             = number
    allow_internal_frontend = optional(bool, false)
--- a/contrib/terraform/upcloud/versions.tf
+++ b/contrib/terraform/upcloud/versions.tf
@@ -3,7 +3,7 @@ terraform {
  required_providers {
    upcloud = {
      source  = "UpCloudLtd/upcloud"
-      version = "~>5.29.1"
+      version = "~>5.9.0"
    }
  }
  required_version = ">= 0.13"
--- a/docs/CNI/calico.md
+++ b/docs/CNI/calico.md
@@ -180,7 +180,7 @@ calico_group_id=rr1
 The inventory above will deploy the following topology assuming that calico's
 `global_as_num` is set to `65400`:

-![Image](../figures/kubespray-calico-rr.png?raw=true)
+![Image](figures/kubespray-calico-rr.png?raw=true)

 ### Optional : Define default endpoint to host action

@@ -377,7 +377,7 @@ To clean up any ipvs leftovers:

 ### Calico access to the kube-api

-Calico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.tigera.io/calico/latest/operations/ebpf/enabling-ebpf) for guidelines on how to do this.
+Calico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.projectcalico.org/maintenance/ebpf/enabling-bpf) for guidelines on how to do this.

 Kubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `loadbalancer_apiserver` inventory variable documented in [HA Mode](/docs/operations/ha-mode.md).

--- a/docs/CNI/cilium.md
+++ b/docs/CNI/cilium.md
@@ -1,13 +1,5 @@
 # Cilium

-## Unprivileged agent configuration
-
-By default, Cilium is installed with `securityContext.privileged: false`. You need to set the `kube_owner` variable to `root` in the inventory:
-
-```yml
-kube_owner: root
-```
-
 ## IP Address Management (IPAM)

 IP Address Management (IPAM) is responsible for the allocation and management of IP addresses used by network endpoints (container and others) managed by Cilium. The default mode is "Cluster Scope".
@@ -62,10 +54,6 @@ cilium_loadbalancer_ip_pools:
  - name: "blue-pool"
    cidrs:
      - "10.0.10.0/24"
-    ranges:
-      - start: "20.0.20.100"
-        stop: "20.0.20.200"
-      - start: "1.2.3.4"
 ```

 For further information, check [LB IPAM documentation](https://docs.cilium.io/en/stable/network/lb-ipam/)
@@ -245,7 +233,7 @@ cilium_operator_extra_volume_mounts:
 ## Choose Cilium version

 ```yml
-cilium_version: "1.18.6"
+cilium_version: "1.15.9"
 ```

 ## Add variable to config
--- a/docs/CNI/macvlan.md
+++ b/docs/CNI/macvlan.md
@@ -32,7 +32,7 @@ add `kube_proxy_masquerade_all: true` in `group_vars/all/all.yml`

 * Disable nodelocaldns

-The nodelocal dns IP is not reachable.
+The nodelocal dns IP is not reacheable.

 Disable it in `sample/group_vars/k8s_cluster/k8s_cluster.yml`

--- a/docs/CNI/weave.md
+++ b/docs/CNI/weave.md
@@ -0,0 +1,79 @@
+# Weave
+
+Weave 2.0.1 is supported by kubespray
+
+Weave uses [**consensus**](https://www.weave.works/docs/net/latest/ipam/##consensus) mode (default mode) and [**seed**](https://www.weave.works/docs/net/latest/ipam/#seed) mode.
+
+`Consensus` mode is best to use on static size cluster and `seed` mode is best to use on dynamic size cluster
+
+Weave encryption is supported for all communication
+
+* To use Weave encryption, specify a strong password (if no password, no encryption)
+
+```ShellSession
+# In file ./inventory/sample/group_vars/k8s_cluster.yml
+weave_password: EnterPasswordHere
+```
+
+This password is used to set an environment variable inside weave container.
+
+Weave is deployed by kubespray using a daemonSet
+
+* Check the status of Weave containers
+
+```ShellSession
+# From client
+kubectl -n kube-system get pods | grep weave
+# output
+weave-net-50wd2                       2/2       Running   0          2m
+weave-net-js9rb                       2/2       Running   0          2m
+```
+
+There must be as many pods as nodes (here kubernetes have 2 nodes so there are 2 weave pods).
+
+* Check status of weave (connection,encryption ...) for each node
+
+```ShellSession
+# On nodes
+curl http://127.0.0.1:6784/status
+# output on node1
+Version: 2.0.1 (up to date; next check at 2017/08/01 13:51:34)
+
+        Service: router
+       Protocol: weave 1..2
+           Name: fa:16:3e:b3:d6:b2(node1)
+     Encryption: enabled
+  PeerDiscovery: enabled
+        Targets: 2
+    Connections: 2 (1 established, 1 failed)
+          Peers: 2 (with 2 established connections)
+ TrustedSubnets: none
+
+        Service: ipam
+         Status: ready
+          Range: 10.233.64.0/18
+  DefaultSubnet: 10.233.64.0/18
+```
+
+* Check parameters of weave for each node
+
+```ShellSession
+# On nodes
+ps -aux | grep weaver
+# output on node1 (here its use seed mode)
+root      8559  0.2  3.0 365280 62700 ?        Sl   08:25   0:00 /home/weave/weaver --name=fa:16:3e:b3:d6:b2 --port=6783 --datapath=datapath --host-root=/host --http-addr=127.0.0.1:6784 --status-addr=0.0.0.0:6782 --docker-api= --no-dns --db-prefix=/weavedb/weave-net --ipalloc-range=10.233.64.0/18 --nickname=node1 --ipalloc-init seed=fa:16:3e:b3:d6:b2,fa:16:3e:f0:50:53 --conn-limit=30 --expect-npc 192.168.208.28 192.168.208.19
+```
+
+## Consensus mode (default mode)
+
+This mode is best to use on static size cluster
+
+### Seed mode
+
+This mode is best to use on dynamic size cluster
+
+The seed mode also allows multi-clouds and hybrid on-premise/cloud clusters deployment.
+
+* Switch from consensus mode to seed/Observation mode
+
+See [weave ipam documentation](https://www.weave.works/docs/net/latest/tasks/ipam/ipam/) and use `weave_extra_args` to enable.
--- a/docs/CRI/containerd.md
+++ b/docs/CRI/containerd.md
@@ -65,10 +65,11 @@ In kubespray, the default runtime name is "runc", and it can be configured with
 containerd_runc_runtime:
  name: runc
  type: "io.containerd.runc.v2"
+  engine: ""
+  root: ""
  options:
-    Root: ""
-    SystemdCgroup: "false"
-    BinaryName: /usr/local/bin/my-runc
+    systemdCgroup: "false"
+    binaryName: /usr/local/bin/my-runc
  base_runtime_spec: cri-base.json
 ```

@@ -148,11 +149,3 @@ following configuration:
 ```yaml
 nri_enabled: true
 ```
-
-### Optional : Static Binary
-
-To ensure compatibility with older distributions (such as Debian 11), you can use a static containerd binary. By default, this is static binary if the system's glibc version is less than 2.34; otherwise, it is the default binary.
-
-```yaml
-containerd_static_binary: true
-```
--- a/docs/CRI/cri-o.md
+++ b/docs/CRI/cri-o.md
@@ -80,7 +80,7 @@ The `crio_remap_enable` configures the `/etc/subuid` and `/etc/subgid` files to
 By default, 16M uids and gids are reserved for user namespaces (256 pods * 65536 uids/gids) at the end of the uid/gid space.

 The `crio_default_capabilities` configure the default containers capabilities for the crio.
-Defaults capabilities are:
+Defaults capabilties are:

 ```yaml
 crio_default_capabilities:
--- a/docs/_sidebar.md
+++ b/docs/_sidebar.md
@@ -6,6 +6,7 @@
  * [Downloads](/docs/advanced/downloads.md)
  * [Gcp-lb](/docs/advanced/gcp-lb.md)
  * [Kubernetes-reliability](/docs/advanced/kubernetes-reliability.md)
+  * [Mitogen](/docs/advanced/mitogen.md)
  * [Netcheck](/docs/advanced/netcheck.md)
  * [Ntp](/docs/advanced/ntp.md)
  * [Proxy](/docs/advanced/proxy.md)
@@ -22,6 +23,7 @@
  * [Aws](/docs/cloud_providers/aws.md)
  * [Azure](/docs/cloud_providers/azure.md)
  * [Cloud](/docs/cloud_providers/cloud.md)
+  * [Equinix-metal](/docs/cloud_providers/equinix-metal.md)
 * CNI
  * [Calico](/docs/CNI/calico.md)
  * [Cilium](/docs/CNI/cilium.md)
@@ -31,6 +33,7 @@
  * [Kube-router](/docs/CNI/kube-router.md)
  * [Macvlan](/docs/CNI/macvlan.md)
  * [Multus](/docs/CNI/multus.md)
+  * [Weave](/docs/CNI/weave.md)
 * CRI
  * [Containerd](/docs/CRI/containerd.md)
  * [Cri-o](/docs/CRI/cri-o.md)
@@ -49,7 +52,9 @@
  * [Test Cases](/docs/developers/test_cases.md)
  * [Vagrant](/docs/developers/vagrant.md)
 * External Storage Provisioners
+  * [Cephfs Provisioner](/docs/external_storage_provisioners/cephfs_provisioner.md)
  * [Local Volume Provisioner](/docs/external_storage_provisioners/local_volume_provisioner.md)
+  * [Rbd Provisioner](/docs/external_storage_provisioners/rbd_provisioner.md)
  * [Scheduler Plugins](/docs/external_storage_provisioners/scheduler_plugins.md)
 * Getting Started
  * [Comparisons](/docs/getting_started/comparisons.md)
--- a/docs/advanced/arch.md
+++ b/docs/advanced/arch.md
@@ -9,6 +9,7 @@ The following table shows the impact of the CPU architecture on compatible featu
 | kube_network_plugin | amd64 | arm64 | amd64 + arm64 |
 |---------------------|-------|-------|---------------|
 | Calico              | Y     | Y     | Y             |
+| Weave               | Y     | Y     | Y             |
 | Flannel             | Y     | N     | N             |
 | Canal               | Y     | N     | N             |
 | Cilium              | Y     | Y     | N             |
--- a/docs/advanced/cert_manager.md
+++ b/docs/advanced/cert_manager.md
@@ -6,7 +6,7 @@
    - [Create New TLS Root CA Certificate and Key](#create-new-tls-root-ca-certificate-and-key)
      - [Install Cloudflare PKI/TLS `cfssl` Toolkit.](#install-cloudflare-pkitls-cfssl-toolkit)
      - [Create Root Certificate Authority (CA) Configuration File](#create-root-certificate-authority-ca-configuration-file)
-      - [Create Certificate Signing Request (CSR) Configuration File](#create-certificate-signing-request-csr-configuration-file)
+      - [Create Certficate Signing Request (CSR) Configuration File](#create-certficate-signing-request-csr-configuration-file)
      - [Create TLS Root CA Certificate and Key](#create-tls-root-ca-certificate-and-key)

 Cert-Manager is a native Kubernetes certificate management controller. It can help with issuing certificates from a variety of sources, such as Let’s Encrypt, HashiCorp Vault, Venafi, a simple signing key pair, or self signed. It will ensure certificates are valid and up to date, and attempt to renew certificates at a configured time before expiry.
@@ -134,7 +134,7 @@ $ cat > ca-config.json <<EOF
 EOF
 ```

-#### Create Certificate Signing Request (CSR) Configuration File
+#### Create Certficate Signing Request (CSR) Configuration File

 The TLS certificate `names` details can be updated to your own specific requirements.

--- a/docs/advanced/gcp-lb.md
+++ b/docs/advanced/gcp-lb.md
@@ -1,4 +1,4 @@
-# GCP Load Balancers for type=LoadBalancer of Kubernetes Services
+# GCP Load Balancers for type=LoadBalacer of Kubernetes Services

 > **Removed**: Since v1.31 (the Kubespray counterpart is v2.27), Kubernetes no longer supports `cloud_provider`. (except external cloud provider)

--- a/docs/advanced/mitogen.md
+++ b/docs/advanced/mitogen.md
@@ -0,0 +1,30 @@
+# Mitogen
+
+*Warning:* Mitogen support is now deprecated in kubespray due to upstream not releasing an updated version to support ansible 4.x (ansible-base 2.11.x) and above. The CI support has been stripped for mitogen and we are no longer validating any support or regressions for it. The supporting mitogen install playbook and integration documentation will be removed in a later version.
+
+[Mitogen for Ansible](https://mitogen.networkgenomics.com/ansible_detailed.html) allow a 1.25x - 7x speedup and a CPU usage reduction of at least 2x, depending on network conditions, modules executed, and time already spent by targets on useful work. Mitogen cannot improve a module once it is executing, it can only ensure the module executes as quickly as possible.
+
+## Install
+
+```ShellSession
+ansible-playbook contrib/mitogen/mitogen.yml
+```
+
+The above playbook sets the ansible `strategy` and `strategy_plugins` in `ansible.cfg` but you can also enable them if you use your own `ansible.cfg` by setting the environment varialbles:
+
+```ShellSession
+export ANSIBLE_STRATEGY=mitogen_linear
+export ANSIBLE_STRATEGY_PLUGINS=plugins/mitogen/ansible_mitogen/plugins/strategy
+```
+
+... or `ansible.cfg` setup:
+
+```ini
+[defaults]
+strategy_plugins = plugins/mitogen/ansible_mitogen/plugins/strategy
+strategy=mitogen_linear
+```
+
+## Limitation
+
+If you are experiencing problems, please see the [documentation](https://mitogen.networkgenomics.com/ansible_detailed.html#noteworthy-differences).
--- a/docs/advanced/proxy.md
+++ b/docs/advanced/proxy.md
@@ -1,6 +1,6 @@
 # Setting up Environment Proxy

-If you set http and https proxy, all nodes and loadbalancer will be excluded from proxy with generating no_proxy variable in `roles/kubespray_defaults/tasks/no_proxy.yml`, if you have additional resources for exclude add them to `additional_no_proxy` variable. If you want fully override your `no_proxy` setting, then fill in just `no_proxy` and no nodes or loadbalancer addresses will be added to no_proxy.
+If you set http and https proxy, all nodes and loadbalancer will be excluded from proxy with generating no_proxy variable in `roles/kubespray-defaults/tasks/no_proxy.yml`, if you have additional resources for exclude add them to `additional_no_proxy` variable. If you want fully override your `no_proxy` setting, then fill in just `no_proxy` and no nodes or loadbalancer addresses will be added to no_proxy.

 ## Set proxy for http and https

--- a/docs/ansible/ansible.md
+++ b/docs/ansible/ansible.md
@@ -13,7 +13,7 @@ KUBESPRAYDIR=kubespray
 python3 -m venv $VENVDIR
 source $VENVDIR/bin/activate
 cd $KUBESPRAYDIR
-pip install -r requirements.txt
+pip install -U -r requirements.txt
 ```

 In case you have a similar message when installing the requirements:
@@ -32,7 +32,7 @@ Based on the table below and the available python version for your ansible host

 | Ansible Version | Python Version |
 |-----------------|----------------|
-| >= 2.17.3       | 3.10-3.12      |
+| >= 2.16.4       | 3.10-3.12      |

 ## Customize Ansible vars

@@ -42,10 +42,13 @@ Kubespray expects users to use one of the following variables sources for settin
 |----------------------------------------|------------------------------------------------------------------------------|
 | inventory vars                         |                                                                              |
 |  - **inventory group_vars**            | most used                                                                    |
-|  - inventory host_vars                 | host specific vars overrides, group_vars is usually more practical           |
+|  - inventory host_vars                 | host specifc vars overrides, group_vars is usually more practical            |
 | **extra vars** (always win precedence) | override with ``ansible-playbook -e @foo.yml``                               |

-> Extra vars are best used to override kubespray internal variables, for instances, roles/vars/. Those vars are usually **not expected** (by Kubespray developers) to be modified by end users, and not part of Kubespray interface. Thus they can change, disappear, or break stuff unexpectedly.
+[!IMPORTANT]
+Extra vars are best used to override kubespray internal variables, for instances, roles/vars/.
+Those vars are usually **not expected** (by Kubespray developers) to be modified by end users, and not part of Kubespray
+interface. Thus they can change, disappear, or break stuff unexpectedly.

 ## Ansible tags

@@ -59,9 +62,10 @@ The following tags are defined in playbooks:
 | aws-ebs-csi-driver             | Configuring csi driver: aws-ebs                       |
 | azure-csi-driver               | Configuring csi driver: azure                         |
 | bastion                        | Setup ssh config for bastion                          |
-| bootstrap_os                   | Anything related to host OS configuration             |
+| bootstrap-os                   | Anything related to host OS configuration             |
 | calico                         | Network plugin Calico                                 |
 | calico_rr                      | Configuring Calico route reflector                    |
+| cephfs-provisioner             | Configuring CephFS                                    |
 | cert-manager                   | Configuring certificate manager for K8s               |
 | cilium                         | Network plugin Cilium                                 |
 | cinder-csi-driver              | Configuring csi driver: cinder                        |
@@ -115,11 +119,12 @@ The following tags are defined in playbooks:
 | local-path-provisioner         | Configure External provisioner: local-path            |
 | local-volume-provisioner       | Configure External provisioner: local-volume          |
 | macvlan                        | Network plugin macvlan                                |
+| master (DEPRECATED)            | Deprecated - see `control-plane`                      |
 | metallb                        | Installing and configuring metallb                    |
 | metrics_server                 | Configuring metrics_server                            |
 | netchecker                     | Installing netchecker K8s app                         |
 | network                        | Configuring networking plugins for K8s                |
-| mounts                         | Umount kubelet dirs when resetting                    |
+| mounts                         | Umount kubelet dirs when reseting                     |
 | multus                         | Network plugin multus                                 |
 | nginx                          | Configuring LB for kube-apiserver instances           |
 | node                           | Configuring K8s minion (compute) node role            |
@@ -142,6 +147,7 @@ The following tags are defined in playbooks:
 | registry                       | Configuring local docker registry                     |
 | reset                          | Tasks running doing the node reset                    |
 | resolvconf                     | Configuring /etc/resolv.conf for hosts/apps           |
+| rbd-provisioner                | Configure External provisioner: rdb                   |
 | services                       | Remove services (etcd, kubelet etc...) when resetting |
 | snapshot                       | Enabling csi snapshot                                 |
 | snapshot-controller            | Configuring csi snapshot controller                   |
@@ -149,16 +155,21 @@ The following tags are defined in playbooks:
 | upgrade                        | Upgrading, f.e. container images/binaries             |
 | upload                         | Distributing images/binaries across hosts             |
 | vsphere-csi-driver             | Configuring csi driver: vsphere                       |
+| weave                          | Network plugin Weave                                  |
 | win_nodes                      | Running windows specific tasks                        |
 | youki                          | Configuring youki runtime                             |

+Note: Use the ``bash scripts/gen_tags.sh`` command to generate a list of all
+tags found in the codebase. New tags will be listed with the empty "Used for"
+field.
+
 ## Example commands

 Example command to filter and apply only DNS configuration tasks and skip
 everything else related to host OS configuration and downloading images of containers:

 ```ShellSession
-ansible-playbook -i inventory/sample/hosts.ini cluster.yml --tags preinstall,facts --skip-tags=download,bootstrap_os
+ansible-playbook -i inventory/sample/hosts.ini cluster.yml --tags preinstall,facts --skip-tags=download,bootstrap-os
 ```

 And this play only removes the K8s cluster DNS resolver IP from hosts' /etc/resolv.conf files:
@@ -178,13 +189,17 @@ ansible-playbook -i inventory/sample/hosts.ini cluster.yml \

 Note: use `--tags` and `--skip-tags` wisely and only if you're 100% sure what you're doing.

+## Mitogen
+
+Mitogen support is deprecated, please see [mitogen related docs](/docs/advanced/mitogen.md) for usage and reasons for deprecation.
+
 ## Troubleshooting Ansible issues

 Having the wrong version of ansible, ansible collections or python dependencies can cause issue.
-In particular, Kubespray ship custom modules which Ansible needs to find, for which you should specify [ANSIBLE_LIBRARY](https://docs.ansible.com/ansible/latest/dev_guide/developing_locally.html#adding-a-module-or-plugin-outside-of-a-collection)
+In particular, Kubespray ship custom modules which Ansible needs to find, for which you should specify [ANSIBLE_LIBRAY](https://docs.ansible.com/ansible/latest/dev_guide/developing_locally.html#adding-a-module-or-plugin-outside-of-a-collection)

 ```ShellSession
-export ANSIBLE_LIBRARY=<kubespray_dir>/library`
+export ANSIBLE_LIBRAY=<kubespray_dir>/library`
 ```

 A simple way to ensure you get all the correct version of Ansible is to use
@@ -193,11 +208,11 @@ You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mou
 to access the inventory and SSH key in the container, like this:

 ```ShellSession
-git checkout v2.30.0
-docker pull quay.io/kubespray/kubespray:v2.30.0
+git checkout v2.27.0
+docker pull quay.io/kubespray/kubespray:v2.27.0
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
  --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.30.0 bash
+  quay.io/kubespray/kubespray:v2.27.0 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```
--- a/docs/ansible/ansible_collection.md
+++ b/docs/ansible/ansible_collection.md
@@ -2,13 +2,14 @@

 Kubespray can be installed as an [Ansible collection](https://docs.ansible.com/ansible/latest/user_guide/collections_using.html).

+## Requirements
+
+- An inventory file with the appropriate host groups. See the [README](../README.md#usage).
+- A `group_vars` directory. These group variables **need** to match the appropriate variable names under `inventory/local/group_vars`. See the [README](../README.md#usage).
+
 ## Usage

-1. Set up an inventory with the appropriate host groups and required group vars.
-   See also the documentation on [kubespray inventories](./inventory.md) and the
-   general ["Getting started" documentation](../getting_started/getting-started.md#building-your-own-inventory).
-
-2. Add Kubespray to your requirements.yml file
+1. Add Kubespray to your requirements.yml file

   ```yaml
   collections:
@@ -17,20 +18,20 @@ Kubespray can be installed as an [Ansible collection](https://docs.ansible.com/a
     version: master # use the appropriate tag or branch for the version you need
   ```

-3. Install your collection
+2. Install your collection

   ```ShellSession
   ansible-galaxy install -r requirements.yml
   ```

-4. Create a playbook to install your Kubernetes cluster
+3. Create a playbook to install your Kubernetes cluster

   ```yaml
   - name: Install Kubernetes
     ansible.builtin.import_playbook: kubernetes_sigs.kubespray.cluster
   ```

-5. Update INVENTORY and PLAYBOOK so that they point to your inventory file and the playbook you created above, and then install Kubespray
+4. Update INVENTORY and PLAYBOOK so that they point to your inventory file and the playbook you created above, and then install Kubespray

   ```ShellSession
   ansible-playbook -i INVENTORY --become --become-user=root PLAYBOOK
--- a/docs/ansible/vars.md
+++ b/docs/ansible/vars.md
@@ -103,13 +103,13 @@ following default cluster parameters:

 * *kube_service_addresses_ipv6* - Subnet for cluster IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1000/116``). Must not overlap with ``kube_pods_subnet_ipv6``.

-* *kube_service_subnets* - All service subnets separated by commas (default is a mix of ``kube_service_addresses`` and ``kube_service_addresses_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stack`` options),
+* *kube_service_subnets* - All service subnets separated by commas (default is a mix of ``kube_service_addresses`` and ``kube_service_addresses_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stacke`` options),
  for example ``10.233.0.0/18,fd85:ee78:d8a6:8607::1000/116`` for dual stack(ipv4_stack/ipv6_stack set to `true`).
  It is not recommended to change this variable directly.

 * *kube_pods_subnet_ipv6* - Subnet for Pod IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1:0000/112``). Must not overlap with ``kube_service_addresses_ipv6``.

-* *kube_pods_subnets* - All pods subnets separated by commas (default is a mix of ``kube_pods_subnet`` and ``kube_pod_subnet_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stack`` options),
+* *kube_pods_subnets* - All pods subnets separated by commas (default is a mix of ``kube_pods_subnet`` and ``kube_pod_subnet_ipv6`` depending on ``ipv4_stack`` and ``ipv6_stacke`` options),
  for example ``10.233.64.0/18,fd85:ee78:d8a6:8607::1:0000/112`` for dual stack(ipv4_stack/ipv6_stack set to `true`).
  It is not recommended to change this variable directly.

@@ -180,7 +180,7 @@ and ``kube_pods_subnet``, for example from the ``172.18.0.0/16``.

 IPv4 stack enable by *ipv4_stack* is set to ``true``, by default.
 IPv6 stack enable by *ipv6_stack* is set to ``false`` by default.
-This will use the default IPv4 and IPv6 subnets specified in the defaults file in the ``kubespray_defaults`` role, unless overridden of course. The default config will give you room for up to 256 nodes with 126 pods per node, and up to 4096 services.
+This will use the default IPv4 and IPv6 subnets specified in the defaults file in the ``kubespray-defaults`` role, unless overridden of course. The default config will give you room for up to 256 nodes with 126 pods per node, and up to 4096 services.
 Set both variables to ``true`` for Dual Stack mode.
 IPv4 has higher priority in Dual Stack mode(e.g. in variables `main_ip`, `main_access_ip` and other).
 You can also make IPv6 only clusters with ``false`` in *ipv4_stack*.
--- a/docs/cloud_controllers/openstack.md
+++ b/docs/cloud_controllers/openstack.md
@@ -73,7 +73,6 @@ The cloud provider is configured to have Octavia by default in Kubespray.
  external_openstack_lbaas_method: ROUND_ROBIN
  external_openstack_lbaas_provider: amphora
  external_openstack_lbaas_subnet_id: "Neutron subnet ID to create LBaaS VIP"
-  external_openstack_lbaas_member_subnet_id: "Neutron subnet ID on which to create the members of the load balancer"
  external_openstack_lbaas_network_id: "Neutron network ID to create LBaaS VIP"
  external_openstack_lbaas_manage_security_groups: false
  external_openstack_lbaas_create_monitor: false
--- a/docs/cloud_providers/equinix-metal.md
+++ b/docs/cloud_providers/equinix-metal.md
@@ -0,0 +1,100 @@
+# Equinix Metal
+
+Kubespray provides support for bare metal deployments using the [Equinix Metal](http://metal.equinix.com).
+Deploying upon bare metal allows Kubernetes to run at locations where an existing public or private cloud might not exist such
+as cell tower, edge collocated installations. The deployment mechanism used by Kubespray for Equinix Metal is similar to that used for
+AWS and OpenStack clouds (notably using Terraform to deploy the infrastructure). Terraform uses the Equinix Metal provider plugin
+to provision and configure hosts which are then used by the Kubespray Ansible playbooks. The Ansible inventory is generated
+dynamically from the Terraform state file.
+
+## Local Host Configuration
+
+To perform this installation, you will need a localhost to run Terraform/Ansible (laptop, VM, etc) and an account with Equinix Metal.
+In this example, we are provisioning a m1.large CentOS7 OpenStack VM as the localhost for the Kubernetes installation.
+You'll need Ansible, Git, and PIP.
+
+```bash
+sudo yum install epel-release
+sudo yum install ansible
+sudo yum install git
+sudo yum install python-pip
+```
+
+## Playbook SSH Key
+
+An SSH key is needed by Kubespray/Ansible to run the playbooks.
+This key is installed into the bare metal hosts during the Terraform deployment.
+You can generate a key new key or use an existing one.
+
+```bash
+ssh-keygen -f ~/.ssh/id_rsa
+```
+
+## Install Terraform
+
+Terraform is required to deploy the bare metal infrastructure. The steps below are for installing on CentOS 7.
+[More terraform installation options are available.](https://learn.hashicorp.com/terraform/getting-started/install.html)
+
+Grab the latest version of Terraform and install it.
+
+```bash
+echo "https://releases.hashicorp.com/terraform/$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version')/terraform_$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r -M '.current_version')_linux_amd64.zip"
+sudo yum install unzip
+sudo unzip terraform_0.14.10_linux_amd64.zip -d /usr/local/bin/
+```
+
+## Download Kubespray
+
+Pull over Kubespray and setup any required libraries.
+
+```bash
+git clone https://github.com/kubernetes-sigs/kubespray
+cd kubespray
+```
+
+## Install Ansible
+
+Install Ansible according to [Ansible installation guide](/docs/ansible/ansible.md#installing-ansible)
+
+## Cluster Definition
+
+In this example, a new cluster called "alpha" will be created.
+
+```bash
+cp -LRp contrib/terraform/packet/sample-inventory inventory/alpha
+cd inventory/alpha/
+ln -s ../../contrib/terraform/packet/hosts
+```
+
+Details about the cluster, such as the name, as well as the authentication tokens and project ID
+for Equinix Metal need to be defined. To find these values see [Equinix Metal API Accounts](https://metal.equinix.com/developers/docs/accounts/).
+
+```bash
+vi cluster.tfvars
+```
+
+* cluster_name = alpha
+* packet_project_id = ABCDEFGHIJKLMNOPQRSTUVWXYZ123456
+* public_key_path = 12345678-90AB-CDEF-GHIJ-KLMNOPQRSTUV
+
+## Deploy Bare Metal Hosts
+
+Initializing Terraform will pull down any necessary plugins/providers.
+
+```bash
+terraform init ../../contrib/terraform/packet/
+```
+
+Run Terraform to deploy the hardware.
+
+```bash
+terraform apply -var-file=cluster.tfvars ../../contrib/terraform/packet
+```
+
+## Run Kubespray Playbooks
+
+With the bare metal infrastructure deployed, Kubespray can now install Kubernetes and setup the cluster.
+
+```bash
+ansible-playbook --become -i inventory/alpha/hosts cluster.yml
+```
--- a/docs/developers/ci-setup.md
+++ b/docs/developers/ci-setup.md
@@ -2,14 +2,19 @@

 ## Pipeline

-See [.gitlab-ci.yml](/.gitlab-ci.yml) and the included files for an overview.
+1. build: build a docker image to be used in the pipeline
+2. unit-tests: fast jobs for fast feedback (linting, etc...)
+3. deploy-part1: small number of jobs to test if the PR works with default settings
+4. deploy-extended: slow jobs testing different platforms, OS, settings, CNI, etc...
+5. deploy-extended: very slow jobs (upgrades, etc...)

 ## Runners

-Kubespray has 2 types of GitLab runners, both deployed on the Kubespray CI cluster (hosted on Oracle Cloud Infrastructure):
+Kubespray has 3 types of GitLab runners:

- pods: use the [gitlab-ci kubernetes executor](https://docs.gitlab.com/runner/executors/kubernetes/)
- vagrant: custom executor running in pods with access to the libvirt socket on the nodes
+- packet runners: used for E2E jobs (usually long), running on Equinix Metal (ex-packet), on kubevirt managed VMs
+- light runners: used for short lived jobs, running on Equinix Metal (ex-packet), as managed pods
+- auto scaling runners (managed via docker-machine on Equinix Metal): used for on-demand resources, see [GitLab docs](https://docs.gitlab.com/runner/configuration/autoscale.html) for more info

 ## Vagrant

@@ -17,17 +22,18 @@ Vagrant jobs are using the [quay.io/kubespray/vagrant](/test-infra/vagrant-docke

 ## CI Variables

-In CI we have a [set of extra vars](/test/common_vars.yml) we use to ensure greater success of our CI jobs and avoid throttling by various APIs we depend on.
+In CI we have a set of overrides we use to ensure greater success of our CI jobs and avoid throttling by various APIs we depend on. See:

-## CI clusters
+- [Docker mirrors](/tests/common/_docker_hub_registry_mirror.yml)
+- [Test settings](/tests/common/_kubespray_test_settings.yml)

-DISCLAIMER: The following information is not fully up to date, in particular, the CI cluster is now on Oracle Cloud Infrastcture, not Equinix.
+## CI Environment

-The cluster is deployed with kubespray itself and maintained by the kubespray maintainers.
+The CI packet and light runners are deployed on a kubernetes cluster on Equinix Metal. The cluster is deployed with kubespray itself and maintained by the kubespray maintainers.

 The following files are used for that inventory:

-### cluster.tfvars (OBSOLETE: this section is no longer accurate)
+### cluster.tfvars

 ```ini
 # your Kubernetes cluster name here
@@ -156,10 +162,22 @@ kube_feature_gates:
  - "NodeSwap=True"
 ```

-## Additional files
+## Aditional files

 This section documents additional files used to complete a deployment of the kubespray CI, these files sit on the control-plane node and assume a working kubernetes cluster.

+### /root/nscleanup.sh
+
+```bash
+#!/bin/bash
+
+kubectl=/usr/local/bin/kubectl
+
+$kubectl get ns | grep -P "(\d.+-\d.+)" | awk 'match($3,/[0-9]+d/) {print $1}' | xargs -r $kubectl delete ns
+$kubectl get ns | grep -P "(\d.+-\d.+)" | awk 'match($3,/[3-9]+h/) {print $1}' | xargs -r $kubectl delete ns
+$kubectl get ns | grep Terminating | awk '{print $1}' | xargs -i $kubectl delete vmi/instance-1 vmi/instance-0 vmi/instance-2 -n {} --force --grace-period=0 &
+```
+
 ### /root/path-calico.sh

 ```bash
--- a/docs/developers/ci.md
+++ b/docs/developers/ci.md
@@ -6,52 +6,55 @@ To generate this Matrix run `./tests/scripts/md-table/main.py`

 | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |
 |---| --- | --- | --- | --- | --- | --- | --- |
+almalinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 almalinux9 |  :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
 amazon |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: |
 debian12 |  :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: |
-debian13 |  :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
 fedora40 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-flatcar4081 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-rockylinux10 |  :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
+opensuse15 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+rockylinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
+ubuntu20 |  :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: |
 ubuntu22 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu24 |  :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: |
+ubuntu24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |

 ## crio

 | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |
 |---| --- | --- | --- | --- | --- | --- | --- |
+almalinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 almalinux9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian13 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora40 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-flatcar4081 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora40 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-rockylinux10 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+opensuse15 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu22 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu22 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |

 ## docker

 | OS / CNI | calico | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan |
 |---| --- | --- | --- | --- | --- | --- | --- |
+almalinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 almalinux9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian13 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora40 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-flatcar4081 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-rockylinux10 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+opensuse15 |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
+rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu22 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
--- a/docs/external_storage_provisioners/cephfs_provisioner.md
+++ b/docs/external_storage_provisioners/cephfs_provisioner.md
@@ -0,0 +1,73 @@
+# CephFS Volume Provisioner for Kubernetes 1.5+
+
+[![Docker Repository on Quay](https://quay.io/repository/external_storage/cephfs-provisioner/status "Docker Repository on Quay")](https://quay.io/repository/external_storage/cephfs-provisioner)
+
+Using Ceph volume client
+
+## Development
+
+Compile the provisioner
+
+``` console
+make
+```
+
+Make the container image and push to the registry
+
+``` console
+make push
+```
+
+## Test instruction
+
+- Start Kubernetes local cluster
+
+See [Kubernetes](https://kubernetes.io/)
+
+- Create a Ceph admin secret
+
+``` bash
+ceph auth get client.admin 2>&1 |grep "key = " |awk '{print  $3'} |xargs echo -n > /tmp/secret
+kubectl create ns cephfs
+kubectl create secret generic ceph-secret-admin --from-file=/tmp/secret --namespace=cephfs
+```
+
+- Start CephFS provisioner
+
+The following example uses `cephfs-provisioner-1` as the identity for the instance and assumes kubeconfig is at `/root/.kube`. The identity should remain the same if the provisioner restarts. If there are multiple provisioners, each should have a different identity.
+
+``` bash
+docker run -ti -v /root/.kube:/kube -v /var/run/kubernetes:/var/run/kubernetes --privileged --net=host cephfs-provisioner /usr/local/bin/cephfs-provisioner -master=http://127.0.0.1:8080 -kubeconfig=/kube/config -id=cephfs-provisioner-1
+```
+
+Alternatively, deploy it in kubernetes, see [deployment](deploy/README.md).
+
+- Create a CephFS Storage Class
+
+Replace Ceph monitor's IP in [example class](example/class.yaml) with your own and create storage class:
+
+``` bash
+kubectl create -f example/class.yaml
+```
+
+- Create a claim
+
+``` bash
+kubectl create -f example/claim.yaml
+```
+
+- Create a Pod using the claim
+
+``` bash
+kubectl create -f example/test-pod.yaml
+```
+
+## Known limitations
+
+- Kernel CephFS doesn't work with SELinux, setting SELinux label in Pod's securityContext will not work.
+- Kernel CephFS doesn't support quota or capacity, capacity requested by PVC is not enforced or validated.
+- Currently each Ceph user created by the provisioner has `allow r` MDS cap to permit CephFS mount.
+
+## Acknowledgement
+
+Inspired by CephFS Manila provisioner and conversation with John Spray
--- a/docs/external_storage_provisioners/rbd_provisioner.md
+++ b/docs/external_storage_provisioners/rbd_provisioner.md
@@ -0,0 +1,79 @@
+# RBD Volume Provisioner for Kubernetes 1.5+
+
+`rbd-provisioner` is an out-of-tree dynamic provisioner for Kubernetes 1.5+.
+You can use it quickly & easily deploy ceph RBD storage that works almost
+anywhere.
+
+It works just like in-tree dynamic provisioner. For more information on how
+dynamic provisioning works, see [the docs](https://kubernetes.io/docs/concepts/storage/persistent-volumes/)
+or [this blog post](http://blog.kubernetes.io/2016/10/dynamic-provisioning-and-storage-in-kubernetes.html).
+
+## Development
+
+Compile the provisioner
+
+```console
+make
+```
+
+Make the container image and push to the registry
+
+```console
+make push
+```
+
+## Test instruction
+
+* Start Kubernetes local cluster
+
+See [Kubernetes](https://kubernetes.io/).
+
+* Create a Ceph admin secret
+
+```bash
+ceph auth get client.admin 2>&1 |grep "key = " |awk '{print  $3'} |xargs echo -n > /tmp/secret
+kubectl create secret generic ceph-admin-secret --from-file=/tmp/secret --namespace=kube-system
+```
+
+* Create a Ceph pool and a user secret
+
+```bash
+ceph osd pool create kube 8 8
+ceph auth add client.kube mon 'allow r' osd 'allow rwx pool=kube'
+ceph auth get-key client.kube > /tmp/secret
+kubectl create secret generic ceph-secret --from-file=/tmp/secret --namespace=kube-system
+```
+
+* Start RBD provisioner
+
+The following example uses `rbd-provisioner-1` as the identity for the instance and assumes kubeconfig is at `/root/.kube`. The identity should remain the same if the provisioner restarts. If there are multiple provisioners, each should have a different identity.
+
+```bash
+docker run -ti -v /root/.kube:/kube -v /var/run/kubernetes:/var/run/kubernetes --privileged --net=host quay.io/external_storage/rbd-provisioner /usr/local/bin/rbd-provisioner -master=http://127.0.0.1:8080 -kubeconfig=/kube/config -id=rbd-provisioner-1
+```
+
+Alternatively, deploy it in kubernetes, see [deployment](deploy/README.md).
+
+* Create a RBD Storage Class
+
+Replace Ceph monitor's IP in [examples/class.yaml](examples/class.yaml) with your own and create storage class:
+
+```bash
+kubectl create -f examples/class.yaml
+```
+
+* Create a claim
+
+```bash
+kubectl create -f examples/claim.yaml
+```
+
+* Create a Pod using the claim
+
+```bash
+kubectl create -f examples/test-pod.yaml
+```
+
+## Acknowledgements
+
+* This provisioner is extracted from [Kubernetes core](https://github.com/kubernetes/kubernetes) with some modifications for this project.
--- a/docs/external_storage_provisioners/scheduler_plugins.md
+++ b/docs/external_storage_provisioners/scheduler_plugins.md
@@ -7,7 +7,7 @@ The kube-scheduler binary includes a list of plugins:
 - [CapacityScheduling](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/capacityscheduling) [Beta]
 - [CoScheduling](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/coscheduling) [Beta]
 - [NodeResources](https://github.com/kubernetes-sigs/scheduler-plugins/tree/master/pkg/noderesources) [Beta]
- [NodeResourceTopology](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/noderesourcetopology/README.md) [Beta]
+- [NodeResouceTopology](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/noderesourcetopology/README.md) [Beta]
 - [PreemptionToleration](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/preemptiontoleration/README.md) [Alpha]
 - [Trimaran](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/trimaran/README.md) [Alpha]
 - [NetworkAware](https://github.com/kubernetes-sigs/scheduler-plugins/blob/master/pkg/networkaware/README.md) [Sample]
--- a/docs/getting_started/setting-up-your-first-cluster.md
+++ b/docs/getting_started/setting-up-your-first-cluster.md
@@ -61,12 +61,12 @@ gcloud compute networks subnets create kubernetes \
 #### Firewall Rules

 Create a firewall rule that allows internal communication across all protocols.
-It is important to note that the vxlan (udp) protocol has to be allowed in order for
+It is important to note that the vxlan protocol has to be allowed in order for
 the calico (see later) networking plugin to work.

 ```ShellSession
 gcloud compute firewall-rules create kubernetes-the-kubespray-way-allow-internal \
-  --allow tcp,udp,icmp \
+  --allow tcp,udp,icmp,vxlan \
  --network kubernetes-the-kubespray-way \
  --source-ranges 10.240.0.0/24
 ```
@@ -88,7 +88,7 @@ cluster.

 ### Compute Instances

-The compute instances in this lab will be provisioned using [Ubuntu Server](https://www.ubuntu.com/server) 24.04.
+The compute instances in this lab will be provisioned using [Ubuntu Server](https://www.ubuntu.com/server) 18.04.
 Each compute instance will be provisioned with a fixed private IP address and
 a public IP address (that can be fixed - see [guide](https://cloud.google.com/compute/docs/ip-addresses/reserve-static-external-ip-address)).
 Using fixed public IP addresses has the advantage that our cluster node
@@ -103,7 +103,7 @@ for i in 0 1 2; do
    --async \
    --boot-disk-size 200GB \
    --can-ip-forward \
-    --image-family ubuntu-2404-lts-amd64 \
+    --image-family ubuntu-1804-lts \
    --image-project ubuntu-os-cloud \
    --machine-type e2-standard-2 \
    --private-network-ip 10.240.0.1${i} \
@@ -124,7 +124,7 @@ for i in 0 1 2; do
    --async \
    --boot-disk-size 200GB \
    --can-ip-forward \
-    --image-family ubuntu-2404-lts-amd64 \
+    --image-family ubuntu-1804-lts \
    --image-project ubuntu-os-cloud \
    --machine-type e2-standard-2 \
    --private-network-ip 10.240.0.2${i} \
--- a/docs/ingress/ingress_nginx.md
+++ b/docs/ingress/ingress_nginx.md
@@ -35,7 +35,7 @@ kubectl create clusterrolebinding cluster-admin-binding \
 The following **Mandatory Command** is required for all deployments except for AWS. See below for the AWS version.

 ```console
-kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.13.3/deploy/static/provider/cloud/deploy.yaml
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.0/deploy/static/provider/cloud/deploy.yaml
 ```

 ### Provider Specific Steps
--- a/docs/ingress/metallb.md
+++ b/docs/ingress/metallb.md
@@ -21,12 +21,6 @@ metallb_enabled: true
 metallb_speaker_enabled: true
 ```

-By default, MetalLB resources are deployed into the `metallb-system` namespace. You can override this namespace using a variable.
-
-```yaml
-metallb_namespace: woodenlb-system
-```
-
 By default only the MetalLB BGP speaker is allowed to run on control plane nodes. If you have a single node cluster or a cluster where control plane are also worker nodes you may need to enable tolerations for the MetalLB controller:

 ```yaml
@@ -41,7 +35,7 @@ metallb_config:
      effect: "NoSchedule"
 ```

-If you'd like to set additional nodeSelector and tolerations values, you can do so in the following fashion:
+If you'd like to set additional nodeSelector and tolerations values, you can do so in the following fasion:

 ```yaml
 metallb_config:
--- a/docs/operating_systems/bootstrap-os.md
+++ b/docs/operating_systems/bootstrap-os.md
@@ -1,4 +1,4 @@
-# bootstrap_os
+# bootstrap-os

 Bootstrap an Ansible host to be able to run Ansible modules.

@@ -48,8 +48,8 @@ Remember to disable fact gathering since Python might not be present on hosts.
 - hosts: all
  gather_facts: false  # not all hosts might be able to run modules yet
  roles:
-    - kubespray_defaults
-    - bootstrap_os
+    - kubespray-defaults
+    - bootstrap-os
 ```

 ## License
--- a/docs/operating_systems/rhel.md
+++ b/docs/operating_systems/rhel.md
@@ -37,12 +37,4 @@ If you have containers that are using iptables in the host network namespace (`h
 you need to ensure they are using iptables-nft.
 An example how k8s do the autodetection can be found [in this PR](https://github.com/kubernetes/kubernetes/pull/82966)

-The kernel version is lower than the kubernetes 1.32 system validation, please refer to the [kernel requirements](../operations/kernel-requirements.md).
-
-## Rocky Linux 10
-
-(Experimental in Kubespray CI)
-
-The official Rocky Linux 10 cloud image does not include `kernel-module-extra`. Both Kube Proxy and CNI rely on this package, and since it relates to kernel version compatibility (which may require VM reboots, etc.), we haven't found an ideal solution.
-
-However, some users report that it doesn't affect them (minimal version). Therefore, the Kubespray CI Rocky Linux 10 image is built by Kubespray maintainers using `diskimage-builder`. For detailed methods, please refer to [the comments](https://github.com/kubernetes-sigs/kubespray/pull/12355#issuecomment-3705400093).
+The kernel version is lower than the kubenretes 1.32 system validation, please refer to the [kernel requirements](../operations/kernel-requirements.md).
--- a/Show More
+++ b/Show More