Patch versions updates

2026-02-01 17:48:12 -03:30 · 2025-03-28 02:47:47 +00:00
170 changed files with 1714 additions and 664 deletions
--- a/.ansible-lint
+++ b/.ansible-lint
@@ -39,7 +39,5 @@ exclude_paths:
  - .github
  - .ansible
  - .cache
-  - .gitlab-ci.yml
-  - .gitlab-ci
 mock_modules:
  - gluster.gluster.gluster_volume
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -16,6 +16,5 @@ updates:
    directory: "/"
    labels:
      - release-note-none
-      - ci-short
    schedule:
      interval: "weekly"
--- a/.github/workflows/auto-label-os.yml
+++ b/.github/workflows/auto-label-os.yml
@@ -13,16 +13,16 @@ jobs:
      issues: write

    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      - uses: actions/checkout@v3

      - name: Parse issue form
-        uses: stefanbuck/github-issue-parser@2ea9b35a8c584529ed00891a8f7e41dc46d0441e
+        uses: stefanbuck/github-issue-parser@v3
        id: issue-parser
        with:
          template-path: .github/ISSUE_TEMPLATE/bug-report.yaml

      - name: Set labels based on OS field
-        uses: redhat-plumbers-in-action/advanced-issue-labeler@39087a4b30cb98d57f25f34d617a6af8163c17d9
+        uses: redhat-plumbers-in-action/advanced-issue-labeler@v2
        with:
          issue-form: ${{ steps.issue-parser.outputs.jsonString }}
          section: os
--- a/.github/workflows/upgrade-patch-versions-schedule.yml
+++ b/.github/workflows/upgrade-patch-versions-schedule.yml
@@ -12,7 +12,7 @@ jobs:
    outputs:
      branches: ${{ steps.get-branches.outputs.data }}
    steps:
-    - uses: octokit/graphql-action@8ad880e4d437783ea2ab17010324de1075228110
+    - uses: octokit/graphql-action@v2.3.2
      id: get-branches
      with:
        query: |
--- a/.github/workflows/upgrade-patch-versions.yml
+++ b/.github/workflows/upgrade-patch-versions.yml
@@ -11,7 +11,7 @@ jobs:
  update-patch-versions:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+    - uses: actions/checkout@v4
      with:
        ref: ${{ inputs.branch }}
    - uses: actions/setup-python@v5
@@ -29,12 +29,12 @@ jobs:
          ~/.cache/pre-commit
    - run: pre-commit run --all-files propagate-ansible-variables
      continue-on-error: true
-    - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e
+    - uses: peter-evans/create-pull-request@v7
      with:
        commit-message: Patch versions updates
        title: Patch versions updates - ${{ inputs.branch }}
        labels: bot
-        branch: component_hash_update/${{ inputs.branch }}
+        branch: ${{ inputs.branch }}-patch-updates
        sign-commits: true
        body: |
          /kind feature
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -31,12 +31,12 @@ variables:
  ANSIBLE_VERBOSITY: 2
  RECOVER_CONTROL_PLANE_TEST: "false"
  RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
-  TF_VERSION: 1.3.7
+  TERRAFORM_VERSION: 1.3.7
  PIPELINE_IMAGE: "$CI_REGISTRY_IMAGE/pipeline:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"

 before_script:
  - ./tests/scripts/rebase.sh
-  - mkdir -p cluster-dump $ANSIBLE_INVENTORY
+  - mkdir -p /.ssh

 .job: &job
  tags:
@@ -59,6 +59,18 @@ before_script:
    - pre-commit            # lint
    - vagrant-validate      # lint

+.testcases: &testcases
+  extends: .job-moderated
+  interruptible: true
+  before_script:
+    - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+    - ./tests/scripts/rebase.sh
+    - ./tests/scripts/testcases_prepare.sh
+  script:
+    - ./tests/scripts/testcases_run.sh
+  after_script:
+    - ./tests/scripts/testcases_cleanup.sh
+
 # For failfast, at least 1 job must be defined in .gitlab-ci.yml
 # Premoderated with manual actions
 ci-not-authorized:
@@ -90,6 +102,6 @@ include:
  - .gitlab-ci/build.yml
  - .gitlab-ci/lint.yml
  - .gitlab-ci/terraform.yml
-  - .gitlab-ci/kubevirt.yml
+  - .gitlab-ci/packet.yml
  - .gitlab-ci/vagrant.yml
  - .gitlab-ci/molecule.yml
--- a/.gitlab-ci/build.yml
+++ b/.gitlab-ci/build.yml
@@ -1,5 +1,5 @@
 ---
-pipeline-image:
+.build-container:
  cache:
    key: $CI_COMMIT_REF_SLUG
    paths:
@@ -11,19 +11,23 @@ pipeline-image:
    name: gcr.io/kaniko-project/executor:debug
    entrypoint: ['']
  variables:
+    TAG: $CI_COMMIT_SHORT_SHA
+    PROJECT_DIR: $CI_PROJECT_DIR
+    DOCKERFILE: Dockerfile
    GODEBUG: "http2client=0"
-  # TODO: remove the override
-  # currently rebase.sh depends on bash (not available in the kaniko image)
-  # once we have a simpler rebase (which should be easy if the target branch ref is available as variable
-  # we'll be able to rebase here as well hopefully
-  before_script: []
-  script:
+  before_script:
    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json
+  script:
    - /kaniko/executor --cache=true
                       --cache-dir=image-cache
-                       --context $CI_PROJECT_DIR
-                       --dockerfile $CI_PROJECT_DIR/pipeline.Dockerfile
+                       --context $PROJECT_DIR
+                       --dockerfile $PROJECT_DIR/$DOCKERFILE
                       --label 'git-branch'=$CI_COMMIT_REF_SLUG
                       --label 'git-tag=$CI_COMMIT_TAG'
                       --destination $PIPELINE_IMAGE
                       --log-timestamp=true
+
+pipeline-image:
+  extends: .build-container
+  variables:
+    DOCKERFILE: pipeline.Dockerfile
--- a/.gitlab-ci/kubevirt.yml
+++ b/.gitlab-ci/kubevirt.yml
@@ -1,148 +0,0 @@
---
-.kubevirt:
-  extends: .job-moderated
-  interruptible: true
-  script:
-    - ansible-playbook tests/cloud_playbooks/create-kubevirt.yml
-                      -c local -e @"tests/files/${TESTCASE}.yml"
-    - ./tests/scripts/testcases_run.sh
-  variables:
-    ANSIBLE_TIMEOUT: "120"
-  tags:
-    - ffci
-  needs:
-    - pipeline-image
-    - ci-not-authorized
-
-# TODO: generate testcases matrixes from the files in tests/files/
-# this is needed to avoid the need for PR rebasing when a job was added or remvoed in the target branch
-# (currently, a removed job in the target branch breaks the tests, because the
-# pipeline definition is parsed by gitlab before the rebase.sh script)
-# CI template for PRs
-pr:
-  stage: deploy-part1
-  rules:
-    - if: $PR_LABELS =~ /.*ci-short.*/
-      when: manual
-      allow_failure: true
-    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-      when: on_success
-    - when: manual
-      allow_failure: true
-  extends: .kubevirt
-  parallel:
-    matrix:
-      - TESTCASE:
-          - almalinux8-calico
-          - almalinux9-crio
-          - almalinux9-kube-ovn
-          - debian11-calico-collection
-          - debian11-macvlan
-          - debian12-cilium
-          - fedora39-kube-router
-            # FIXME: this test if broken (perma-failing)
-          - openeuler24-calico
-          - opensuse15-6-calico
-          - rockylinux8-calico
-          - rockylinux9-cilium
-          - ubuntu20-calico-all-in-one-hardening
-          - ubuntu20-cilium-sep
-          - ubuntu20-flannel-collection
-          - ubuntu20-kube-router-sep
-          - ubuntu20-kube-router-svc-proxy
-          - ubuntu22-calico-all-in-one
-          - ubuntu22-calico-all-in-one-upgrade
-          - ubuntu24-calico-etcd-datastore
-          - flatcar4081-calico
-
-# The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
-ubuntu20-calico-all-in-one:
-  stage: deploy-part1
-  extends: .kubevirt
-  variables:
-    TESTCASE: ubuntu20-calico-all-in-one
-  rules:
-    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
-      when: on_success
-    - when: manual
-      allow_failure: true
-
-pr_full:
-  extends: .kubevirt
-  stage: deploy-extended
-  rules:
-    - if: $PR_LABELS =~ /.*ci-full.*/
-      when: on_success
-    # Else run as manual
-    - when: manual
-      allow_failure: true
-  parallel:
-    matrix:
-      - TESTCASE:
-          - almalinux9-calico-ha-ebpf
-          - almalinux9-calico-nodelocaldns-secondary
-          - debian11-custom-cni
-          - debian11-kubelet-csr-approver
-          - debian12-custom-cni-helm
-          - fedora39-calico-swap-selinux
-          - fedora39-crio
-          - ubuntu20-all-in-one-docker
-          - ubuntu20-calico-ha-wireguard
-          - ubuntu20-flannel-ha
-          - ubuntu20-flannel-ha-once
-
-# Need an update of the container image to use schema v2
-# update: quay.io/kubespray/vm-amazon-linux-2:latest
-manual:
-  extends: pr_full
-  parallel:
-    matrix:
-      - TESTCASE:
-          - amazon-linux-2-all-in-one
-  rules:
-    - when: manual
-      allow_failure: true
-
-pr_extended:
-  extends: .kubevirt
-  stage: deploy-extended
-  rules:
-    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
-      when: on_success
-    - when: manual
-      allow_failure: true
-  parallel:
-    matrix:
-      - TESTCASE:
-          - almalinux9-calico
-          - almalinux9-calico-remove-node
-          - almalinux9-docker
-          - debian11-docker
-          - debian12-calico
-          - debian12-docker
-          - opensuse15-6-docker-cilium
-          - rockylinux9-calico
-          - ubuntu20-calico-etcd-kubeadm
-          - ubuntu20-flannel
-          - ubuntu22-all-in-one-docker
-          - ubuntu24-all-in-one-docker
-          - ubuntu24-calico-all-in-one
-
-# Enabled when PERIODIC_CI_ENABLED var is set
-periodic:
-  only:
-    variables:
-      - $PERIODIC_CI_ENABLED
-  allow_failure: true
-  extends: .kubevirt
-  parallel:
-    matrix:
-      - TESTCASE:
-          - debian11-calico-upgrade
-          - debian11-calico-upgrade-once
-          - debian12-cilium-svc-proxy
-          - fedora39-calico-selinux
-          - fedora40-docker-calico
-          - ubuntu20-calico-etcd-kubeadm-upgrade-ha
-          - ubuntu20-calico-ha-recover
-          - ubuntu20-calico-ha-recover-noquorum
--- a/.gitlab-ci/molecule.yml
+++ b/.gitlab-ci/molecule.yml
@@ -8,6 +8,8 @@
  needs:
  - pipeline-image
  # - ci-not-authorized
+  before_script:
+  - ./tests/scripts/rebase.sh
  script:
  - ./tests/scripts/molecule_run.sh
  after_script:
--- a/.gitlab-ci/packet.yml
+++ b/.gitlab-ci/packet.yml
@@ -0,0 +1,257 @@
+---
+.packet:
+  extends: .testcases
+  variables:
+    ANSIBLE_TIMEOUT: "120"
+    CI_PLATFORM: packet
+    SSH_USER: kubespray
+  tags:
+    - ffci
+  needs:
+    - pipeline-image
+    - ci-not-authorized
+
+# CI template for PRs
+.packet_pr:
+  stage: deploy-part1
+  rules:
+    - if: $PR_LABELS =~ /.*ci-short.*/
+      when: manual
+      allow_failure: true
+    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
+      when: on_success
+    - when: manual
+      allow_failure: true
+  extends: .packet
+
+  ## Uncomment this to have multiple stages
+  # needs:
+  #   - packet_ubuntu20-calico-all-in-one
+
+.packet_pr_short:
+  stage: deploy-part1
+  extends: .packet
+  rules:
+    - if: $CI_COMMIT_BRANCH =~ /^pr-.*$/
+      when: on_success
+    - when: manual
+      allow_failure: true
+
+.packet_pr_manual:
+  extends: .packet_pr
+  stage: deploy-extended
+  rules:
+    - if: $PR_LABELS =~ /.*ci-full.*/
+      when: on_success
+    # Else run as manual
+    - when: manual
+      allow_failure: true
+
+.packet_pr_extended:
+  extends: .packet_pr
+  stage: deploy-extended
+  rules:
+    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
+      when: on_success
+    - when: manual
+      allow_failure: true
+
+# CI template for periodic CI jobs
+# Enabled when PERIODIC_CI_ENABLED var is set
+.packet_periodic:
+  only:
+    variables:
+      - $PERIODIC_CI_ENABLED
+  allow_failure: true
+  extends: .packet
+
+# The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
+packet_ubuntu20-calico-all-in-one:
+  stage: deploy-part1
+  extends: .packet_pr_short
+  variables:
+    RESET_CHECK: "true"
+
+# ### PR JOBS PART2
+
+packet_ubuntu20-crio:
+  extends: .packet_pr_manual
+
+packet_ubuntu22-calico-all-in-one:
+  extends: .packet_pr
+
+packet_ubuntu22-calico-all-in-one-upgrade:
+  extends: .packet_pr
+  variables:
+    UPGRADE_TEST: graceful
+
+packet_ubuntu24-calico-etcd-datastore:
+  extends: .packet_pr
+
+packet_almalinux9-crio:
+  extends: .packet_pr
+
+packet_almalinux9-kube-ovn:
+  extends: .packet_pr
+
+packet_debian11-calico-collection:
+  extends: .packet_pr
+
+packet_debian11-macvlan:
+  extends: .packet_pr
+
+packet_debian12-cilium:
+  extends: .packet_pr
+
+packet_almalinux8-calico:
+  extends: .packet_pr
+
+packet_rockylinux8-calico:
+  extends: .packet_pr
+
+packet_rockylinux9-cilium:
+  extends: .packet_pr
+  variables:
+    RESET_CHECK: "true"
+
+# Need an update of the container image to use schema v2
+# update: quay.io/kubespray/vm-amazon-linux-2:latest
+packet_amazon-linux-2-all-in-one:
+  extends: .packet_pr_manual
+  rules:
+    - when: manual
+      allow_failure: true
+
+packet_opensuse15-6-calico:
+  extends: .packet_pr
+
+packet_ubuntu20-cilium-sep:
+  extends: .packet_pr
+
+packet_openeuler24-calico:
+  extends: .packet_pr
+
+packet_ubuntu20-calico-all-in-one-hardening:
+  extends: .packet_pr
+
+## Extended
+packet_debian11-docker:
+  extends: .packet_pr_extended
+
+packet_debian12-docker:
+  extends: .packet_pr_extended
+
+packet_debian12-calico:
+  extends: .packet_pr_extended
+
+packet_almalinux9-calico-remove-node:
+  extends: .packet_pr_extended
+  variables:
+    REMOVE_NODE_CHECK: "true"
+    REMOVE_NODE_NAME: "instance-3"
+
+packet_rockylinux9-calico:
+  extends: .packet_pr_extended
+
+packet_almalinux9-calico:
+  extends: .packet_pr_extended
+
+packet_almalinux9-docker:
+  extends: .packet_pr_extended
+
+packet_opensuse15-6-docker-cilium:
+  extends: .packet_pr_extended
+
+packet_ubuntu24-calico-all-in-one:
+  extends: .packet_pr_extended
+
+packet_ubuntu20-calico-etcd-kubeadm:
+  extends: .packet_pr_extended
+
+packet_ubuntu24-all-in-one-docker:
+  extends: .packet_pr_extended
+
+packet_ubuntu22-all-in-one-docker:
+  extends: .packet_pr_extended
+
+# ### MANUAL JOBS
+packet_fedora39-crio:
+  extends: .packet_pr_manual
+
+packet_ubuntu20-flannel-ha:
+  extends: .packet_pr_manual
+
+packet_ubuntu20-all-in-one-docker:
+  extends: .packet_pr_manual
+
+packet_ubuntu20-flannel-ha-once:
+  extends: .packet_pr_manual
+
+packet_fedora39-calico-swap-selinux:
+  extends: .packet_pr_manual
+
+packet_almalinux9-calico-ha-ebpf:
+  extends: .packet_pr_manual
+
+packet_almalinux9-calico-nodelocaldns-secondary:
+  extends: .packet_pr_manual
+
+packet_debian11-custom-cni:
+  extends: .packet_pr_manual
+
+packet_debian11-kubelet-csr-approver:
+  extends: .packet_pr_manual
+
+packet_debian12-custom-cni-helm:
+  extends: .packet_pr_manual
+
+packet_ubuntu20-calico-ha-wireguard:
+  extends: .packet_pr_manual
+
+# PERIODIC
+packet_fedora40-docker-calico:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    RESET_CHECK: "true"
+
+packet_fedora39-calico-selinux:
+  stage: deploy-extended
+  extends: .packet_periodic
+
+packet_ubuntu20-calico-etcd-kubeadm-upgrade-ha:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    UPGRADE_TEST: basic
+
+
+packet_debian11-calico-upgrade-once:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    UPGRADE_TEST: graceful
+
+packet_ubuntu20-calico-ha-recover:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    RECOVER_CONTROL_PLANE_TEST: "true"
+    RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
+
+packet_ubuntu20-calico-ha-recover-noquorum:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    RECOVER_CONTROL_PLANE_TEST: "true"
+    RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[1:]:kube_control_plane[1:]"
+
+packet_debian11-calico-upgrade:
+  stage: deploy-extended
+  extends: .packet_periodic
+  variables:
+    UPGRADE_TEST: graceful
+
+packet_debian12-cilium-svc-proxy:
+  stage: deploy-extended
+  extends: .packet_periodic
--- a/.gitlab-ci/terraform.yml
+++ b/.gitlab-ci/terraform.yml
@@ -5,21 +5,28 @@
  needs:
    - ci-not-authorized
    - pipeline-image
-  variables:
-    TF_VAR_public_key_path: "${ANSIBLE_PRIVATE_KEY_FILE}.pub"
-    TF_VAR_ssh_private_key_path: $ANSIBLE_PRIVATE_KEY_FILE
-    CLUSTER: $CI_COMMIT_REF_NAME
-    TERRAFORM_STATE_ROOT: $CI_PROJECT_DIR
  stage: deploy-part1
  before_script:
+    - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
    - ./tests/scripts/rebase.sh
-    - mkdir -p cluster-dump $ANSIBLE_INVENTORY
+    - ./tests/scripts/testcases_prepare.sh
    - ./tests/scripts/terraform_install.sh
+    # Set Ansible config
+    - cp ansible.cfg ~/.ansible.cfg
+    # Prepare inventory
    - cp contrib/terraform/$PROVIDER/sample-inventory/cluster.tfvars .
-    - ln -rs -t $ANSIBLE_INVENTORY contrib/terraform/$PROVIDER/hosts
+    - ln -s contrib/terraform/$PROVIDER/hosts
    - terraform -chdir="contrib/terraform/$PROVIDER" init
+    # Copy SSH keypair
+    - mkdir -p ~/.ssh
+    - echo "$PACKET_PRIVATE_KEY" | base64 -d > ~/.ssh/id_rsa
+    - chmod 400 ~/.ssh/id_rsa
+    - echo "$PACKET_PUBLIC_KEY" | base64 -d > ~/.ssh/id_rsa.pub
+    - mkdir -p contrib/terraform/$PROVIDER/group_vars
+    # Random subnet to avoid routing conflicts
+    - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"

-terraform_validate:
+.terraform_validate:
  extends: .terraform_install
  tags: [ffci]
  only: ['master', /^pr-.*$/]
@@ -29,17 +36,6 @@ terraform_validate:
  stage: test
  needs:
    - pipeline-image
-  parallel:
-    matrix:
-      - PROVIDER:
-          - openstack
-          - equinix
-          - aws
-          - exoscale
-          - hetzner
-          - vsphere
-          - upcloud
-          - nifcloud

 .terraform_apply:
  extends: .terraform_install
@@ -47,24 +43,99 @@ terraform_validate:
  stage: deploy-extended
  when: manual
  only: [/^pr-.*$/]
+  artifacts:
+    when: always
+    paths:
+      - cluster-dump/
  variables:
    ANSIBLE_INVENTORY_UNPARSED_FAILED: "true"
-    ANSIBLE_REMOTE_USER: ubuntu # the openstack terraform module does not handle custom user correctly
-    ANSIBLE_SSH_RETRIES: 15
-    TF_VAR_ssh_user: $ANSIBLE_REMOTE_USER
+    ANSIBLE_INVENTORY: hosts
+    CI_PLATFORM: tf
+    TF_VAR_ssh_user: $SSH_USER
    TF_VAR_cluster_name: $CI_JOB_ID
  script:
-    # Set Ansible config
-    - cp ansible.cfg ~/.ansible.cfg
-    - ssh-keygen -N '' -f $ANSIBLE_PRIVATE_KEY_FILE -t rsa
-    - mkdir -p contrib/terraform/$PROVIDER/group_vars
-    # Random subnet to avoid routing conflicts
-    - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
-    - terraform -chdir="contrib/terraform/$PROVIDER" apply -auto-approve -parallelism=1
    - tests/scripts/testcases_run.sh
  after_script:
    # Cleanup regardless of exit code
-    - terraform -chdir="contrib/terraform/$PROVIDER" destroy -auto-approve
+    - ./tests/scripts/testcases_cleanup.sh
+
+tf-validate-openstack:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: openstack
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-equinix:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: equinix
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-aws:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: aws
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-exoscale:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: exoscale
+
+tf-validate-hetzner:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: hetzner
+
+tf-validate-vsphere:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: vsphere
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-upcloud:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: upcloud
+    CLUSTER: $CI_COMMIT_REF_NAME
+
+tf-validate-nifcloud:
+  extends: .terraform_validate
+  variables:
+    TF_VERSION: $TERRAFORM_VERSION
+    PROVIDER: nifcloud
+
+# tf-packet-ubuntu20-default:
+#   extends: .terraform_apply
+#   variables:
+#     TF_VERSION: $TERRAFORM_VERSION
+#     PROVIDER: packet
+#     CLUSTER: $CI_COMMIT_REF_NAME
+#     TF_VAR_number_of_k8s_masters: "1"
+#     TF_VAR_number_of_k8s_nodes: "1"
+#     TF_VAR_plan_k8s_masters: t1.small.x86
+#     TF_VAR_plan_k8s_nodes: t1.small.x86
+#     TF_VAR_metro: am
+#     TF_VAR_public_key_path: ""
+#     TF_VAR_operating_system: ubuntu_20_04
+
+.ovh_variables: &ovh_variables
+  OS_AUTH_URL: https://auth.cloud.ovh.net/v3
+  OS_PROJECT_ID: 8d3cd5d737d74227ace462dee0b903fe
+  OS_PROJECT_NAME: "9361447987648822"
+  OS_USER_DOMAIN_NAME: Default
+  OS_PROJECT_DOMAIN_ID: default
+  OS_USERNAME: 8XuhBMfkKVrk
+  OS_REGION_NAME: UK1
+  OS_INTERFACE: public
+  OS_IDENTITY_API_VERSION: "3"

 # Elastx is generously donating resources for Kubespray on Openstack CI
 # Contacts: @gix @bl0m1
@@ -98,8 +169,11 @@ tf-elastx_ubuntu20-calico:
  allow_failure: true
  variables:
    <<: *elastx_variables
+    TF_VERSION: $TERRAFORM_VERSION
    PROVIDER: openstack
+    CLUSTER: $CI_COMMIT_REF_NAME
    ANSIBLE_TIMEOUT: "60"
+    SSH_USER: ubuntu
    TF_VAR_number_of_k8s_masters: "1"
    TF_VAR_number_of_k8s_masters_no_floating_ip: "0"
    TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
@@ -120,3 +194,46 @@ tf-elastx_ubuntu20-calico:
    TF_VAR_flavor_k8s_node: 3f73fc93-ec61-4808-88df-2580d94c1a9b      # v1-standard-2
    TF_VAR_image: ubuntu-20.04-server-latest
    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
+
+# OVH voucher expired, commenting job until things are sorted  out
+
+# tf-ovh_cleanup:
+#  stage: unit-tests
+#  tags: [light]
+#  image: python
+#  environment: ovh
+#  variables:
+#    <<: *ovh_variables
+#  before_script:
+#    - pip install -r scripts/openstack-cleanup/requirements.txt
+#  script:
+#    - ./scripts/openstack-cleanup/main.py
+
+# tf-ovh_ubuntu20-calico:
+#  extends: .terraform_apply
+#  when: on_success
+#  environment: ovh
+#  variables:
+#    <<: *ovh_variables
+#    TF_VERSION: $TERRAFORM_VERSION
+#    PROVIDER: openstack
+#    CLUSTER: $CI_COMMIT_REF_NAME
+#    ANSIBLE_TIMEOUT: "60"
+#    SSH_USER: ubuntu
+#    TF_VAR_number_of_k8s_masters: "0"
+#    TF_VAR_number_of_k8s_masters_no_floating_ip: "1"
+#    TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
+#    TF_VAR_number_of_etcd: "0"
+#    TF_VAR_number_of_k8s_nodes: "0"
+#    TF_VAR_number_of_k8s_nodes_no_floating_ip: "1"
+#    TF_VAR_number_of_gfs_nodes_no_floating_ip: "0"
+#    TF_VAR_number_of_bastions: "0"
+#    TF_VAR_number_of_k8s_masters_no_etcd: "0"
+#    TF_VAR_use_neutron: "0"
+#    TF_VAR_floatingip_pool: "Ext-Net"
+#    TF_VAR_external_net: "6011fbc9-4cbf-46a4-8452-6890a340b60b"
+#    TF_VAR_network_name: "Ext-Net"
+#    TF_VAR_flavor_k8s_master: "defa64c3-bd46-43b4-858a-d93bbae0a229"    # s1-8
+#    TF_VAR_flavor_k8s_node: "defa64c3-bd46-43b4-858a-d93bbae0a229"      # s1-8
+#    TF_VAR_image: "Ubuntu 20.04"
+#    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
--- a/.gitlab-ci/vagrant.yml
+++ b/.gitlab-ci/vagrant.yml
@@ -1,13 +1,13 @@
 ---
-vagrant:
-  extends: .job-moderated
+.vagrant:
+  extends: .testcases
  needs:
    - ci-not-authorized
  variables:
    CI_PLATFORM: "vagrant"
    SSH_USER: "vagrant"
    VAGRANT_DEFAULT_PROVIDER: "libvirt"
-    KUBESPRAY_VAGRANT_CONFIG: tests/files/${TESTCASE}.rb
+    KUBESPRAY_VAGRANT_CONFIG: tests/files/${CI_JOB_NAME}.rb
    DOCKER_NAME: vagrant
    VAGRANT_ANSIBLE_TAGS: facts
    VAGRANT_HOME: "$CI_PROJECT_DIR/.vagrant.d"
@@ -28,22 +28,54 @@ vagrant:
    - pip install --no-compile --no-cache-dir -r $CI_PROJECT_DIR/tests/requirements.txt
    - ./tests/scripts/vagrant_clean.sh
  script:
-    - vagrant up
    - ./tests/scripts/testcases_run.sh
-  after_script:
-    - vagrant destroy -f
  cache:
    key: $CI_JOB_NAME_SLUG
    paths:
      - .vagrant.d/boxes
      - .cache/pip
    policy: pull-push # TODO: change to "pull" when not on main
+
+vagrant_ubuntu24-calico-dual-stack:
  stage: deploy-extended
+  extends: .vagrant
  rules:
    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
      when: on_success
-  parallel:
-    matrix:
-      - TESTCASE:
-          - ubuntu24-calico-dual-stack
-          - ubuntu24-calico-ipv6only-stack
+  allow_failure: false
+
+vagrant_ubuntu24-calico-ipv6only-stack:
+  stage: deploy-extended
+  extends: .vagrant
+  rules:
+    - if: $PR_LABELS =~ /.*(ci-extended|ci-full).*/
+      when: on_success
+  allow_failure: false
+
+vagrant_ubuntu20-flannel:
+  stage: deploy-part1
+  extends: .vagrant
+  when: on_success
+  allow_failure: false
+
+vagrant_ubuntu20-flannel-collection:
+  stage: deploy-extended
+  extends: .vagrant
+  when: manual
+
+vagrant_ubuntu20-kube-router-sep:
+  stage: deploy-extended
+  extends: .vagrant
+  when: manual
+
+# Service proxy test fails connectivity testing
+vagrant_ubuntu20-kube-router-svc-proxy:
+  stage: deploy-extended
+  extends: .vagrant
+  when: manual
+
+vagrant_fedora39-kube-router:
+  stage: deploy-extended
+  extends: .vagrant
+  when: manual
+# FIXME: this test if broken (perma-failing)
--- a/README.md
+++ b/README.md
@@ -114,11 +114,11 @@ Note:
  - [kubernetes](https://github.com/kubernetes/kubernetes) 1.32.3
  - [etcd](https://github.com/etcd-io/etcd) 3.5.16
  - [docker](https://www.docker.com/) 28.0
-  - [containerd](https://containerd.io/) 2.0.3
+  - [containerd](https://containerd.io/) 2.0.4
  - [cri-o](http://cri-o.io/) 1.32.0 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
  - [cni-plugins](https://github.com/containernetworking/plugins) 1.4.1
-  - [calico](https://github.com/projectcalico/calico) 3.29.2
+  - [calico](https://github.com/projectcalico/calico) 3.29.3
  - [cilium](https://github.com/cilium/cilium) 1.15.9
  - [flannel](https://github.com/flannel-io/flannel) 0.22.0
  - [kube-ovn](https://github.com/alauda/kube-ovn) 1.12.21
@@ -135,6 +135,8 @@ Note:
  - [metallb](https://metallb.universe.tf/) 0.13.9
  - [registry](https://github.com/distribution/distribution) 2.8.1
 - Storage Plugin
+  - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) 2.1.0-k8s1.11
+  - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) 2.1.1-k8s1.11
  - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) 0.5.0
  - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) 1.10.0
  - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) 1.30.0
--- a/contrib/kvm-setup/README.md
+++ b/contrib/kvm-setup/README.md
@@ -0,0 +1,11 @@
+# Kubespray on KVM Virtual Machines hypervisor preparation
+
+A simple playbook to ensure your system has the right settings to enable Kubespray
+deployment on VMs.
+
+This playbook does not create Virtual Machines, nor does it run Kubespray itself.
+
+## User creation
+
+If you want to create a user for running Kubespray deployment, you should specify
+both `k8s_deployment_user` and `k8s_deployment_user_pkey_path`.
--- a/contrib/kvm-setup/group_vars/all
+++ b/contrib/kvm-setup/group_vars/all
@@ -0,0 +1,2 @@
+#k8s_deployment_user: kubespray
+#k8s_deployment_user_pkey_path: /tmp/ssh_rsa
--- a/contrib/kvm-setup/kvm-setup.yml
+++ b/contrib/kvm-setup/kvm-setup.yml
@@ -0,0 +1,9 @@
+---
+- name: Prepare Hypervisor to later install kubespray VMs
+  hosts: localhost
+  gather_facts: false
+  become: true
+  vars:
+    bootstrap_os: none
+  roles:
+    - { role: kvm-setup }
--- a/contrib/kvm-setup/roles/kvm-setup/tasks/main.yml
+++ b/contrib/kvm-setup/roles/kvm-setup/tasks/main.yml
@@ -0,0 +1,30 @@
+---
+
+- name: Install required packages
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - bind-utils
+    - ntp
+  when: ansible_os_family == "RedHat"
+
+- name: Install required packages
+  apt:
+    upgrade: true
+    update_cache: true
+    cache_valid_time: 3600
+    name: "{{ item }}"
+    state: present
+    install_recommends: false
+  with_items:
+    - dnsutils
+    - ntp
+  when: ansible_os_family == "Debian"
+
+- name: Create deployment user if required
+  include_tasks: user.yml
+  when: k8s_deployment_user is defined
+
+- name: Set proper sysctl values
+  import_tasks: sysctl.yml
--- a/contrib/kvm-setup/roles/kvm-setup/tasks/sysctl.yml
+++ b/contrib/kvm-setup/roles/kvm-setup/tasks/sysctl.yml
@@ -0,0 +1,46 @@
+---
+- name: Load br_netfilter module
+  community.general.modprobe:
+    name: br_netfilter
+    state: present
+  register: br_netfilter
+
+- name: Add br_netfilter into /etc/modules
+  lineinfile:
+    dest: /etc/modules
+    state: present
+    line: 'br_netfilter'
+  when: br_netfilter is defined and ansible_os_family == 'Debian'
+
+- name: Add br_netfilter into /etc/modules-load.d/kubespray.conf
+  copy:
+    dest: /etc/modules-load.d/kubespray.conf
+    content: |-
+      ### This file is managed by Ansible
+      br-netfilter
+    owner: root
+    group: root
+    mode: "0644"
+  when: br_netfilter is defined
+
+
+- name: Enable net.ipv4.ip_forward in sysctl
+  ansible.posix.sysctl:
+    name: net.ipv4.ip_forward
+    value: 1
+    sysctl_file: "{{ sysctl_file_path }}"
+    state: present
+    reload: true
+
+- name: Set bridge-nf-call-{arptables,iptables} to 0
+  ansible.posix.sysctl:
+    name: "{{ item }}"
+    state: present
+    value: 0
+    sysctl_file: "{{ sysctl_file_path }}"
+    reload: true
+  with_items:
+    - net.bridge.bridge-nf-call-arptables
+    - net.bridge.bridge-nf-call-ip6tables
+    - net.bridge.bridge-nf-call-iptables
+  when: br_netfilter is defined
--- a/contrib/kvm-setup/roles/kvm-setup/tasks/user.yml
+++ b/contrib/kvm-setup/roles/kvm-setup/tasks/user.yml
@@ -0,0 +1,47 @@
+---
+- name: Create user {{ k8s_deployment_user }}
+  user:
+    name: "{{ k8s_deployment_user }}"
+    groups: adm
+    shell: /bin/bash
+
+- name: Ensure that .ssh exists
+  file:
+    path: "/home/{{ k8s_deployment_user }}/.ssh"
+    state: directory
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+    mode: "0700"
+
+- name: Configure sudo for deployment user
+  copy:
+    content: |
+      %{{ k8s_deployment_user }} ALL=(ALL) NOPASSWD: ALL
+    dest: "/etc/sudoers.d/55-k8s-deployment"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Write private SSH key
+  copy:
+    src: "{{ k8s_deployment_user_pkey_path }}"
+    dest: "/home/{{ k8s_deployment_user }}/.ssh/id_rsa"
+    mode: "0400"
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+  when: k8s_deployment_user_pkey_path is defined
+
+- name: Write public SSH key
+  shell: "ssh-keygen -y -f /home/{{ k8s_deployment_user }}/.ssh/id_rsa \
+            > /home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+  args:
+    creates: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+  when: k8s_deployment_user_pkey_path is defined
+
+- name: Fix ssh-pub-key permissions
+  file:
+    path: "/home/{{ k8s_deployment_user }}/.ssh/authorized_keys"
+    mode: "0600"
+    owner: "{{ k8s_deployment_user }}"
+    group: "{{ k8s_deployment_user }}"
+  when: k8s_deployment_user_pkey_path is defined
--- a/contrib/misc/clusteradmin-rbac.yml
+++ b/contrib/misc/clusteradmin-rbac.yml
@@ -0,0 +1,15 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubernetes-dashboard
+  labels:
+    k8s-app: kubernetes-dashboard
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- kind: ServiceAccount
+  name: kubernetes-dashboard
+  namespace: kube-system
--- a/contrib/mitogen/mitogen.yml
+++ b/contrib/mitogen/mitogen.yml
@@ -0,0 +1,51 @@
+---
+- name: Check ansible version
+  import_playbook: kubernetes_sigs.kubespray.ansible_version
+
+- name: Install mitogen
+  hosts: localhost
+  strategy: linear
+  vars:
+    mitogen_version: 0.3.2
+    mitogen_url: https://github.com/mitogen-hq/mitogen/archive/refs/tags/v{{ mitogen_version }}.tar.gz
+    ansible_connection: local
+  tasks:
+    - name: Create mitogen plugin dir
+      file:
+        path: "{{ item }}"
+        state: directory
+        mode: "0755"
+      become: false
+      loop:
+        - "{{ playbook_dir }}/plugins/mitogen"
+        - "{{ playbook_dir }}/dist"
+
+    - name: Download mitogen release
+      get_url:
+        url: "{{ mitogen_url }}"
+        dest: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.tar.gz"
+        validate_certs: true
+        mode: "0644"
+
+    - name: Extract archive
+      unarchive:
+        src: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.tar.gz"
+        dest: "{{ playbook_dir }}/dist/"
+
+    - name: Copy plugin
+      ansible.posix.synchronize:
+        src: "{{ playbook_dir }}/dist/mitogen-{{ mitogen_version }}/"
+        dest: "{{ playbook_dir }}/plugins/mitogen"
+
+    - name: Add strategy to ansible.cfg
+      community.general.ini_file:
+        path: ansible.cfg
+        mode: "0644"
+        section: "{{ item.section | d('defaults') }}"
+        option: "{{ item.option }}"
+        value: "{{ item.value }}"
+      with_items:
+        - option: strategy
+          value: mitogen_linear
+        - option: strategy_plugins
+          value: plugins/mitogen/ansible_mitogen/plugins/strategy
--- a/contrib/terraform/hetzner/README.md
+++ b/contrib/terraform/hetzner/README.md
@@ -102,8 +102,7 @@ Please read the instructions in both repos on how to install it.
 You can teardown your infrastructure using the following Terraform command:

 ```bash
-cd ./kubespray
-terraform -chdir=./contrib/terraform/hetzner/ destroy --var-file=../../../inventory/$CLUSTER/default.tfvars
+terraform destroy --var-file default.tfvars ../../contrib/terraform/hetzner
 ```

 ## Variables
--- a/contrib/terraform/upcloud/README.md
+++ b/contrib/terraform/upcloud/README.md
@@ -2,6 +2,35 @@

 Provision a Kubernetes cluster on [UpCloud](https://upcloud.com/) using Terraform and Kubespray

+## Overview
+
+The setup looks like following
+
+```text
+   Kubernetes cluster
+--------------------------+
+|      +--------------+    |
+|      | +--------------+  |
+| -->  | |              |  |
+|      | | Master/etcd  |  |
+|      | | node(s)      |  |
+|      +-+              |  |
+|        +--------------+  |
+|              ^           |
+|              |           |
+|              v           |
+|      +--------------+    |
+|      | +--------------+  |
+| -->  | |              |  |
+|      | |    Worker    |  |
+|      | |    node(s)   |  |
+|      +-+              |  |
+|        +--------------+  |
+--------------------------+
+```
+
+The nodes uses a private network for node to node communication and a public interface for all external communication.
+
 ## Requirements

 * Terraform 0.13.0 or newer
@@ -71,8 +100,6 @@ terraform destroy --var-file cluster-settings.tfvars \
 * `template_name`: The name or UUID  of a base image
 * `username`: a user to access the nodes, defaults to "ubuntu"
 * `private_network_cidr`: CIDR to use for the private network, defaults to "172.16.0.0/24"
-* `dns_servers`: DNS servers that will be used by the nodes. Until [this is solved](https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562) this is done using user_data to reconfigure resolved. Defaults to `[]`
-* `use_public_ips`: If a NIC connencted to the Public network should be attached to all nodes by default. Can be overridden by `force_public_ip` if this is set to `false`. Defaults to `true`
 * `ssh_public_keys`: List of public SSH keys to install on all machines
 * `zone`: The zone where to run the cluster
 * `machines`: Machines to provision. Key of this object will be used as the name of the machine
@@ -81,8 +108,6 @@ terraform destroy --var-file cluster-settings.tfvars \
  * `cpu`: number of cpu cores
  * `mem`: memory size in MB
  * `disk_size`: The size of the storage in GB
-  * `force_public_ip`: If `use_public_ips` is set to `false`, this forces a public NIC onto the machine anyway when set to `true`. Useful if you're migrating from public nodes to only private. Defaults to `false`
-  * `dns_servers`: This works the same way as the global `dns_severs` but only applies to a single node. If set to `[]` while the global `dns_servers` is set to something else, then it will not add the user_data and thus will not be recreated. Useful if you're migrating from public nodes to only private. Defaults to `null`
  * `additional_disks`: Additional disks to attach to the node.
    * `size`: The size of the additional disk in GB
    * `tier`: The tier of disk to use (`maxiops` is the only one you can choose atm)
@@ -114,7 +139,6 @@ terraform destroy --var-file cluster-settings.tfvars \
  * `port`: Port to load balance.
  * `target_port`: Port to the backend servers.
  * `backend_servers`: List of servers that traffic to the port should be forwarded to.
-  * `proxy_protocol`: If the loadbalancer should set up the backend using proxy protocol.
 * `router_enable`: If a router should be connected to the private network or not
 * `gateways`: Gateways that should be connected to the router, requires router_enable is set to true
  * `features`: List of features for the gateway
@@ -147,27 +171,3 @@ terraform destroy --var-file cluster-settings.tfvars \
 * `server_groups`: Group servers together
  * `servers`: The servers that should be included in the group.
  * `anti_affinity_policy`: Defines if a server group is an anti-affinity group. Setting this to "strict" or yes" will result in all servers in the group being placed on separate compute hosts. The value can be "strict", "yes" or "no". "strict" refers to strict policy doesn't allow servers in the same server group to be on the same host. "yes" refers to best-effort policy and tries to put servers on different hosts, but this is not guaranteed.
-
-## Migration
-
-When `null_resource.inventories` and `data.template_file.inventory` was changed to `local_file.inventory` the old state file needs to be cleaned of the old state.
-The error messages you'll see if you encounter this is:
-
-```text
-Error: failed to read schema for null_resource.inventories in registry.terraform.io/hashicorp/null: failed to instantiate provider "registry.terraform.io/hashicorp/null" to obtain schema: unavailable provider "registry.terraform.io/hashicorp/null"
-Error: failed to read schema for data.template_file.inventory in registry.terraform.io/hashicorp/template: failed to instantiate provider "registry.terraform.io/hashicorp/template" to obtain schema: unavailable provider "registry.terraform.io/hashicorp/template"
-```
-
-This can be fixed with the following lines
-
-```bash
-terraform state rm -state=terraform.tfstate null_resource.inventories
-terraform state rm -state=terraform.tfstate data.template_file.inventory
-```
-
-### Public to Private only migration
-
-Since there's no way to remove the public NIC on a machine without recreating its private NIC it's not possible to inplace change a cluster to only use private IPs.
-The way to migrate is to first set `use_public_ips` to `false`, `dns_servers` to some DNS servers and then update all existing servers to have `force_public_ip` set to `true` and `dns_severs` set to `[]`.
-After that you can add new nodes without `force_public_ip` and `dns_servers` set and create them.
-Add the new nodes into the cluster and when all of them are added, remove the old nodes.
--- a/contrib/terraform/upcloud/cluster-settings.tfvars
+++ b/contrib/terraform/upcloud/cluster-settings.tfvars
@@ -122,11 +122,11 @@ k8s_allowed_remote_ips = [
 master_allowed_ports = []
 worker_allowed_ports = []

-loadbalancer_enabled = false
-loadbalancer_plan    = "development"
+loadbalancer_enabled        = false
+loadbalancer_plan           = "development"
+loadbalancer_proxy_protocol = false
 loadbalancers = {
  # "http" : {
-  #   "proxy_protocol" : false
  #   "port" : 80,
  #   "target_port" : 80,
  #   "backend_servers" : [
--- a/contrib/terraform/upcloud/main.tf
+++ b/contrib/terraform/upcloud/main.tf
@@ -20,26 +20,24 @@ module "kubernetes" {
  username      = var.username

  private_network_cidr = var.private_network_cidr
-  dns_servers          = var.dns_servers
-  use_public_ips       = var.use_public_ips

  machines = var.machines

  ssh_public_keys = var.ssh_public_keys

-  firewall_enabled           = var.firewall_enabled
-  firewall_default_deny_in   = var.firewall_default_deny_in
-  firewall_default_deny_out  = var.firewall_default_deny_out
-  master_allowed_remote_ips  = var.master_allowed_remote_ips
-  k8s_allowed_remote_ips     = var.k8s_allowed_remote_ips
-  bastion_allowed_remote_ips = var.bastion_allowed_remote_ips
-  master_allowed_ports       = var.master_allowed_ports
-  worker_allowed_ports       = var.worker_allowed_ports
+  firewall_enabled          = var.firewall_enabled
+  firewall_default_deny_in  = var.firewall_default_deny_in
+  firewall_default_deny_out = var.firewall_default_deny_out
+  master_allowed_remote_ips = var.master_allowed_remote_ips
+  k8s_allowed_remote_ips    = var.k8s_allowed_remote_ips
+  master_allowed_ports      = var.master_allowed_ports
+  worker_allowed_ports      = var.worker_allowed_ports

-  loadbalancer_enabled        = var.loadbalancer_enabled
-  loadbalancer_plan           = var.loadbalancer_plan
-  loadbalancer_legacy_network = var.loadbalancer_legacy_network
-  loadbalancers               = var.loadbalancers
+  loadbalancer_enabled                 = var.loadbalancer_enabled
+  loadbalancer_plan                    = var.loadbalancer_plan
+  loadbalancer_outbound_proxy_protocol = var.loadbalancer_proxy_protocol ? "v2" : ""
+  loadbalancer_legacy_network          = var.loadbalancer_legacy_network
+  loadbalancers                        = var.loadbalancers

  router_enable    = var.router_enable
  gateways         = var.gateways
@@ -54,12 +52,32 @@ module "kubernetes" {
 # Generate ansible inventory
 #

-resource "local_file" "inventory" {
-  content = templatefile("${path.module}/templates/inventory.tpl", {
-    master_ip  = module.kubernetes.master_ip
-    worker_ip  = module.kubernetes.worker_ip
-    bastion_ip = module.kubernetes.bastion_ip
-    username   = var.username
-  })
-  filename = var.inventory_file
+data "template_file" "inventory" {
+  template = file("${path.module}/templates/inventory.tpl")
+
+  vars = {
+    connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d",
+      keys(module.kubernetes.master_ip),
+      values(module.kubernetes.master_ip).*.public_ip,
+      values(module.kubernetes.master_ip).*.private_ip,
+    range(1, length(module.kubernetes.master_ip) + 1)))
+    connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s",
+      keys(module.kubernetes.worker_ip),
+      values(module.kubernetes.worker_ip).*.public_ip,
+    values(module.kubernetes.worker_ip).*.private_ip))
+    list_master = join("\n", formatlist("%s",
+    keys(module.kubernetes.master_ip)))
+    list_worker = join("\n", formatlist("%s",
+    keys(module.kubernetes.worker_ip)))
+  }
+}
+
+resource "null_resource" "inventories" {
+  provisioner "local-exec" {
+    command = "echo '${data.template_file.inventory.rendered}' > ${var.inventory_file}"
+  }
+
+  triggers = {
+    template = data.template_file.inventory.rendered
+  }
 }
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
@@ -53,44 +53,6 @@ locals {
  # If prefix is set, all resources will be prefixed with "${var.prefix}-"
  # Else don't prefix with anything
  resource-prefix = "%{if var.prefix != ""}${var.prefix}-%{endif}"
-
-  master_ip = {
-    for instance in upcloud_server.master :
-      instance.hostname => {
-        for nic in instance.network_interface :
-          nic.type => nic.ip_address
-        if nic.ip_address != null
-      }
-  }
-  worker_ip = {
-    for instance in upcloud_server.worker :
-      instance.hostname => {
-        for nic in instance.network_interface :
-          nic.type => nic.ip_address
-        if nic.ip_address != null
-      }
-  }
-
-  bastion_ip = {
-    for instance in upcloud_server.bastion :
-      instance.hostname => {
-        for nic in instance.network_interface :
-          nic.type => nic.ip_address
-        if nic.ip_address != null
-      }
-  }
-
-  node_user_data = {
-    for name, machine in var.machines :
-      name => <<EOF
-%{ if ( length(machine.dns_servers != null ? machine.dns_servers : [] ) > 0 ) || ( length(var.dns_servers) > 0 && machine.dns_servers == null ) ~}
-#!/bin/bash
-echo -e "[Resolve]\nDNS=${ join(" ", length(machine.dns_servers != null ? machine.dns_servers : []) > 0 ? machine.dns_servers : var.dns_servers) }" > /etc/systemd/resolved.conf
-
-systemctl restart systemd-resolved
-%{ endif ~}
-EOF
-  }
 }

 resource "upcloud_network" "private" {
@@ -100,9 +62,6 @@ resource "upcloud_network" "private" {
  ip_network {
    address            = var.private_network_cidr
    dhcp_default_route = var.router_enable
-    # TODO: When support for dhcp_dns for private networks are in, remove the user_data and enable it here.
-    #       See more here https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562
-    # dhcp_dns           = length(var.private_network_dns) > 0 ? var.private_network_dns : null
    dhcp               = true
    family             = "IPv4"
  }
@@ -130,8 +89,8 @@ resource "upcloud_server" "master" {

  hostname     = "${local.resource-prefix}${each.key}"
  plan         = each.value.plan
-  cpu          = each.value.cpu
-  mem          = each.value.mem
+  cpu          = each.value.plan == null ? null : each.value.cpu
+  mem          = each.value.plan == null ? null : each.value.mem
  zone         = var.zone
  server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id

@@ -140,12 +99,9 @@ resource "upcloud_server" "master" {
    size    = each.value.disk_size
  }

-  dynamic "network_interface" {
-    for_each = each.value.force_public_ip || var.use_public_ips ? [1] : []
-
-    content {
-      type = "public"
-    }
+  # Public network interface
+  network_interface {
+    type = "public"
  }

  # Private network interface
@@ -180,9 +136,6 @@ resource "upcloud_server" "master" {
    keys            = var.ssh_public_keys
    create_password = false
  }
-
-  metadata = local.node_user_data[each.key] != "" ? true : null
-  user_data = local.node_user_data[each.key] != "" ? local.node_user_data[each.key] : null
 }

 resource "upcloud_server" "worker" {
@@ -194,8 +147,8 @@ resource "upcloud_server" "worker" {

  hostname     = "${local.resource-prefix}${each.key}"
  plan         = each.value.plan
-  cpu          = each.value.cpu
-  mem          = each.value.mem
+  cpu          = each.value.plan == null ? null : each.value.cpu
+  mem          = each.value.plan == null ? null : each.value.mem
  zone         = var.zone
  server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id

@@ -205,12 +158,9 @@ resource "upcloud_server" "worker" {
    size    = each.value.disk_size
  }

-  dynamic "network_interface" {
-    for_each = each.value.force_public_ip || var.use_public_ips ? [1] : []
-
-    content {
-      type = "public"
-    }
+  # Public network interface
+  network_interface {
+    type = "public"
  }

  # Private network interface
@@ -245,63 +195,6 @@ resource "upcloud_server" "worker" {
    keys            = var.ssh_public_keys
    create_password = false
  }
-
-  metadata = local.node_user_data[each.key] != "" ? true : null
-  user_data = local.node_user_data[each.key] != "" ? local.node_user_data[each.key] : null
-}
-
-resource "upcloud_server" "bastion" {
-  for_each = {
-    for name, machine in var.machines :
-    name => machine
-    if machine.node_type == "bastion"
-  }
-
-  hostname     = "${local.resource-prefix}${each.key}"
-  plan         = each.value.plan
-  cpu          = each.value.cpu
-  mem          = each.value.mem
-  zone         = var.zone
-  server_group = each.value.server_group == null ? null : upcloud_server_group.server_groups[each.value.server_group].id
-
-
-  template {
-    storage = var.template_name
-    size    = each.value.disk_size
-  }
-
-  # Private network interface
-  network_interface {
-    type    = "private"
-    network = upcloud_network.private.id
-  }
-
-  # Private network interface
-  network_interface {
-    type    = "public"
-  }
-
-  firewall = var.firewall_enabled
-
-  dynamic "storage_devices" {
-    for_each = {
-      for disk_key_name, disk in upcloud_storage.additional_disks :
-      disk_key_name => disk
-      # Only add the disk if it matches the node name in the start of its name
-      if length(regexall("^${each.key}_.+", disk_key_name)) > 0
-    }
-
-    content {
-      storage = storage_devices.value.id
-    }
-  }
-
-  # Include at least one public SSH key
-  login {
-    user            = var.username
-    keys            = var.ssh_public_keys
-    create_password = false
-  }
 }

 resource "upcloud_firewall_rules" "master" {
@@ -650,53 +543,6 @@ resource "upcloud_firewall_rules" "k8s" {
  }
 }

-resource "upcloud_firewall_rules" "bastion" {
-  for_each  = upcloud_server.bastion
-  server_id = each.value.id
-
-  dynamic "firewall_rule" {
-    for_each = var.bastion_allowed_remote_ips
-
-    content {
-      action                 = "accept"
-      comment                = "Allow bastion SSH access from this network"
-      destination_port_end   = "22"
-      destination_port_start = "22"
-      direction              = "in"
-      family                 = "IPv4"
-      protocol               = "tcp"
-      source_address_end     = firewall_rule.value.end_address
-      source_address_start   = firewall_rule.value.start_address
-    }
-  }
-
-  dynamic "firewall_rule" {
-    for_each = length(var.bastion_allowed_remote_ips) > 0 ? [1] : []
-
-    content {
-      action                 = "drop"
-      comment                = "Drop bastion SSH access from other networks"
-      destination_port_end   = "22"
-      destination_port_start = "22"
-      direction              = "in"
-      family                 = "IPv4"
-      protocol               = "tcp"
-      source_address_end     = "255.255.255.255"
-      source_address_start   = "0.0.0.0"
-    }
-  }
-
-  firewall_rule {
-    action    = var.firewall_default_deny_in ? "drop" : "accept"
-    direction = "in"
-  }
-
-  firewall_rule {
-    action    = var.firewall_default_deny_out ? "drop" : "accept"
-    direction = "out"
-  }
-}
-
 resource "upcloud_loadbalancer" "lb" {
  count             = var.loadbalancer_enabled ? 1 : 0
  configured_status = "started"
@@ -737,7 +583,7 @@ resource "upcloud_loadbalancer_backend" "lb_backend" {
  loadbalancer = upcloud_loadbalancer.lb[0].id
  name         = "lb-backend-${each.key}"
  properties {
-    outbound_proxy_protocol = each.value.proxy_protocol ? "v2" : ""
+    outbound_proxy_protocol = var.loadbalancer_outbound_proxy_protocol
  }
 }

@@ -776,7 +622,7 @@ resource "upcloud_loadbalancer_static_backend_member" "lb_backend_member" {

  backend      = upcloud_loadbalancer_backend.lb_backend[each.value.lb_name].id
  name         = "${local.resource-prefix}${each.key}"
-  ip           = merge(local.master_ip, local.worker_ip)["${local.resource-prefix}${each.value.server_name}"].private
+  ip           = merge(upcloud_server.master, upcloud_server.worker)[each.value.server_name].network_interface[1].ip_address
  port         = each.value.port
  weight       = 100
  max_sessions = var.loadbalancer_plan == "production-small" ? 50000 : 1000
@@ -816,7 +662,7 @@ resource "upcloud_router" "router" {
 resource "upcloud_gateway" "gateway" {
  for_each = var.router_enable ? var.gateways : {}
  name = "${local.resource-prefix}${each.key}-gateway"
-  zone = var.private_cloud ? var.public_zone : var.zone
+  zone = var.zone

  features = each.value.features
  plan = each.value.plan
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf
@@ -1,13 +1,22 @@
+
 output "master_ip" {
-  value = local.master_ip
+  value = {
+    for instance in upcloud_server.master :
+    instance.hostname => {
+      "public_ip" : instance.network_interface[0].ip_address
+      "private_ip" : instance.network_interface[1].ip_address
+    }
+  }
 }

 output "worker_ip" {
-  value = local.worker_ip
-}
-
-output "bastion_ip" {
-  value = local.bastion_ip
+  value = {
+    for instance in upcloud_server.worker :
+    instance.hostname => {
+      "public_ip" : instance.network_interface[0].ip_address
+      "private_ip" : instance.network_interface[1].ip_address
+    }
+  }
 }

 output "loadbalancer_domain" {
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
@@ -20,21 +20,15 @@ variable "username" {}

 variable "private_network_cidr" {}

-variable "dns_servers" {}
-
-variable "use_public_ips" {}
-
 variable "machines" {
  description = "Cluster machines"
  type = map(object({
    node_type = string
    plan      = string
-    cpu       = optional(number)
-    mem       = optional(number)
+    cpu       = string
+    mem       = string
    disk_size = number
    server_group : string
-    force_public_ip : optional(bool, false)
-    dns_servers : optional(set(string))
    additional_disks = map(object({
      size = number
      tier = string
@@ -64,13 +58,6 @@ variable "k8s_allowed_remote_ips" {
  }))
 }

-variable "bastion_allowed_remote_ips" {
-  type = list(object({
-    start_address = string
-    end_address   = string
-  }))
-}
-
 variable "master_allowed_ports" {
  type = list(object({
    protocol       = string
@@ -107,6 +94,10 @@ variable "loadbalancer_plan" {
  type = string
 }

+variable "loadbalancer_outbound_proxy_protocol" {
+  type = string
+}
+
 variable "loadbalancer_legacy_network" {
  type = bool
  default = false
@@ -116,7 +107,6 @@ variable "loadbalancers" {
  description = "Load balancers"

  type = map(object({
-    proxy_protocol          = bool
    port                    = number
    target_port             = number
    allow_internal_frontend = optional(bool)
--- a/contrib/terraform/upcloud/output.tf
+++ b/contrib/terraform/upcloud/output.tf
@@ -7,10 +7,6 @@ output "worker_ip" {
  value = module.kubernetes.worker_ip
 }

-output "bastion_ip" {
-  value = module.kubernetes.bastion_ip
-}
-
 output "loadbalancer_domain" {
  value = module.kubernetes.loadbalancer_domain
 }
--- a/contrib/terraform/upcloud/templates/inventory.tpl
+++ b/contrib/terraform/upcloud/templates/inventory.tpl
@@ -1,33 +1,17 @@
+
 [all]
-%{ for name, ips in master_ip ~}
-${name} ansible_user=${username} ansible_host=${lookup(ips, "public", ips.private)} ip=${ips.private}
-%{ endfor ~}
-%{ for name, ips in worker_ip ~}
-${name} ansible_user=${username} ansible_host=${lookup(ips, "public", ips.private)} ip=${ips.private}
-%{ endfor ~}
+${connection_strings_master}
+${connection_strings_worker}

 [kube_control_plane]
-%{ for name, ips in master_ip ~}
-${name}
-%{ endfor ~}
+${list_master}

 [etcd]
-%{ for name, ips in master_ip ~}
-${name}
-%{ endfor ~}
+${list_master}

 [kube_node]
-%{ for name, ips in worker_ip ~}
-${name}
-%{ endfor ~}
+${list_worker}

 [k8s_cluster:children]
 kube_control_plane
 kube_node
-
-%{ if length(bastion_ip) > 0 ~}
-[bastion]
-%{ for name, ips in bastion_ip ~}
-bastion ansible_user=${username} ansible_host=${ips.public}
-%{ endfor ~}
-%{ endif ~}
--- a/contrib/terraform/upcloud/variables.tf
+++ b/contrib/terraform/upcloud/variables.tf
@@ -32,31 +32,16 @@ variable "private_network_cidr" {
  default     = "172.16.0.0/24"
 }

-variable "dns_servers" {
-  description = "DNS servers that will be used by the nodes. Until [this is solved](https://github.com/UpCloudLtd/terraform-provider-upcloud/issues/562) this is done using user_data to reconfigure resolved"
-
-  type    = set(string)
-  default = []
-}
-
-variable "use_public_ips" {
-  description = "If all nodes should get a public IP"
-  type        = bool
-  default     = true
-}
-
 variable "machines" {
  description = "Cluster machines"

  type = map(object({
    node_type = string
    plan      = string
-    cpu       = optional(number)
-    mem       = optional(number)
+    cpu       = string
+    mem       = string
    disk_size = number
    server_group : string
-    force_public_ip : optional(bool, false)
-    dns_servers : optional(set(string))
    additional_disks = map(object({
      size = number
      tier = string
@@ -104,15 +89,6 @@ variable "k8s_allowed_remote_ips" {
  default = []
 }

-variable "bastion_allowed_remote_ips" {
-  description = "List of IP start/end addresses allowed to SSH to bastion"
-  type = list(object({
-    start_address = string
-    end_address   = string
-  }))
-  default = []
-}
-
 variable "master_allowed_ports" {
  description = "List of ports to allow on masters"
  type = list(object({
@@ -155,6 +131,11 @@ variable "loadbalancer_plan" {
  default     = "development"
 }

+variable "loadbalancer_proxy_protocol" {
+  type    = bool
+  default = false
+}
+
 variable "loadbalancer_legacy_network" {
  description = "If the loadbalancer should use the deprecated network field instead of networks blocks. You probably want to have this set to false"

@@ -166,7 +147,6 @@ variable "loadbalancers" {
  description = "Load balancers"

  type = map(object({
-    proxy_protocol          = bool
    port                    = number
    target_port             = number
    allow_internal_frontend = optional(bool, false)
--- a/docs/CNI/calico.md
+++ b/docs/CNI/calico.md
@@ -377,7 +377,7 @@ To clean up any ipvs leftovers:

 ### Calico access to the kube-api

-Calico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.tigera.io/calico/latest/operations/ebpf/enabling-ebpf) for guidelines on how to do this.
+Calico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.projectcalico.org/maintenance/ebpf/enabling-bpf) for guidelines on how to do this.

 Kubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `loadbalancer_apiserver` inventory variable documented in [HA Mode](/docs/operations/ha-mode.md).

--- a/docs/_sidebar.md
+++ b/docs/_sidebar.md
@@ -52,7 +52,9 @@
  * [Test Cases](/docs/developers/test_cases.md)
  * [Vagrant](/docs/developers/vagrant.md)
 * External Storage Provisioners
+  * [Cephfs Provisioner](/docs/external_storage_provisioners/cephfs_provisioner.md)
  * [Local Volume Provisioner](/docs/external_storage_provisioners/local_volume_provisioner.md)
+  * [Rbd Provisioner](/docs/external_storage_provisioners/rbd_provisioner.md)
  * [Scheduler Plugins](/docs/external_storage_provisioners/scheduler_plugins.md)
 * Getting Started
  * [Comparisons](/docs/getting_started/comparisons.md)
--- a/docs/ansible/ansible.md
+++ b/docs/ansible/ansible.md
@@ -65,6 +65,7 @@ The following tags are defined in playbooks:
 | bootstrap-os                   | Anything related to host OS configuration             |
 | calico                         | Network plugin Calico                                 |
 | calico_rr                      | Configuring Calico route reflector                    |
+| cephfs-provisioner             | Configuring CephFS                                    |
 | cert-manager                   | Configuring certificate manager for K8s               |
 | cilium                         | Network plugin Cilium                                 |
 | cinder-csi-driver              | Configuring csi driver: cinder                        |
@@ -146,6 +147,7 @@ The following tags are defined in playbooks:
 | registry                       | Configuring local docker registry                     |
 | reset                          | Tasks running doing the node reset                    |
 | resolvconf                     | Configuring /etc/resolv.conf for hosts/apps           |
+| rbd-provisioner                | Configure External provisioner: rdb                   |
 | services                       | Remove services (etcd, kubelet etc...) when resetting |
 | snapshot                       | Enabling csi snapshot                                 |
 | snapshot-controller            | Configuring csi snapshot controller                   |
--- a/docs/developers/ci.md
+++ b/docs/developers/ci.md
@@ -13,7 +13,6 @@ debian11 |  :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :w
 debian12 |  :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :x: |
 fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
 fedora40 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-flatcar4081 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 opensuse15 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
@@ -33,7 +32,6 @@ debian11 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora40 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-flatcar4081 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 opensuse15 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
@@ -53,7 +51,6 @@ debian11 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian12 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora39 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora40 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-flatcar4081 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 openeuler24 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 opensuse15 |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
 rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: |
--- a/docs/external_storage_provisioners/cephfs_provisioner.md
+++ b/docs/external_storage_provisioners/cephfs_provisioner.md
@@ -0,0 +1,73 @@
+# CephFS Volume Provisioner for Kubernetes 1.5+
+
+[![Docker Repository on Quay](https://quay.io/repository/external_storage/cephfs-provisioner/status "Docker Repository on Quay")](https://quay.io/repository/external_storage/cephfs-provisioner)
+
+Using Ceph volume client
+
+## Development
+
+Compile the provisioner
+
+``` console
+make
+```
+
+Make the container image and push to the registry
+
+``` console
+make push
+```
+
+## Test instruction
+
+- Start Kubernetes local cluster
+
+See [Kubernetes](https://kubernetes.io/)
+
+- Create a Ceph admin secret
+
+``` bash
+ceph auth get client.admin 2>&1 |grep "key = " |awk '{print  $3'} |xargs echo -n > /tmp/secret
+kubectl create ns cephfs
+kubectl create secret generic ceph-secret-admin --from-file=/tmp/secret --namespace=cephfs
+```
+
+- Start CephFS provisioner
+
+The following example uses `cephfs-provisioner-1` as the identity for the instance and assumes kubeconfig is at `/root/.kube`. The identity should remain the same if the provisioner restarts. If there are multiple provisioners, each should have a different identity.
+
+``` bash
+docker run -ti -v /root/.kube:/kube -v /var/run/kubernetes:/var/run/kubernetes --privileged --net=host cephfs-provisioner /usr/local/bin/cephfs-provisioner -master=http://127.0.0.1:8080 -kubeconfig=/kube/config -id=cephfs-provisioner-1
+```
+
+Alternatively, deploy it in kubernetes, see [deployment](deploy/README.md).
+
+- Create a CephFS Storage Class
+
+Replace Ceph monitor's IP in [example class](example/class.yaml) with your own and create storage class:
+
+``` bash
+kubectl create -f example/class.yaml
+```
+
+- Create a claim
+
+``` bash
+kubectl create -f example/claim.yaml
+```
+
+- Create a Pod using the claim
+
+``` bash
+kubectl create -f example/test-pod.yaml
+```
+
+## Known limitations
+
+- Kernel CephFS doesn't work with SELinux, setting SELinux label in Pod's securityContext will not work.
+- Kernel CephFS doesn't support quota or capacity, capacity requested by PVC is not enforced or validated.
+- Currently each Ceph user created by the provisioner has `allow r` MDS cap to permit CephFS mount.
+
+## Acknowledgement
+
+Inspired by CephFS Manila provisioner and conversation with John Spray
--- a/docs/external_storage_provisioners/rbd_provisioner.md
+++ b/docs/external_storage_provisioners/rbd_provisioner.md
@@ -0,0 +1,79 @@
+# RBD Volume Provisioner for Kubernetes 1.5+
+
+`rbd-provisioner` is an out-of-tree dynamic provisioner for Kubernetes 1.5+.
+You can use it quickly & easily deploy ceph RBD storage that works almost
+anywhere.
+
+It works just like in-tree dynamic provisioner. For more information on how
+dynamic provisioning works, see [the docs](https://kubernetes.io/docs/concepts/storage/persistent-volumes/)
+or [this blog post](http://blog.kubernetes.io/2016/10/dynamic-provisioning-and-storage-in-kubernetes.html).
+
+## Development
+
+Compile the provisioner
+
+```console
+make
+```
+
+Make the container image and push to the registry
+
+```console
+make push
+```
+
+## Test instruction
+
+* Start Kubernetes local cluster
+
+See [Kubernetes](https://kubernetes.io/).
+
+* Create a Ceph admin secret
+
+```bash
+ceph auth get client.admin 2>&1 |grep "key = " |awk '{print  $3'} |xargs echo -n > /tmp/secret
+kubectl create secret generic ceph-admin-secret --from-file=/tmp/secret --namespace=kube-system
+```
+
+* Create a Ceph pool and a user secret
+
+```bash
+ceph osd pool create kube 8 8
+ceph auth add client.kube mon 'allow r' osd 'allow rwx pool=kube'
+ceph auth get-key client.kube > /tmp/secret
+kubectl create secret generic ceph-secret --from-file=/tmp/secret --namespace=kube-system
+```
+
+* Start RBD provisioner
+
+The following example uses `rbd-provisioner-1` as the identity for the instance and assumes kubeconfig is at `/root/.kube`. The identity should remain the same if the provisioner restarts. If there are multiple provisioners, each should have a different identity.
+
+```bash
+docker run -ti -v /root/.kube:/kube -v /var/run/kubernetes:/var/run/kubernetes --privileged --net=host quay.io/external_storage/rbd-provisioner /usr/local/bin/rbd-provisioner -master=http://127.0.0.1:8080 -kubeconfig=/kube/config -id=rbd-provisioner-1
+```
+
+Alternatively, deploy it in kubernetes, see [deployment](deploy/README.md).
+
+* Create a RBD Storage Class
+
+Replace Ceph monitor's IP in [examples/class.yaml](examples/class.yaml) with your own and create storage class:
+
+```bash
+kubectl create -f examples/class.yaml
+```
+
+* Create a claim
+
+```bash
+kubectl create -f examples/claim.yaml
+```
+
+* Create a Pod using the claim
+
+```bash
+kubectl create -f examples/test-pod.yaml
+```
+
+## Acknowledgements
+
+* This provisioner is extracted from [Kubernetes core](https://github.com/kubernetes/kubernetes) with some modifications for this project.
--- a/docs/operations/offline-environment.md
+++ b/docs/operations/offline-environment.md
@@ -22,45 +22,6 @@ Then you need to setup the following services on your offline environment:
 You can get artifact lists with [generate_list.sh](/contrib/offline/generate_list.sh) script.
 In addition, you can find some tools for offline deployment under [contrib/offline](/contrib/offline/README.md).

-## Access Control
-
-### Note: access controlled files_repo
-
-To specify a username and password for "{{ files_repo }}", used to download the binaries, you can use url-encoding. Be aware that the Boolean `unsafe_show_logs` will show these credentials when `roles/download/tasks/download_file.yml` runs the task "Download_file | Show url of file to download". You can disable that Boolean in a job-template when running AWX/AAP/Semaphore.
-
-```yaml
-files_repo_host: example.com
-files_repo_path: /repo
-files_repo_user: download
-files_repo_pass: !vault |
-          $ANSIBLE_VAULT;1.1;AES256
-          61663232643236353864663038616361373739613338623338656434386662363539613462626661
-          6435333438313034346164313631303534346564316361370a306661393232626364376436386439
-          64653965663965356137333436616536643132336630313235333232336661373761643766356366
-          6232353233386534380a373262313634613833623537626132633033373064336261383166323230
-          3164
-files_repo: "https://{{ files_repo_user ~ ':' ~ files_repo_pass ~ '@' ~ files_repo_host ~ files_repo_path }}"
-```
-
-### Note: access controlled registry
-
-To specify a username and password for "{{ registry_host }}", used to download the container images, you can use url-encoding too.
-
-```yaml
-registry_pass: !vault |
-          $ANSIBLE_VAULT;1.1;AES256
-          61663232643236353864663038616361373739613338623338656434386662363539613462626661
-          6435333438313034346164313631303534346564316361370a306661393232626364376436386439
-          64653965663965356137333436616536643132336630313235333232336661373761643766356366
-          6232353233386534380a373262313634613833623537626132633033373064336261383166323230
-          3164
-
-containerd_registry_auth:
-  - registry: "{{ registry_host }}"
-    username: "{{ registry_user }}"
-    password: "{{ registry_pass }}"
-```
-
 ## Configure Inventory

 Once all artifacts are accessible from your internal network, **adjust** the following variables
@@ -74,7 +35,6 @@ docker_image_repo: "{{ registry_host }}"
 quay_image_repo: "{{ registry_host }}"
 github_image_repo: "{{ registry_host }}"

-local_path_provisioner_helper_image_repo: "{{ registry_host }}/busybox"
 kubeadm_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubeadm"
 kubectl_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubectl"
 kubelet_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubelet"
@@ -90,7 +50,6 @@ calico_crds_download_url: "{{ files_repo }}/kubernetes/calico/{{ calico_version
 containerd_download_url: "{{ files_repo }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"
 runc_download_url: "{{ files_repo }}/runc.{{ image_arch }}"
 nerdctl_download_url: "{{ files_repo }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
-get_helm_url: "{{ files_repo }}/get.helm.sh"
 # Insecure registries for containerd
 containerd_registries_mirrors:
  - prefix: "{{ registry_addr }}"
--- a/inventory/sample/group_vars/k8s_cluster/addons.yml
+++ b/inventory/sample/group_vars/k8s_cluster/addons.yml
@@ -65,6 +65,37 @@ local_volume_provisioner_enabled: false
 # csi snapshot namespace
 # snapshot_controller_namespace: kube-system

+# CephFS provisioner deployment
+cephfs_provisioner_enabled: false
+# cephfs_provisioner_namespace: "cephfs-provisioner"
+# cephfs_provisioner_cluster: ceph
+# cephfs_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789"
+# cephfs_provisioner_admin_id: admin
+# cephfs_provisioner_secret: secret
+# cephfs_provisioner_storage_class: cephfs
+# cephfs_provisioner_reclaim_policy: Delete
+# cephfs_provisioner_claim_root: /volumes
+# cephfs_provisioner_deterministic_names: true
+
+# RBD provisioner deployment
+rbd_provisioner_enabled: false
+# rbd_provisioner_namespace: rbd-provisioner
+# rbd_provisioner_replicas: 2
+# rbd_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789"
+# rbd_provisioner_pool: kube
+# rbd_provisioner_admin_id: admin
+# rbd_provisioner_secret_name: ceph-secret-admin
+# rbd_provisioner_secret: ceph-key-admin
+# rbd_provisioner_user_id: kube
+# rbd_provisioner_user_secret_name: ceph-secret-user
+# rbd_provisioner_user_secret: ceph-key-user
+# rbd_provisioner_user_secret_namespace: rbd-provisioner
+# rbd_provisioner_fs_type: ext4
+# rbd_provisioner_image_format: "2"
+# rbd_provisioner_image_features: layering
+# rbd_provisioner_storage_class: rbd
+# rbd_provisioner_reclaim_policy: Delete
+
 # Gateway API CRDs
 gateway_api_enabled: false
 # gateway_api_experimental_channel: false
--- a/roles/adduser/molecule/default/molecule.yml
+++ b/roles/adduser/molecule/default/molecule.yml
@@ -14,6 +14,6 @@ provisioner:
      callbacks_enabled: profile_tasks
      timeout: 120
  playbooks:
-    create: ../../../../tests/cloud_playbooks/create-kubevirt.yml
+    create: ../../../../tests/cloud_playbooks/create-packet.yml
 verifier:
  name: testinfra
--- a/roles/bastion-ssh-config/molecule/default/molecule.yml
+++ b/roles/bastion-ssh-config/molecule/default/molecule.yml
@@ -22,6 +22,6 @@ provisioner:
            hosts:
              bastion-01:
  playbooks:
-    create: ../../../../tests/cloud_playbooks/create-kubevirt.yml
+    create: ../../../../tests/cloud_playbooks/create-packet.yml
 verifier:
  name: testinfra
--- a/roles/bootstrap-os/defaults/main.yml
+++ b/roles/bootstrap-os/defaults/main.yml
@@ -2,8 +2,6 @@
 ## CentOS/RHEL/AlmaLinux specific variables
 # Use the fastestmirror yum plugin
 centos_fastestmirror_enabled: false
-# Timeout (in seconds) for checking RHEL subscription status
-rh_subscription_check_timeout: 180

 ## Flatcar Container Linux specific variables
 # Disable locksmithd or leave it in its current state
--- a/roles/bootstrap-os/molecule/default/molecule.yml
+++ b/roles/bootstrap-os/molecule/default/molecule.yml
@@ -32,6 +32,6 @@ provisioner:
          name: foo
          comment: My test comment
  playbooks:
-    create: ../../../../tests/cloud_playbooks/create-kubevirt.yml
+    create: ../../../../tests/cloud_playbooks/create-packet.yml
 verifier:
  name: testinfra
--- a/roles/bootstrap-os/tasks/main.yml
+++ b/roles/bootstrap-os/tasks/main.yml
@@ -48,6 +48,13 @@
    name: "{{ inventory_hostname }}"
  when: override_system_hostname

+- name: Install ceph-commmon package
+  package:
+    name:
+    - ceph-common
+    state: present
+  when: rbd_provisioner_enabled | default(false)
+
 - name: Ensure bash_completion.d folder exists
  file:
    name: /etc/bash_completion.d/
--- a/roles/bootstrap-os/tasks/rhel.yml
+++ b/roles/bootstrap-os/tasks/rhel.yml
@@ -28,7 +28,6 @@
  register: rh_subscription_status
  changed_when: "rh_subscription_status.rc != 0"
  ignore_errors: true  # noqa ignore-errors
-  timeout: "{{ rh_subscription_check_timeout }}"
  become: true

 - name: RHEL subscription Organization ID/Activation Key registration
--- a/roles/container-engine/containerd/molecule/default/molecule.yml
+++ b/roles/container-engine/containerd/molecule/default/molecule.yml
@@ -34,6 +34,6 @@ provisioner:
      callbacks_enabled: profile_tasks
      timeout: 120
  playbooks:
-    create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml
+    create: ../../../../../tests/cloud_playbooks/create-packet.yml
 verifier:
  name: testinfra
--- a/roles/container-engine/cri-dockerd/molecule/default/molecule.yml
+++ b/roles/container-engine/cri-dockerd/molecule/default/molecule.yml
@@ -26,6 +26,6 @@ provisioner:
      all:
        become: true
  playbooks:
-    create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml
+    create: ../../../../../tests/cloud_playbooks/create-packet.yml
 verifier:
  name: testinfra
--- a/roles/container-engine/cri-o/molecule/default/molecule.yml
+++ b/roles/container-engine/cri-o/molecule/default/molecule.yml
@@ -42,6 +42,6 @@ provisioner:
      callbacks_enabled: profile_tasks
      timeout: 120
  playbooks:
-    create: ../../../../../tests/cloud_playbooks/create-kubevirt.yml
+    create: ../../../../../tests/cloud_playbooks/create-packet.yml
 verifier:
  name: testinfra
--- a/roles/kubernetes-apps/external_cloud_controller/openstack/defaults/main.yml
+++ b/roles/kubernetes-apps/external_cloud_controller/openstack/defaults/main.yml
@@ -21,6 +21,6 @@ external_openstack_cacert: "{{ lookup('env', 'OS_CACERT') }}"
 ##    arg1: "value1"
 ##    arg2: "value2"
 external_openstack_cloud_controller_extra_args: {}
-external_openstack_cloud_controller_image_tag: "v1.32.0"
+external_openstack_cloud_controller_image_tag: "v1.30.0"
 external_openstack_cloud_controller_bind_address: 127.0.0.1
 external_openstack_cloud_controller_dns_policy: ClusterFirst
--- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/defaults/main.yml
+++ b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/defaults/main.yml
@@ -0,0 +1,10 @@
+---
+cephfs_provisioner_namespace: "cephfs-provisioner"
+cephfs_provisioner_cluster: ceph
+cephfs_provisioner_monitors: ~
+cephfs_provisioner_admin_id: admin
+cephfs_provisioner_secret: secret
+cephfs_provisioner_storage_class: cephfs
+cephfs_provisioner_reclaim_policy: Delete
+cephfs_provisioner_claim_root: /volumes
+cephfs_provisioner_deterministic_names: true
--- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/tasks/main.yml
+++ b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/tasks/main.yml
@@ -0,0 +1,71 @@
+---
+
+- name: CephFS Provisioner | Remove legacy addon dir and manifests
+  file:
+    path: "{{ kube_config_dir }}/addons/cephfs_provisioner"
+    state: absent
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+  tags:
+    - upgrade
+
+- name: CephFS Provisioner | Remove legacy namespace
+  command: >
+    {{ kubectl }} delete namespace {{ cephfs_provisioner_namespace }}
+  ignore_errors: true  # noqa ignore-errors
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+  tags:
+    - upgrade
+
+- name: CephFS Provisioner | Remove legacy storageclass
+  command: >
+    {{ kubectl }} delete storageclass {{ cephfs_provisioner_storage_class }}
+  ignore_errors: true  # noqa ignore-errors
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+  tags:
+    - upgrade
+
+- name: CephFS Provisioner | Create addon dir
+  file:
+    path: "{{ kube_config_dir }}/addons/cephfs_provisioner"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: CephFS Provisioner | Templates list
+  set_fact:
+    cephfs_provisioner_templates:
+      - { name: 00-namespace, file: 00-namespace.yml, type: ns }
+      - { name: secret-cephfs-provisioner, file: secret-cephfs-provisioner.yml, type: secret }
+      - { name: sa-cephfs-provisioner, file: sa-cephfs-provisioner.yml, type: sa }
+      - { name: clusterrole-cephfs-provisioner, file: clusterrole-cephfs-provisioner.yml, type: clusterrole }
+      - { name: clusterrolebinding-cephfs-provisioner, file: clusterrolebinding-cephfs-provisioner.yml, type: clusterrolebinding }
+      - { name: role-cephfs-provisioner, file: role-cephfs-provisioner.yml, type: role }
+      - { name: rolebinding-cephfs-provisioner, file: rolebinding-cephfs-provisioner.yml, type: rolebinding }
+      - { name: deploy-cephfs-provisioner, file: deploy-cephfs-provisioner.yml, type: deploy }
+      - { name: sc-cephfs-provisioner, file: sc-cephfs-provisioner.yml, type: sc }
+
+- name: CephFS Provisioner | Create manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.file }}"
+    mode: "0644"
+  with_items: "{{ cephfs_provisioner_templates }}"
+  register: cephfs_provisioner_manifests
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: CephFS Provisioner | Apply manifests
+  kube:
+    name: "{{ item.item.name }}"
+    namespace: "{{ cephfs_provisioner_namespace }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.item.file }}"
+    state: "latest"
+  with_items: "{{ cephfs_provisioner_manifests.results }}"
+  when: inventory_hostname == groups['kube_control_plane'][0]
--- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/00-namespace.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/00-namespace.yml.j2
@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ cephfs_provisioner_namespace }}
+  labels:
+    name: {{ cephfs_provisioner_namespace }}
--- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrole-cephfs-provisioner.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrole-cephfs-provisioner.yml.j2
@@ -0,0 +1,22 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cephfs-provisioner
+  namespace: {{ cephfs_provisioner_namespace }}
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "create", "delete"]
--- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrolebinding-cephfs-provisioner.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrolebinding-cephfs-provisioner.yml.j2
@@ -0,0 +1,13 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cephfs-provisioner
+subjects:
+  - kind: ServiceAccount
+    name: cephfs-provisioner
+    namespace: {{ cephfs_provisioner_namespace }}
+roleRef:
+  kind: ClusterRole
+  name: cephfs-provisioner
+  apiGroup: rbac.authorization.k8s.io
--- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/deploy-cephfs-provisioner.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/deploy-cephfs-provisioner.yml.j2
@@ -0,0 +1,34 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cephfs-provisioner
+  namespace: {{ cephfs_provisioner_namespace }}
+  labels:
+    app: cephfs-provisioner
+    version: {{ cephfs_provisioner_image_tag }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cephfs-provisioner
+      version: {{ cephfs_provisioner_image_tag }}
+  template:
+    metadata:
+      labels:
+        app: cephfs-provisioner
+        version: {{ cephfs_provisioner_image_tag }}
+    spec:
+      priorityClassName: {% if cephfs_provisioner_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{ '' }}
+      serviceAccount: cephfs-provisioner
+      containers:
+        - name: cephfs-provisioner
+          image: {{ cephfs_provisioner_image_repo }}:{{ cephfs_provisioner_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          env:
+            - name: PROVISIONER_NAME
+              value: ceph.com/cephfs
+          command:
+            - "/usr/local/bin/cephfs-provisioner"
+          args:
+            - "-id=cephfs-provisioner-1"
--- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/role-cephfs-provisioner.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/role-cephfs-provisioner.yml.j2
@@ -0,0 +1,13 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: cephfs-provisioner
+  namespace: {{ cephfs_provisioner_namespace }}
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["create", "get", "delete"]
+  - apiGroups: [""]
+    resources: ["endpoints"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
--- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/rolebinding-cephfs-provisioner.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/rolebinding-cephfs-provisioner.yml.j2
@@ -0,0 +1,14 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: cephfs-provisioner
+  namespace: {{ cephfs_provisioner_namespace }}
+subjects:
+  - kind: ServiceAccount
+    name: cephfs-provisioner
+    namespace: {{ cephfs_provisioner_namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: cephfs-provisioner
--- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/sa-cephfs-provisioner.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/sa-cephfs-provisioner.yml.j2
@@ -0,0 +1,6 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cephfs-provisioner
+  namespace: {{ cephfs_provisioner_namespace }}
--- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/sc-cephfs-provisioner.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/sc-cephfs-provisioner.yml.j2
@@ -0,0 +1,15 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: {{ cephfs_provisioner_storage_class }}
+provisioner: ceph.com/cephfs
+reclaimPolicy: {{ cephfs_provisioner_reclaim_policy }}
+parameters:
+  cluster: {{ cephfs_provisioner_cluster }}
+  monitors: {{ cephfs_provisioner_monitors }}
+  adminId: {{ cephfs_provisioner_admin_id }}
+  adminSecretName: cephfs-provisioner
+  adminSecretNamespace: {{ cephfs_provisioner_namespace }}
+  claimRoot: {{ cephfs_provisioner_claim_root }}
+  deterministicNames: "{{ cephfs_provisioner_deterministic_names | bool | lower }}"
--- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/secret-cephfs-provisioner.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/secret-cephfs-provisioner.yml.j2
@@ -0,0 +1,9 @@
+---
+kind: Secret
+apiVersion: v1
+metadata:
+  name: cephfs-provisioner
+  namespace: {{ cephfs_provisioner_namespace }}
+type: Opaque
+data:
+  secret: {{ cephfs_provisioner_secret | b64encode }}
--- a/roles/kubernetes-apps/external_provisioner/meta/main.yml
+++ b/roles/kubernetes-apps/external_provisioner/meta/main.yml
@@ -9,6 +9,19 @@ dependencies:
      - local-volume-provisioner
      - external-provisioner

+  - role: kubernetes-apps/external_provisioner/cephfs_provisioner
+    when: cephfs_provisioner_enabled
+    tags:
+      - apps
+      - cephfs-provisioner
+      - external-provisioner
+
+  - role: kubernetes-apps/external_provisioner/rbd_provisioner
+    when: rbd_provisioner_enabled
+    tags:
+      - apps
+      - rbd-provisioner
+      - external-provisioner
  - role: kubernetes-apps/external_provisioner/local_path_provisioner
    when: local_path_provisioner_enabled
    tags:
--- a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/defaults/main.yml
+++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/defaults/main.yml
@@ -0,0 +1,17 @@
+---
+rbd_provisioner_namespace: "rbd-provisioner"
+rbd_provisioner_replicas: 2
+rbd_provisioner_monitors: ~
+rbd_provisioner_pool: kube
+rbd_provisioner_admin_id: admin
+rbd_provisioner_secret_name: ceph-secret-admin
+rbd_provisioner_secret: ceph-key-admin
+rbd_provisioner_user_id: kube
+rbd_provisioner_user_secret_name: ceph-secret-user
+rbd_provisioner_user_secret: ceph-key-user
+rbd_provisioner_user_secret_namespace: rbd-provisioner
+rbd_provisioner_fs_type: ext4
+rbd_provisioner_image_format: "2"
+rbd_provisioner_image_features: layering
+rbd_provisioner_storage_class: rbd
+rbd_provisioner_reclaim_policy: Delete
--- a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/tasks/main.yml
+++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/tasks/main.yml
@@ -0,0 +1,71 @@
+---
+
+- name: RBD Provisioner | Remove legacy addon dir and manifests
+  file:
+    path: "{{ kube_config_dir }}/addons/rbd_provisioner"
+    state: absent
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+  tags:
+    - upgrade
+
+- name: RBD Provisioner | Remove legacy namespace
+  command: >
+    {{ kubectl }} delete namespace {{ rbd_provisioner_namespace }}
+  ignore_errors: true  # noqa ignore-errors
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+  tags:
+    - upgrade
+
+- name: RBD Provisioner | Remove legacy storageclass
+  command: >
+    {{ kubectl }} delete storageclass {{ rbd_provisioner_storage_class }}
+  ignore_errors: true  # noqa ignore-errors
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+  tags:
+    - upgrade
+
+- name: RBD Provisioner | Create addon dir
+  file:
+    path: "{{ kube_config_dir }}/addons/rbd_provisioner"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: RBD Provisioner | Templates list
+  set_fact:
+    rbd_provisioner_templates:
+      - { name: 00-namespace, file: 00-namespace.yml, type: ns }
+      - { name: secret-rbd-provisioner, file: secret-rbd-provisioner.yml, type: secret }
+      - { name: sa-rbd-provisioner, file: sa-rbd-provisioner.yml, type: sa }
+      - { name: clusterrole-rbd-provisioner, file: clusterrole-rbd-provisioner.yml, type: clusterrole }
+      - { name: clusterrolebinding-rbd-provisioner, file: clusterrolebinding-rbd-provisioner.yml, type: clusterrolebinding }
+      - { name: role-rbd-provisioner, file: role-rbd-provisioner.yml, type: role }
+      - { name: rolebinding-rbd-provisioner, file: rolebinding-rbd-provisioner.yml, type: rolebinding }
+      - { name: deploy-rbd-provisioner, file: deploy-rbd-provisioner.yml, type: deploy }
+      - { name: sc-rbd-provisioner, file: sc-rbd-provisioner.yml, type: sc }
+
+- name: RBD Provisioner | Create manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/addons/rbd_provisioner/{{ item.file }}"
+    mode: "0644"
+  with_items: "{{ rbd_provisioner_templates }}"
+  register: rbd_provisioner_manifests
+  when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: RBD Provisioner | Apply manifests
+  kube:
+    name: "{{ item.item.name }}"
+    namespace: "{{ rbd_provisioner_namespace }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/addons/rbd_provisioner/{{ item.item.file }}"
+    state: "latest"
+  with_items: "{{ rbd_provisioner_manifests.results }}"
+  when: inventory_hostname == groups['kube_control_plane'][0]
--- a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/00-namespace.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/00-namespace.yml.j2
@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ rbd_provisioner_namespace }}
+  labels:
+    name: {{ rbd_provisioner_namespace }}
--- a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrole-rbd-provisioner.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrole-rbd-provisioner.yml.j2
@@ -0,0 +1,26 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: rbd-provisioner
+  namespace: {{ rbd_provisioner_namespace }}
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["services"]
+    resourceNames: ["kube-dns","coredns"]
+    verbs: ["list", "get"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "create", "delete"]
--- a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrolebinding-rbd-provisioner.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrolebinding-rbd-provisioner.yml.j2
@@ -0,0 +1,13 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: rbd-provisioner
+subjects:
+  - kind: ServiceAccount
+    name: rbd-provisioner
+    namespace: {{ rbd_provisioner_namespace }}
+roleRef:
+  kind: ClusterRole
+  name: rbd-provisioner
+  apiGroup: rbac.authorization.k8s.io
--- a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/deploy-rbd-provisioner.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/deploy-rbd-provisioner.yml.j2
@@ -0,0 +1,40 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rbd-provisioner
+  namespace: {{ rbd_provisioner_namespace }}
+  labels:
+    app: rbd-provisioner
+    version: {{ rbd_provisioner_image_tag }}
+spec:
+  replicas: {{ rbd_provisioner_replicas }}
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: rbd-provisioner
+      version: {{ rbd_provisioner_image_tag }}
+  template:
+    metadata:
+      labels:
+        app: rbd-provisioner
+        version: {{ rbd_provisioner_image_tag }}
+    spec:
+      priorityClassName: {% if rbd_provisioner_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{ '' }}
+      serviceAccount: rbd-provisioner
+      containers:
+        - name: rbd-provisioner
+          image: {{ rbd_provisioner_image_repo }}:{{ rbd_provisioner_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          env:
+            - name: PROVISIONER_NAME
+              value: ceph.com/rbd
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          command:
+            - "/usr/local/bin/rbd-provisioner"
+          args:
+            - "-id=${POD_NAME}"
--- a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/role-rbd-provisioner.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/role-rbd-provisioner.yml.j2
@@ -0,0 +1,13 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: rbd-provisioner
+  namespace: {{ rbd_provisioner_namespace }}
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get"]
+  - apiGroups: [""]
+    resources: ["endpoints"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
--- a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/rolebinding-rbd-provisioner.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/rolebinding-rbd-provisioner.yml.j2
@@ -0,0 +1,14 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: rbd-provisioner
+  namespace: {{ rbd_provisioner_namespace }}
+subjects:
+  - kind: ServiceAccount
+    name: rbd-provisioner
+    namespace: {{ rbd_provisioner_namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: rbd-provisioner
--- a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sa-rbd-provisioner.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sa-rbd-provisioner.yml.j2
@@ -0,0 +1,6 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: rbd-provisioner
+  namespace: {{ rbd_provisioner_namespace }}
--- a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sc-rbd-provisioner.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sc-rbd-provisioner.yml.j2
@@ -0,0 +1,19 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: {{ rbd_provisioner_storage_class }}
+provisioner: ceph.com/rbd
+reclaimPolicy: {{ rbd_provisioner_reclaim_policy }}
+parameters:
+  monitors: {{ rbd_provisioner_monitors }}
+  adminId: {{ rbd_provisioner_admin_id }}
+  adminSecretNamespace: {{ rbd_provisioner_namespace }}
+  adminSecretName: {{ rbd_provisioner_secret_name }}
+  pool: {{ rbd_provisioner_pool }}
+  userId: {{ rbd_provisioner_user_id }}
+  userSecretNamespace: {{ rbd_provisioner_user_secret_namespace }}
+  userSecretName: {{ rbd_provisioner_user_secret_name }}
+  fsType: "{{ rbd_provisioner_fs_type }}"
+  imageFormat: "{{ rbd_provisioner_image_format }}"
+  imageFeatures: {{ rbd_provisioner_image_features }}
--- a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/secret-rbd-provisioner.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/secret-rbd-provisioner.yml.j2
@@ -0,0 +1,18 @@
+---
+kind: Secret
+apiVersion: v1
+metadata:
+  name: {{ rbd_provisioner_secret_name }}
+  namespace: {{ rbd_provisioner_namespace }}
+type: Opaque
+data:
+  secret: {{ rbd_provisioner_secret | b64encode }}
+---
+kind: Secret
+apiVersion: v1
+metadata:
+  name: {{ rbd_provisioner_user_secret_name }}
+  namespace: {{ rbd_provisioner_user_secret_namespace }}
+type: Opaque
+data:
+  key: {{ rbd_provisioner_user_secret | b64encode }}
--- a/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml
+++ b/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml
@@ -15,6 +15,7 @@
      - not calico_apiserver_version.startswith('v')
      - not calico_ctl_version.startswith('v')
      - not calico_typha_version.startswith('v')
+      - not cephfs_provisioner_version.startswith('v')
      - not cert_manager_version.startswith('v')
      - not cilium_cli_version.startswith('v')
      - not cilium_version.startswith('v')
@@ -36,6 +37,7 @@
      - not metrics_server_version.startswith('v')
      - not multus_version.startswith('v')
      - not netcheck_version.startswith('v')
+      - not rbd_provisioner_version.startswith('v')
      - not runc_version.startswith('v')
      - not skopeo_version.startswith('v')
      - not yq_version.startswith('v')
--- a/roles/kubespray-defaults/defaults/main/checksums.yml
+++ b/roles/kubespray-defaults/defaults/main/checksums.yml
@@ -343,6 +343,8 @@ kubeadm_checksums:
    1.30.0: sha256:a77badcaff292862df8324e17f74ab7ce3c6ea9f390647878f1838a3a832f413
 etcd_binary_checksums:
  arm64:
+    3.5.21: sha256:95bf6918623a097c0385b96f139d90248614485e781ec9bee4768dbb6c79c53f
+    3.5.20: sha256:f034232e6fb64b0d89c45fd78b8b4c3e9fb8d69605dddddcdebf5d7cd96a1531
    3.5.19: sha256:a786fd2c92c3c0404586ffedf1b318e4944a17aefed1fa6566f5712ddb8359ad
    3.5.18: sha256:c2bcaf465537d6d47c8bb82a69e31f786f32257050e3bca445bc4e63479ec714
    3.5.17: sha256:7d717a62520bf39fa1115dfbb1df79479ff74b5eda0914f4132bfa60a48b9549
@@ -358,6 +360,8 @@ etcd_binary_checksums:
    3.5.7: sha256:1a35314900da7db006b198dd917e923459b462128101736c63a3cda57ecdbf51
    3.5.6: sha256:888e25c9c94702ac1254c7655709b44bb3711ebaabd3cb05439f3dd1f2b51a87
  amd64:
+    3.5.21: sha256:adddda4b06718e68671ffabff2f8cee48488ba61ad82900e639d108f2148501c
+    3.5.20: sha256:9ac85616fb8c0e45f485074dde0258ca2b7b42f1dd5320821af5a8b66daf7072
    3.5.19: sha256:16ae742def5f330800590e8d505d72830a3b0b7012e559e6bd76f0bc9864bf42
    3.5.18: sha256:6ddde039a7a506badf34e7edfb38e1ea90e36f05c8cfceba602045df623d86fa
    3.5.17: sha256:eff6ac621d41711085d0f38fab17d8fa3705f6326c3ff11301a1f5a71fc94edd
@@ -373,6 +377,8 @@ etcd_binary_checksums:
    3.5.7: sha256:a43119af79c592a874e8f59c4f23832297849d0c479338f9df36e196b86bc396
    3.5.6: sha256:4db32e3bc06dd0999e2171f76a87c1cffed8369475ec7aa7abee9023635670fb
  ppc64le:
+    3.5.21: sha256:6fb6ecb3d1b331eb177dc610a8efad3aceb1f836d6aeb439ba0bfac5d5c2a38c
+    3.5.20: sha256:563bdac64fc92442cf366c02294dff1cbbd3885a86dbcf7f2e87d9388c3b3223
    3.5.19: sha256:cc8651929f4d5794892eeeabf612a243ea6233125bc5f8b0f711118736e2710f
    3.5.18: sha256:ad90260978a9a94572c8aedfa3c4ab225a451e84ab01a1df35e4124863672999
    3.5.17: sha256:5c737b586a1ebcc12bf0d68a2b56583764f4aba82ab4934629626da93d4a9ecc
@@ -426,6 +432,7 @@ cni_binary_checksums:
    1.0.0: sha256:1a055924b1b859c54a97dc14894ecaa9b81d6d949530b9544f0af4173f5a8f2a
 calicoctl_binary_checksums:
  arm64:
+    3.29.3: sha256:d6cba570af9162dff56714ac5e22dfdd170742bc58a51211f587875a3de79fc4
    3.29.2: sha256:3a9b80335338b7f4af762d4a7cf68e67b40839e50711fbe6e67f9a62b69bafdd
    3.29.1: sha256:6f662d316a267854dc5487242ca7ec8ca70c35b52bed258aafb76c2d113643c2
    3.29.0: sha256:ab23afb283fcdffcf0e1156cdced68d05b6c2b70fd4ea2cbc3189d0ecd43bdfd
@@ -439,6 +446,7 @@ calicoctl_binary_checksums:
    3.27.2: sha256:0fd1f65a511338cf9940835987d420c94ab95b5386288ba9673b736a4d347463
    3.27.0: sha256:b4b8c71f9658165e45336b9b5e4fad865529feeffe4294247eb5b4c4310dcaf9
  amd64:
+    3.29.3: sha256:8101eef6d31ca80db0c64c7ab8930f657dafc1f8696f145ef5d5f162026eedda
    3.29.2: sha256:6076d6745c4d60c0c4322961cbb256a0ffa8476cf7f8dbe5de4ae82c55bca020
    3.29.1: sha256:2ac849181cb1fb40c61c06d075711025cdb909d80562d078cc548d50a0edcd3d
    3.29.0: sha256:df5048549d72a1f7ea4f61c655699d3b16d8a45873f28c3855c39597b73e8a3d
@@ -452,6 +460,7 @@ calicoctl_binary_checksums:
    3.27.2: sha256:692f69dc656e41cd35e23e24f56c98c4aeeb723fed129985b46f71e6eb5e1594
    3.27.0: sha256:46e79ae146b3dd90998f56511cf5d6db64deb97cb784235caf1f99e0672d66e4
  ppc64le:
+    3.29.3: sha256:edb98d2a0d3f8afbf98eb000f0d535d4678af39dd6e10a09ea5615a4824f692f
    3.29.2: sha256:6f3fd72be26fcf52605d9ece716363a73bb194ca59ee34a257156d30fa5c1542
    3.29.1: sha256:ef6064f2ec1a09b5eb8c43ab0c64bd42785c24f5b22b950583fb5074f472c2b7
    3.29.0: sha256:c9c2a29a349c6f681aa79b5f5d6aee738305d95aa7f158b6217f487808758e53
@@ -536,20 +545,19 @@ ciliumcli_binary_checksums:
    0.15.16: sha256:f30095e1a0b926d2114b7a419141bea76e950b643182e97e666950ca05a205d9
    0.15.15: sha256:492279c1f960c79747290a5d1e1b21084a04a93f9e13ab4ae7df4c76fe808aff
 calico_crds_archive_checksums:
-  no_arch:
-    3.29.2: sha256:1866b407213b6191627c0ce7be5a0d7c14a016823b3bbc2a6898c57be6c59917
-    3.29.1: sha256:17894ed9f7487f1418e599fdeff5db9047374dee12d560114e25ff9147a455c3
-    3.29.0: sha256:403a6b8616c4e97b081d7be27e9024f2f66b2d73a0ea037420a29689205b2064
-    3.28.3: sha256:2b3348fb9e022bb6703c460789bd9327d9062c6854262e263fd409ff368034e7
-    3.28.2: sha256:f02a0e99e060850bd9050d4c94d37df899911a5e357e1d26e1b5b79a923bb389
-    3.28.1: sha256:c56f1530e7ded9d5b4afb9d83a7a24da6d2959ef7ad38521813f1c2bf138182d
-    3.28.0: sha256:ee721337db0cd847e91aae1cdfd420596896ebcb865575fd913c2f12ac2cdb76
-    3.27.5: sha256:f9cca65b96ab05732ed1902afd0f3086be54d6bb6b47c86a289ecf4ea5cdc25d
-    3.27.4: sha256:5f6ac510bd6bd8c14542afe91f7dbcf2a846dba02ae3152a3b07a1bfdea96078
-    3.27.3: sha256:d11a32919bff389f642af5df8180ad3cec586030decd35adb2a7d4a8aa3b298e
-    3.27.2: sha256:8154bb4aad887f2a5500b505fe203a918f72c4e602b04c688c4b94f76a26e925
-    3.27.1: sha256:76abb0db222af279e3514cfae02be9259097b565bbb2ffcb776ca00566480edb
-    3.27.0: sha256:2a4b5132035dfd6ac4abc8d545f33de139350eca523e0c5cfe4ac32e43fcb2f1
+  3.29.2: sha256:1866b407213b6191627c0ce7be5a0d7c14a016823b3bbc2a6898c57be6c59917
+  3.29.1: sha256:17894ed9f7487f1418e599fdeff5db9047374dee12d560114e25ff9147a455c3
+  3.29.0: sha256:403a6b8616c4e97b081d7be27e9024f2f66b2d73a0ea037420a29689205b2064
+  3.28.3: sha256:2b3348fb9e022bb6703c460789bd9327d9062c6854262e263fd409ff368034e7
+  3.28.2: sha256:f02a0e99e060850bd9050d4c94d37df899911a5e357e1d26e1b5b79a923bb389
+  3.28.1: sha256:c56f1530e7ded9d5b4afb9d83a7a24da6d2959ef7ad38521813f1c2bf138182d
+  3.28.0: sha256:ee721337db0cd847e91aae1cdfd420596896ebcb865575fd913c2f12ac2cdb76
+  3.27.5: sha256:f9cca65b96ab05732ed1902afd0f3086be54d6bb6b47c86a289ecf4ea5cdc25d
+  3.27.4: sha256:5f6ac510bd6bd8c14542afe91f7dbcf2a846dba02ae3152a3b07a1bfdea96078
+  3.27.3: sha256:d11a32919bff389f642af5df8180ad3cec586030decd35adb2a7d4a8aa3b298e
+  3.27.2: sha256:8154bb4aad887f2a5500b505fe203a918f72c4e602b04c688c4b94f76a26e925
+  3.27.1: sha256:76abb0db222af279e3514cfae02be9259097b565bbb2ffcb776ca00566480edb
+  3.27.0: sha256:2a4b5132035dfd6ac4abc8d545f33de139350eca523e0c5cfe4ac32e43fcb2f1
 helm_archive_checksums:
  arm:
    3.16.4: sha256:432e774d1087d3773737888d384c62477b399227662b42cbf0c32e95e6e72556
@@ -644,6 +652,7 @@ cri_dockerd_archive_checksums:
    0.3.5: sha256:30d47bd89998526d51a8518f9e8ef10baed408ab273879ee0e30350702092938
 runc_checksums:
  arm64:
+    1.2.6: sha256:12c612e2ebe6ca198de676ce75ed557e79fe6109032209bb8e25166c967fe170
    1.2.5: sha256:bfc6575f4c601740539553b639ad6f635c23f76695ed484171bd864df6a23f76
    1.2.4: sha256:285f6c4c3de1d78d9f536a0299ae931219527b2ebd9ad89df5a1072896b7e82a
    1.2.3: sha256:4ef19ab21ce1ae5a01e1d3fa5b005e45cdf59f5d3ab32541c9e262cb2b2d3451
@@ -659,6 +668,7 @@ runc_checksums:
    1.1.9: sha256:b43e9f561e85906f469eef5a7b7992fc586f750f44a0e011da4467e7008c33a0
    1.1.8: sha256:7c22cb618116d1d5216d79e076349f93a672253d564b19928a099c20e4acd658
  amd64:
+    1.2.6: sha256:0774f49d1b1eebb5849e644db5e4dc6f2b06cee05f13b3d17d5d6ba62d6f2ebc
    1.2.5: sha256:fbd851fce6a8e0d67a9d184ea544c2abf67c9fd29b80fcc1adf67dfe9eb036a1
    1.2.4: sha256:e83565aa78ec8f52a4d2b4eb6c4ca262b74c5f6770c1f43670c3029c20175502
    1.2.3: sha256:e6e8c8049b1910fce58fa68c057aaa5f42cee2a73834df5e59e5da7612d2739d
@@ -674,6 +684,7 @@ runc_checksums:
    1.1.9: sha256:b9bfdd4cb27cddbb6172a442df165a80bfc0538a676fbca1a6a6c8f4c6933b43
    1.1.8: sha256:1d05ed79854efc707841dfc7afbf3b86546fc1d0b3a204435ca921c14af8385b
  ppc64le:
+    1.2.6: sha256:0d7fffba4f89920edd3246afd4f07b18a975d0d97193ffae418e8418c236c168
    1.2.5: sha256:3764385971ac719535425629e1ac4d451934392993779ee9e8e8ed7566715f5f
    1.2.4: sha256:141fa41c1f382483ccf374827f99c7843414fceb95e8ceb710aba8bac984d016
    1.2.3: sha256:6d1b771096000a14faae660465faf9626a76afe994cbe60581ec4eac1718f12d
@@ -746,6 +757,8 @@ kata_containers_binary_checksums:
    3.0.1: sha256:e2505482f68cc1b1417b8011f2755bf87171a8dd6daaace28531746118fbddaa
 gvisor_runsc_binary_checksums:
  arm64:
+    '20250326.0': sha512:6946a6a82990fdf39312f63fdf421cec048eea55019fe9ff47df9a5988ee1a8340976d187623201a9f6fd44e3b23cadcc413f32364861382c907055d774c46ae
+    '20250319.0': sha512:8bc6cd22af7e682920d3de134836bd1b787379bf447ccbf554e9f3fc7272aa6650dd9aca52cc36d85910c8bfe40c6d76e11c40518617b91868aeefeb938cb2cd
    '20250304.0': sha512:3e069ef4a4879747ca044b23babbc231367f5dcb36f2755e297d9bef8b8a649f6a05495b43f662a636a92d81312d0eb572663e34f164010b23dda7a707bee3d6
    '20250224.0': sha512:474ccbb94bdbe5d4215b05c27db922d9609af16dc0375bc38d32b0dd07e4f8d1dc2efacea80349175c24b5ed31a2ce51e6735b167a6c74c0a4ee93143d2bb286
    '20250217.0': sha512:3f7e01042a62a800efaff48e9faadfdcfa65e42469795bc0d44e42762b476262e1763fa9aa451865432f077a7d13117c0a520947c0954b489193967e1541441c
@@ -780,6 +793,8 @@ gvisor_runsc_binary_checksums:
    '20240109.0': sha256:51a1b299997834b902192806def688b1e23ff6b14f28a9ed3397f3f6572a189a
    '20231218.0': sha256:86262a78946deacc309c0f08883659ee3298c288048dc30955945e71993c81a8
  amd64:
+    '20250326.0': sha512:1bc7c309c6400f1a700d7bcc6901db632a3e3252ff1313725b9b538dbba8e974fb6789dfc2427f98f70757a4125a29b87c6bf80dd730fc6d07e330bdd72fda40
+    '20250319.0': sha512:ffbb13d965f7da7ef76fdee02ae8473c28252f896680f589f8363914ed9fc0ccfb21ec616595354baee60bdb06847c86b5d6df058f498f9c9bc0e0683fdd335d
    '20250304.0': sha512:b43e386a95dd91f6c5af72c303be8d37b0e0fef60f199916b21760c4e3582865c483bb86d67d60e68342f7512da73bfaa270ff06c5421912ba8c84f272ade48d
    '20250224.0': sha512:9cfff30e4609c0b6fd9719c087827fc9527b2ed64972242937e9c178e85ab7b63ae4a7262e4423a6e7965fbf5c5340a4e536adc35e06493ae9a4aa4d25884f3d
    '20250217.0': sha512:8c61ddb4efc93ce8a6e0da0d270940912db3727dda391d1ab4b8bfaa67c42e5b177195233188858a800d0e8fbe01a4c500aefa1382d8200b15e6f7e84f91289e
@@ -815,6 +830,8 @@ gvisor_runsc_binary_checksums:
    '20231218.0': sha256:c353d36a134dfc2fab8509f72a34abf6a761603975eb00a39e4077c41aeaf31b
 gvisor_containerd_shim_binary_checksums:
  arm64:
+    '20250326.0': sha512:ae89ee8b18f42da15f02c4cb5633679f21bd7e865392b57ab8fda215d3775caf04d447098439826f3257dbde28361febcefe89a43937db0dfaae80b4f914686c
+    '20250319.0': sha512:a8d6b57941be358156ffe729a92df48be3d4f94d3b2c61e19a1bff2dd69aa1220286599b2b2adf456612ddaddc8f812fca82a0aa1571d7ecc07339b4f106206c
    '20250304.0': sha512:3e663aa9fb30fc144ffc02eff1e2104ff9e622dadbba73176933ecacf94372d88dea15e18bbdfb9233224bbb03cdb7fdaaa1ace02c1a7feb9c07b0c0571fab28
    '20250224.0': sha512:6dc610af5a9d974ae416bc4b37e066c513d63566c1f2db157acc61d73bab5e3e589671c111fba2943b88a6cef344f930a2dbc64533ba828c3f2c505a02b19f9a
    '20250217.0': sha512:6dc610af5a9d974ae416bc4b37e066c513d63566c1f2db157acc61d73bab5e3e589671c111fba2943b88a6cef344f930a2dbc64533ba828c3f2c505a02b19f9a
@@ -849,6 +866,8 @@ gvisor_containerd_shim_binary_checksums:
    '20240109.0': sha256:40eb0a4f5f0013afb221e228fd6e71887127c4b09c7f2eb36705a0cd5c746d57
    '20231218.0': sha256:5f66938de981221359a64f05a5c770b228090db3a2697d91ad622c18dd19f4b2
  amd64:
+    '20250326.0': sha512:186c188f722ccb249429083bec31b9879b5f52c42b612e0819dfa84d7765bdc79f8305b6e4af9f9344a3cfaacfb1f650fd1be716d93e8b6c596167d6eed1f4df
+    '20250319.0': sha512:f96512d761656bc3101e46ed5d8f06a5d53ca000c0482af3a0286b6743bac698beb98460b052e393e853c7321562942ff602fa9e7eec5c0979b2621df51a72fc
    '20250304.0': sha512:124be185c421f4620417f21e3a9c50569b6310aeb296dc242fa98ceb403f2990aac77d507690fcdd7d200a861f7c95ee44cfbaa6c59155b23ff66a9dbf25994f
    '20250224.0': sha512:cd1d0dec2cc8789d62e7fcc3823f2b72907b8c264e785a42e46e0e9cb2b2bf54f558c490a7bdd18d79906c967c4433212bb028d99d4953780a218f6f26f40389
    '20250217.0': sha512:cd1d0dec2cc8789d62e7fcc3823f2b72907b8c264e785a42e46e0e9cb2b2bf54f558c490a7bdd18d79906c967c4433212bb028d99d4953780a218f6f26f40389
@@ -884,6 +903,7 @@ gvisor_containerd_shim_binary_checksums:
    '20231218.0': sha256:a0578a357feb9320298730bf5ba683880ba35c476dc74dc82c79f0b5acc42656
 nerdctl_archive_checksums:
  arm:
+    2.0.4: sha256:89e540cb1ac0ed37ec50afc578970a0c9b6a7f1c1b684368da9a726259a3d359
    2.0.3: sha256:d95f238738623ae1f4fb01b6a7f287436ba85493700a9de263b3efbff57424d4
    2.0.2: sha256:910619da11b90d71758e6843543ab2106c20b5149f353289bd6d553151b540ca
    2.0.1: sha256:8f42611dc1554b29dfe990f058ed12920be9cdd78798dcfc6b3845e613eb1252
@@ -897,6 +917,7 @@ nerdctl_archive_checksums:
    1.7.1: sha256:799d35de7a182da35d850308c7f1787cd7321404348ff2d5ba64ad43b06b395a
    1.7.0: sha256:8b9e7cccbcc0a472685d1bc285f591f41005f8699e7265ea5438a3e06aefdcfd
  arm64:
+    2.0.4: sha256:1f394e3aabc2b202ad17a5ece0495ca554ae3ca346a5dd0aa8344a891734763d
    2.0.3: sha256:f2c3f12c99e112cd82ba19ca9b875045c44b2f5a19cecc295ed8d61d415e8851
    2.0.2: sha256:c50ba98be0ef05684948f7873078558504a7cc46ff92cffc764c1625b1cd0d40
    2.0.1: sha256:cbca59744f6e9dea962e1d3a754294b5e64b53b82f4f7f7d603a591f38545fd5
@@ -910,6 +931,7 @@ nerdctl_archive_checksums:
    1.7.1: sha256:46affa0564bb74f595a817e7d5060140099d9cfd9e00e1272b4dbe8b0b85c655
    1.7.0: sha256:1255eea5bc2dbac9339d0a9acfb0651dda117504d52cd52b38cf3c2251db4f39
  amd64:
+    2.0.4: sha256:2f9d22179868db4f0a1daf2fd65f58a24f8a78efd2d9b17659f56bcdce85efd0
    2.0.3: sha256:95ff850688a73eace7453f19e74bf4cc8a1f3e458eeb97ef7a6b74de9825df16
    2.0.2: sha256:1ba015dba039cf6ec2434e88d97707f0b715790e6b7f2e7b6ff7be9200f47bc1
    2.0.1: sha256:96e5e3ed79f189a986cd33a40b0c817d7b6c7d9238f51a0737213f409e5d82af
@@ -923,6 +945,7 @@ nerdctl_archive_checksums:
    1.7.1: sha256:5fc0a6e8c3a71cbba95fbdb6833fb8a7cd8e78f53de10988362d4029c14b905a
    1.7.0: sha256:844c47b175a3d6bc8eaad0c51f23624a5ef10c09e55607803ec2bc846fb04df9
  ppc64le:
+    2.0.4: sha256:93c5bd5f32a3b821fd462a96c6943b06663ff03e8fb16327957fe99576855116
    2.0.3: sha256:8d6283b1fe871e319a2f5cf96fe97aba649eeaac0a2a22c81b9b4d3c613c210a
    2.0.2: sha256:1baed7f4312404da966155856aa1e4b4f48bae73d64fd2cf6c41ef9326a07b10
    2.0.1: sha256:78a3846cacc570e8ee4a1d60928a55954fb4fd1b3b731c0c975a808134166fab
@@ -937,10 +960,12 @@ nerdctl_archive_checksums:
    1.7.0: sha256:e421ae655ff68461bad04b4a1a0ffe40c6f0fcfb0847d5730d66cd95a7fd10cd
 containerd_archive_checksums:
  arm64:
+    2.0.4: sha256:0fde98b24bb55363a54150732e0ac99a43bccf2a9711371bd5470f32790316f2
    2.0.3: sha256:3701008e72e983259afaa594cca5d8126e78e38cf0a586a1f6971cb3f61c4b6b
    2.0.2: sha256:14a2a9f7f75f73e5bcfb8b183d0b84830c54b98ef8c5f6ed70e51f1a230c673e
    2.0.1: sha256:b07120ae227b52edfdb54131d44b13b987b39e8c1f740b0c969b7701e0fad4fa
    2.0.0: sha256:2a00b1553f38aa9e716d61316b661961c2fbfbb7aad7bd73b377be5725ecc0f1
+    1.7.27: sha256:3f03ea60c7dacddf890be3ab18f7ef859d9d104b19627f52038d7984361912bc
    1.7.26: sha256:adea067914e678ac37d5091ead66f1e36e5cced4d395bbd2be60772495e09eff
    1.7.25: sha256:e9201d478e4c931496344b779eb6cb40ce5084ec08c8fff159a02cabb0c6b9bf
    1.7.24: sha256:420406d2b34ebb422ab3755fbeede59bf3bfcfccf5cfa584b558c93769d99064
@@ -968,6 +993,7 @@ containerd_archive_checksums:
    1.7.2: sha256:d75a4ca53d9addd0b2c50172d168b12957e18b2d8b802db2658f2767f15889a6
    1.7.1: sha256:1f828dc063e3c24b0840b284c5635b5a11b1197d564c97f9e873b220bab2b41b
    1.7.0: sha256:e7e5be2d9c92e076f1e2e15c9f0a6e0609ddb75f7616999b843cba92d01e4da2
+    1.6.38: sha256:8a685abe7e5352baab1511933431195964e893e19b4a229af48c7a32ee50ab58
    1.6.37: sha256:c3da773c40a2e509c13a55fbd25a165c5dfaed7c9b67a71bb4033c3d8d2d0b6a
    1.6.36: sha256:48aaf746ad4adc6e5c3b077875ddbd15a8f5b660a5f7dcb533f0205aeeff3785
    1.6.35: sha256:0e0066aeffbd4360bfcf16bd08b6a9e40da7f437aa7b292991ce8d08083bee40
@@ -992,10 +1018,12 @@ containerd_archive_checksums:
    1.6.15: sha256:d63e4d27c51e33cd10f8b5621c559f09ece8a65fec66d80551b36cac9e61a07d
    1.6.14: sha256:3ccb61218e60cbba0e1bbe1e5e2bf809ac1ead8eafbbff36c3195d3edd0e4809
  amd64:
+    2.0.4: sha256:e1c64c5fd60ecd555e750744eaef150b6f78d7f750da5c08c52825aa6b791737
    2.0.3: sha256:ac70856f1d8bd3aa9ca5d62db5516b86dfa0f934c1fd1d1c5fa4422dd12ba45e
    2.0.2: sha256:9bd5b6a1bdf505d520d9a329c520258ed0a17faa9fe3db12712ee858ad59aae3
    2.0.1: sha256:85061a5ce1b306292d5a64f85d5cd3aff93d0982737a1069d370dd6cb7bbfd09
    2.0.0: sha256:6f8da716941f7e89315cefaa6e5a8f1ff10b323ff46611313c455df7ab1ebee1
+    1.7.27: sha256:5b038fb22ab5dbb1ce57dd3d8f102460cd8619ff2afc78870837b06e8c4e840a
    1.7.26: sha256:fdf1fb17086b62fc861103da4e3fda3d79bc543b42d2acef5d07e76b13d35d19
    1.7.25: sha256:02990fa281c0a2c4b073c6d2415d264b682bd693aa7d86c5d8eb4b86d684a18c
    1.7.24: sha256:1a94f15139f37633f39e24f08a4071f4533b285df3cbee6478972d26147bcaef
@@ -1023,6 +1051,7 @@ containerd_archive_checksums:
    1.7.2: sha256:2755c70152ab40856510b4549c2dd530e15f5355eb7bf82868e813c9380e22a7
    1.7.1: sha256:9504771bcb816d3b27fab37a6cf76928ee5e95a31eb41510a7d10ae726e01e85
    1.7.0: sha256:b068b05d58025dc9f2fc336674cac0e377a478930f29b48e068f97c783a423f0
+    1.6.38: sha256:84f6098c96ff4afc6add67fe4dffc2bb206c86f4b8ceea2157124e4c328697a0
    1.6.37: sha256:fd74db561289cede7efcede1dce7da92a7da025b4ca8b2c36fc5559ab1892089
    1.6.36: sha256:e9a53f5f7549afbe9208578609eddecd238b7166663ab273f2954fab77602b3f
    1.6.35: sha256:50f05b2986a8635827e3f015f10d8d1c342e9a9d6886c7392160b5c27ac77c83
@@ -1047,10 +1076,12 @@ containerd_archive_checksums:
    1.6.15: sha256:191bb4f6e4afc237efc5c85b5866b6fdfed731bde12cceaa6017a9c7f8aeda02
    1.6.14: sha256:7da626d46c4edcae1eefe6d48dc6521db3e594a402715afcddc6ac9e67e1bfcd
  ppc64le:
+    2.0.4: sha256:ca970d9a53ae504bc36197d6daa931338c387c83b6948b9f9bfdd1a75e25dcf6
    2.0.3: sha256:2f0faa0086ae81d00680367ee9d75aafd3c4ca4535362db83fea62dd19c47079
    2.0.2: sha256:1b19d31bb8a7f9d26d9b50675e78f397d0b01fa635c33cca456f91c412fa6df1
    2.0.1: sha256:09a25357343c7336fe519e5fd1a9dd0f22da869e9deda50c2bc61b6e8c9384be
    2.0.0: sha256:2e7f4b15ac85c22c1ced102bbb424124078248f0af3183425ff335a998079809
+    1.7.27: sha256:ccdfa16e4bba3a993d74fac794d22ddadc1013d351cd099ea933827050ef05a0
    1.7.26: sha256:34a86b1bd598b34e8c05956c5976fb0c0b347937d3cd0837edbcebc7f9e7e53f
    1.7.25: sha256:0934176e32eace1c23dcb9edff0e78f872bf8f7152b5e6f622e9ccf1ddce8722
    1.7.24: sha256:2ca4d527dac68132a2a6b3971d82ddfd18edc7fa838b7cfcfe6eb11efd017871
@@ -1078,6 +1109,7 @@ containerd_archive_checksums:
    1.7.2: sha256:cbe7ec913cb603ca218bd8867efdce4bee3b0e0115e467e51c910467daf8184e
    1.7.1: sha256:17d97ef55c6ce7af9778dbafb5e73f577d1b34220043a91cccde49dbcc610342
    1.7.0: sha256:051e897d3ee5b8c8097f65be447fea2d29226b583ca5d9ed78e9aebcf4e69889
+    1.6.38: sha256:8971075ef3ab09a478b57342438f5864984b0d38ecd7f432e295727e2035fe4e
    1.6.37: sha256:de4d165271e04b2fabf4c4c6ad5be81dc121c1b79818e43b37d1807e1932b981
    1.6.36: sha256:8978cd8bcd4d5a2640bad26d8ea522a46847b6e4a62da1b07bfa482c8906e5ce
    1.6.35: sha256:99095ab778f6fd532eb01d11771e7f8de8383ef20a00ec536c0cf9c018895115
--- a/roles/kubespray-defaults/defaults/main/download.yml
+++ b/roles/kubespray-defaults/defaults/main/download.yml
@@ -198,6 +198,7 @@ kubectl_binary_checksum: "{{ kubectl_checksums[image_arch][kube_version] }}"
 kubeadm_binary_checksum: "{{ kubeadm_checksums[image_arch][kube_version] }}"
 yq_binary_checksum: "{{ yq_checksums[image_arch][yq_version] }}"
 calicoctl_binary_checksum: "{{ calicoctl_binary_checksums[image_arch][calico_ctl_version] }}"
+calico_crds_archive_checksum: "{{ calico_crds_archive_checksums[calico_version] }}"
 ciliumcli_binary_checksum: "{{ ciliumcli_binary_checksums[image_arch][cilium_cli_version] }}"
 crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}"
 crio_archive_checksum: "{{ crio_archive_checksums[image_arch][crio_version] }}"
@@ -275,7 +276,7 @@ kube_router_image_tag: "v{{ kube_router_version }}"
 multus_image_repo: "{{ github_image_repo }}/k8snetworkplumbingwg/multus-cni"
 multus_image_tag: "v{{ multus_version }}"
 external_openstack_cloud_controller_image_repo: "{{ kube_image_repo }}/provider-os/openstack-cloud-controller-manager"
-external_openstack_cloud_controller_image_tag: "v1.32.0"
+external_openstack_cloud_controller_image_tag: "v1.31.1"

 kube_vip_image_repo: "{{ github_image_repo }}/kube-vip/kube-vip"
 kube_vip_image_tag: v0.8.9
@@ -313,6 +314,12 @@ metrics_server_image_tag: "v{{ metrics_server_version }}"
 local_volume_provisioner_version: "2.5.0"
 local_volume_provisioner_image_repo: "{{ kube_image_repo }}/sig-storage/local-volume-provisioner"
 local_volume_provisioner_image_tag: "v{{ local_volume_provisioner_version }}"
+cephfs_provisioner_version: "2.1.0-k8s1.11"
+cephfs_provisioner_image_repo: "{{ quay_image_repo }}/external_storage/cephfs-provisioner"
+cephfs_provisioner_image_tag: "v{{ cephfs_provisioner_version }}"
+rbd_provisioner_version: "2.1.1-k8s1.11"
+rbd_provisioner_image_repo: "{{ quay_image_repo }}/external_storage/rbd-provisioner"
+rbd_provisioner_image_tag: "v{{ rbd_provisioner_version }}"
 local_path_provisioner_version: "0.0.24"
 local_path_provisioner_image_repo: "{{ docker_image_repo }}/rancher/local-path-provisioner"
 local_path_provisioner_image_tag: "v{{ local_path_provisioner_version }}"
@@ -790,7 +797,7 @@ downloads:
    file: true
    enabled: "{{ kube_network_plugin == 'calico' and calico_datastore == 'kdd' }}"
    dest: "{{ local_release_dir }}/calico-{{ calico_version }}-kdd-crds/{{ calico_version }}.tar.gz"
-    checksum: "{{ calico_crds_archive_checksums.no_arch[calico_version] }}"
+    checksum: "{{ calico_crds_archive_checksum }}"
    url: "{{ calico_crds_download_url }}"
    unarchive: true
    unarchive_extra_opts:
@@ -940,6 +947,24 @@ downloads:
    groups:
      - kube_node

+  cephfs_provisioner:
+    enabled: "{{ cephfs_provisioner_enabled }}"
+    container: true
+    repo: "{{ cephfs_provisioner_image_repo }}"
+    tag: "{{ cephfs_provisioner_image_tag }}"
+    checksum: "{{ cephfs_provisioner_digest_checksum | default(None) }}"
+    groups:
+      - kube_node
+
+  rbd_provisioner:
+    enabled: "{{ rbd_provisioner_enabled }}"
+    container: true
+    repo: "{{ rbd_provisioner_image_repo }}"
+    tag: "{{ rbd_provisioner_image_tag }}"
+    checksum: "{{ rbd_provisioner_digest_checksum | default(None) }}"
+    groups:
+      - kube_node
+
  local_path_provisioner:
    enabled: "{{ local_path_provisioner_enabled }}"
    container: true
--- a/roles/kubespray-defaults/defaults/main/main.yml
+++ b/roles/kubespray-defaults/defaults/main/main.yml
@@ -435,6 +435,8 @@ vsphere_csi_enabled: false
 upcloud_csi_enabled: false
 csi_snapshot_controller_enabled: false
 persistent_volumes_enabled: false
+cephfs_provisioner_enabled: false
+rbd_provisioner_enabled: false
 ingress_nginx_enabled: false
 ingress_alb_enabled: false
 cert_manager_enabled: false
--- a/roles/network_plugin/calico/tasks/check.yml
+++ b/roles/network_plugin/calico/tasks/check.yml
@@ -27,8 +27,8 @@
 - name: Stop if supported Calico versions
  assert:
    that:
-      - "calico_version in calico_crds_archive_checksums.no_arch.keys()"
-    msg: "Calico version not supported {{ calico_version }} not in {{ calico_crds_archive_checksums.no_arch.keys() }}"
+      - "calico_version in calico_crds_archive_checksums.keys()"
+    msg: "Calico version not supported {{ calico_version }} not in {{ calico_crds_archive_checksums.keys() }}"
  run_once: true
  delegate_to: "{{ groups['kube_control_plane'][0] }}"

--- a/scripts/component_hash_update/src/component_hash_update/components.py
+++ b/scripts/component_hash_update/src/component_hash_update/components.py
@@ -7,11 +7,6 @@ infos = {
        "url": "https://github.com/projectcalico/calico/releases/download/v{version}/SHA256SUMS",
        "graphql_id": "R_kgDOA87D0g",
    },
-    "calico_crds_archive": {
-        "url": "https://github.com/projectcalico/calico/archive/v{version}.tar.gz",
-        "graphql_id": "R_kgDOA87D0g",
-        "binary": True,
-    },
    "ciliumcli_binary": {
        "url": "https://github.com/cilium/cilium-cli/releases/download/v{version}/cilium-{os}-{arch}.tar.gz.sha256sum",
        "graphql_id": "R_kgDOE0nmLg",
--- a/scripts/component_hash_update/src/component_hash_update/download.py
+++ b/scripts/component_hash_update/src/component_hash_update/download.py
@@ -47,13 +47,17 @@ arch_alt_name = {
    "arm64": "aarch64",
    "ppc64le": None,
    "arm": None,
-    "no_arch": None,
 }

 # TODO: downloads not supported
+# gvisor: sha512 checksums
 # helm_archive: PGP signatures
+# krew_archive: different yaml structure (in our download)
+# calico_crds_archive: different yaml structure (in our download)

 # TODO:
+# noarch support -> k8s manifests, helm charts
+# different checksum format (needs download role changes)
 # different verification methods (gpg, cosign) ( needs download role changes) (or verify the sig in this script and only use the checksum in the playbook)
 # perf improvements (async)

--- a/scripts/readme_versions.md.j2
+++ b/scripts/readme_versions.md.j2
@@ -23,6 +23,8 @@
  - [metallb](https://metallb.universe.tf/) {{ metallb_version }}
  - [registry](https://github.com/distribution/distribution) {{ registry_version }}
 - Storage Plugin
+  - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) {{ cephfs_provisioner_version }}
+  - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) {{ rbd_provisioner_version }}
  - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) {{ aws_ebs_csi_plugin_version }}
  - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) {{ azure_csi_plugin_version }}
  - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) {{ cinder_csi_plugin_version }}
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -8,7 +8,7 @@ $(ANSIBLE_INVENTORY):
 	mkdir $@

 create-packet: | $(ANSIBLE_INVENTORY)
-	ansible-playbook cloud_playbooks/create-kubevirt.yml -c local \
+	ansible-playbook cloud_playbooks/create-packet.yml -c local \
 		-e @"files/${CI_JOB_NAME}.yml"

 delete-packet: ;
--- a/tests/cloud_playbooks/create-kubevirt.yml
+++ b/tests/cloud_playbooks/create-kubevirt.yml
--- a/tests/cloud_playbooks/roles/packet-ci/tasks/main.yml
+++ b/tests/cloud_playbooks/roles/packet-ci/tasks/main.yml
@@ -1,4 +1,8 @@
 ---
+- name: Include custom vars for ci job
+  include_vars: "../files/{{ lookup('ansible.builtin.env', 'CI_JOB_NAME') }}.yml"
+  when: molecule_yml is not defined
+
 - name: Generate SSH keypair
  community.crypto.openssh_keypair:
    size: 2048
--- a/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2
+++ b/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2
@@ -55,7 +55,7 @@ spec:
      containerDisk:
        image: quay.io/kubespray/vm-{{ cloud_image }}
    - name: cloudinitvolume
-      cloudInitConfigDrive:
+      cloudInitNoCloud:
        userData: |
          #cloud-config
           users:
--- a/tests/files/almalinux9-calico-remove-node
+++ b/tests/files/almalinux9-calico-remove-node
@@ -1,2 +0,0 @@
-REMOVE_NODE_CHECK=true
-REMOVE_NODE_NAME=instance-3
--- a/tests/files/debian11-calico-upgrade
+++ b/tests/files/debian11-calico-upgrade
@@ -1 +0,0 @@
-UPGRADE_TEST=graceful
--- a/tests/files/debian11-calico-upgrade-once
+++ b/tests/files/debian11-calico-upgrade-once
@@ -1 +0,0 @@
-UPGRADE_TEST=graceful
--- a/tests/files/fedora39-kube-router.yml
+++ b/tests/files/fedora39-kube-router.yml
@@ -1,7 +0,0 @@
---
-cloud_image: fedora-39
-cluster_layout:
-  - node_groups: ['kube_control_plane', 'etcd', 'kube_node']
-  - node_groups: ['kube_node']
-
-kube_network_plugin: "kube-router"
--- a/tests/files/fedora40-docker.calico
+++ b/tests/files/fedora40-docker.calico
@@ -1 +0,0 @@
-RESET_CHECK=true
--- a/tests/files/packet_almalinux8-calico.yml
+++ b/tests/files/packet_almalinux8-calico.yml
--- a/tests/files/packet_almalinux9-calico-ha-ebpf.yml
+++ b/tests/files/packet_almalinux9-calico-ha-ebpf.yml
--- a/tests/files/packet_almalinux9-calico-nodelocaldns-secondary.yml
+++ b/tests/files/packet_almalinux9-calico-nodelocaldns-secondary.yml
--- a/tests/files/packet_almalinux9-calico-remove-node.yml
+++ b/tests/files/packet_almalinux9-calico-remove-node.yml
--- a/tests/files/packet_almalinux9-calico.yml
+++ b/tests/files/packet_almalinux9-calico.yml
--- a/tests/files/packet_almalinux9-crio.yml
+++ b/tests/files/packet_almalinux9-crio.yml
--- a/tests/files/packet_almalinux9-docker.yml
+++ b/tests/files/packet_almalinux9-docker.yml
--- a/tests/files/packet_almalinux9-kube-ovn.yml
+++ b/tests/files/packet_almalinux9-kube-ovn.yml
--- a/tests/files/packet_amazon-linux-2-all-in-one.yml
+++ b/tests/files/packet_amazon-linux-2-all-in-one.yml
--- a/tests/files/packet_debian11-calico-collection.yml
+++ b/tests/files/packet_debian11-calico-collection.yml
--- a/tests/files/packet_debian11-calico-upgrade-once.yml
+++ b/tests/files/packet_debian11-calico-upgrade-once.yml
--- a/Show More
+++ b/Show More