[2.20] Fix: install calico-kube-controller on kdd (#9358) (#9470)

* Fix: install policy controller on kdd too

* Remove the calico_policy_version condition altogether

* Install policy controller both on canal and calico under same condition

Co-authored-by: Piotr Kowalczyk <7711184+wayfrro@users.noreply.github.com>

.gitignore

@@ -3,7 +3,10 @@
 **/vagrant_ansible_inventory
 *.iml
 temp
+contrib/offline/offline-files
+contrib/offline/offline-files.tar.gz
 .idea
+.vscode
 .tox
 .cache
 *.bak
@@ -11,6 +14,7 @@ temp
 *.tfstate.backup
 .terraform/
 contrib/terraform/aws/credentials.tfvars
+.terraform.lock.hcl
 /ssh-bastion.conf
 **/*.sw[pon]
 *~
@@ -108,3 +112,4 @@ roles/**/molecule/**/__pycache__/
 
 # Temp location used by our scripts
 scripts/tmp/
+tmp.md

.gitlab-ci.yml

@@ -8,7 +8,7 @@ stages:
   - deploy-special
 
 variables:
-  KUBESPRAY_VERSION: v2.18.1
+  KUBESPRAY_VERSION: v2.19.1
   FAILFASTCI_NAMESPACE: 'kargo-ci'
   GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray'
   ANSIBLE_FORCE_COLOR: "true"
@@ -34,7 +34,7 @@ variables:
   RECOVER_CONTROL_PLANE_TEST: "false"
   RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:],kube_control_plane[1:]"
   TERRAFORM_VERSION: 1.0.8
-  ANSIBLE_MAJOR_VERSION: "2.10"
+  ANSIBLE_MAJOR_VERSION: "2.11"
 
 before_script:
   - ./tests/scripts/rebase.sh

.gitlab-ci/lint.yml

@@ -68,6 +68,13 @@ markdownlint:
   script:
     - markdownlint $(find . -name '*.md' | grep -vF './.git') --ignore docs/_sidebar.md --ignore contrib/dind/README.md
 
+check-readme-versions:
+  stage: unit-tests
+  tags: [light]
+  image: python:3
+  script:
+    - tests/scripts/check_readme_versions.sh
+
 ci-matrix:
   stage: unit-tests
   tags: [light]

.gitlab-ci/molecule.yml

@@ -44,7 +44,7 @@ molecule_no_container_engines:
 molecule_docker:
   extends: .molecule
   script:
-    - ./tests/scripts/molecule_run.sh -i container-engine/docker
+    - ./tests/scripts/molecule_run.sh -i container-engine/cri-dockerd
   when: on_success
 
 molecule_containerd:
@@ -60,13 +60,6 @@ molecule_cri-o:
     - ./tests/scripts/molecule_run.sh -i container-engine/cri-o
   when: on_success
 
-molecule_cri-dockerd:
-  extends: .molecule
-  stage: deploy-part2
-  script:
-    - ./tests/scripts/molecule_run.sh -i container-engine/cri-dockerd
-  when: on_success
-
 # Stage 3 container engines don't get as much attention so allow them to fail
 molecule_kata:
   extends: .molecule

.gitlab-ci/packet.yml

@@ -31,23 +31,6 @@ packet_ubuntu20-calico-aio:
   variables:
     RESET_CHECK: "true"
 
-# Exericse ansible variants during the nightly jobs
-packet_ubuntu20-calico-aio-ansible-2_9:
-  stage: deploy-part1
-  extends: .packet_periodic
-  when: on_success
-  variables:
-    ANSIBLE_MAJOR_VERSION: "2.9"
-    RESET_CHECK: "true"
-
-packet_ubuntu20-calico-aio-ansible-2_10:
-  stage: deploy-part1
-  extends: .packet_periodic
-  when: on_success
-  variables:
-    ANSIBLE_MAJOR_VERSION: "2.10"
-    RESET_CHECK: "true"
-
 packet_ubuntu20-calico-aio-ansible-2_11:
   stage: deploy-part1
   extends: .packet_periodic
@@ -73,6 +56,16 @@ packet_ubuntu18-calico-aio:
   extends: .packet_pr
   when: on_success
 
+packet_ubuntu22-aio-docker:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_ubuntu22-calico-aio:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
 packet_centos7-flannel-addons-ha:
   extends: .packet_pr
   stage: deploy-part2
@@ -153,12 +146,22 @@ packet_almalinux8-calico:
   extends: .packet_pr
   when: on_success
 
+packet_rockylinux8-calico:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
+packet_rockylinux9-calico:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
 packet_almalinux8-docker:
   stage: deploy-part2
   extends: .packet_pr
   when: on_success
 
-packet_fedora34-docker-weave:
+packet_fedora36-docker-weave:
   stage: deploy-part2
   extends: .packet_pr
   when: on_success
@@ -216,19 +219,19 @@ packet_centos7-multus-calico:
   extends: .packet_pr
   when: manual
 
-packet_oracle7-canal-ha:
+packet_centos7-canal-ha:
   stage: deploy-part2
   extends: .packet_pr
   when: manual
 
-packet_fedora35-docker-calico:
+packet_fedora36-docker-calico:
   stage: deploy-part2
   extends: .packet_periodic
   when: on_success
   variables:
     RESET_CHECK: "true"
 
-packet_fedora34-calico-selinux:
+packet_fedora35-calico-selinux:
   stage: deploy-part2
   extends: .packet_periodic
   when: on_success
@@ -248,7 +251,7 @@ packet_almalinux8-calico-nodelocaldns-secondary:
   extends: .packet_pr
   when: manual
 
-packet_fedora34-kube-ovn:
+packet_fedora36-kube-ovn:
   stage: deploy-part2
   extends: .packet_periodic
   when: on_success
@@ -256,7 +259,7 @@ packet_fedora34-kube-ovn:
 # ### PR JOBS PART3
 # Long jobs (45min+)
 
-packet_centos7-docker-weave-upgrade-ha:
+packet_centos7-weave-upgrade-ha:
   stage: deploy-part3
   extends: .packet_periodic
   when: on_success
@@ -276,7 +279,7 @@ packet_ubuntu20-calico-ha-wireguard:
   extends: .packet_pr
   when: manual
 
-packet_debian10-calico-upgrade:
+packet_debian11-calico-upgrade:
   stage: deploy-part3
   extends: .packet_pr
   when: on_success
@@ -291,7 +294,12 @@ packet_almalinux8-calico-remove-node:
     REMOVE_NODE_CHECK: "true"
     REMOVE_NODE_NAME: "instance-3"
 
-packet_debian10-calico-upgrade-once:
+packet_ubuntu20-calico-etcd-kubeadm:
+  stage: deploy-part3
+  extends: .packet_pr
+  when: on_success
+
+packet_debian11-calico-upgrade-once:
   stage: deploy-part3
   extends: .packet_periodic
   when: on_success

.gitlab-ci/shellcheck.yml

@@ -11,6 +11,6 @@ shellcheck:
     - cp shellcheck-"${SHELLCHECK_VERSION}"/shellcheck /usr/bin/
     - shellcheck --version
   script:
-    # Run shellcheck for all *.sh except contrib/
-    - find . -name '*.sh' -not -path './contrib/*' -not -path './.git/*' | xargs shellcheck --severity error
+    # Run shellcheck for all *.sh
+    - find . -name '*.sh' -not -path './.git/*' | xargs shellcheck --severity error
   except: ['triggers', 'master']

.markdownlint.yaml

@@ -1,2 +1,3 @@
 ---
 MD013: false
+MD029: false

.pre-commit-config.yaml

@@ -0,0 +1,48 @@
+---
+repos:
+  - repo: https://github.com/adrienverge/yamllint.git
+    rev: v1.27.1
+    hooks:
+      - id: yamllint
+        args: [--strict]
+
+  - repo: https://github.com/markdownlint/markdownlint
+    rev: v0.11.0
+    hooks:
+      - id: markdownlint
+        args: [ -r, "~MD013,~MD029" ]
+        exclude: "^.git"
+
+  - repo: local
+    hooks:
+      - id: ansible-lint
+        name: ansible-lint
+        entry: ansible-lint -v
+        language: python
+        pass_filenames: false
+        additional_dependencies:
+          - .[community]
+
+      - id: ansible-syntax-check
+        name: ansible-syntax-check
+        entry: env ANSIBLE_INVENTORY=inventory/local-tests.cfg ANSIBLE_REMOTE_USER=root ANSIBLE_BECOME="true" ANSIBLE_BECOME_USER=root ANSIBLE_VERBOSITY="3" ansible-playbook --syntax-check
+        language: python
+        files: "^cluster.yml|^upgrade-cluster.yml|^reset.yml|^extra_playbooks/upgrade-only-k8s.yml"
+
+      - id: tox-inventory-builder
+        name: tox-inventory-builder
+        entry: bash -c "cd contrib/inventory_builder && tox"
+        language: python
+        pass_filenames: false
+
+      - id: check-readme-versions
+        name: check-readme-versions
+        entry: tests/scripts/check_readme_versions.sh
+        language: script
+        pass_filenames: false
+
+      - id: ci-matrix
+        name: ci-matrix
+        entry: tests/scripts/md-table/test.sh
+        language: script
+        pass_filenames: false

CONTRIBUTING.md

@@ -16,7 +16,12 @@ pip install -r tests/requirements.txt
 
 #### Linting
 
-Kubespray uses `yamllint` and `ansible-lint`. To run them locally use `yamllint .` and `ansible-lint`. It is a good idea to add call these tools as part of your pre-commit hook and avoid a lot of back end forth on fixing linting issues (<https://support.gitkraken.com/working-with-repositories/githooksexample/>).
+Kubespray uses [pre-commit](https://pre-commit.com) hook configuration to run several linters, please install this tool and use it to run validation tests before submitting a PR.
+
+```ShellSession
+pre-commit install
+pre-commit run -a  # To run pre-commit hook on all files in the repository, even if they were not modified
+```
 
 #### Molecule
 
@@ -33,7 +38,9 @@ Vagrant with VirtualBox or libvirt driver helps you to quickly spin test cluster
 1. Submit an issue describing your proposed change to the repo in question.
 2. The [repo owners](OWNERS) will respond to your issue promptly.
 3. Fork the desired repo, develop and test your code changes.
-4. Sign the CNCF CLA (<https://git.k8s.io/community/CLA.md#the-contributor-license-agreement>)
-5. Submit a pull request.
-6. Work with the reviewers on their suggestions.
-7. Ensure to rebase to the HEAD of your target branch and squash un-necessary commits (<https://blog.carbonfive.com/always-squash-and-rebase-your-git-commits/>) before final merger of your contribution.
+4. Install [pre-commit](https://pre-commit.com) and install it in your development repo.
+5. Addess any pre-commit validation failures.
+6. Sign the CNCF CLA (<https://git.k8s.io/community/CLA.md#the-contributor-license-agreement>)
+7. Submit a pull request.
+8. Work with the reviewers on their suggestions.
+9. Ensure to rebase to the HEAD of your target branch and squash un-necessary commits (<https://blog.carbonfive.com/always-squash-and-rebase-your-git-commits/>) before final merger of your contribution.

Dockerfile

@@ -1,5 +1,5 @@
 # Use imutable image tags rather than mutable tags (like ubuntu:20.04)
-FROM ubuntu:focal-20220316
+FROM ubuntu:focal-20220531
 
 ARG ARCH=amd64
 ARG TZ=Etc/UTC

OWNERS_ALIASES

@@ -8,6 +8,7 @@ aliases:
     - floryut
     - oomichi
     - cristicalin
+    - liupeng0518
   kubespray-reviewers:
     - holmsten
     - bozzo
@@ -16,6 +17,7 @@ aliases:
     - jayonlau
     - cristicalin
     - liupeng0518
+    - yankay
   kubespray-emeritus_approvers:
     - riverzhang
     - atoms

README.md

@@ -57,10 +57,10 @@ A simple way to ensure you get all the correct version of Ansible is to use the
 You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mounts/) to get the inventory and ssh key into the container, like this:
 
 ```ShellSession
-docker pull quay.io/kubespray/kubespray:v2.18.1
+docker pull quay.io/kubespray/kubespray:v2.20.0
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
   --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.18.1 bash
+  quay.io/kubespray/kubespray:v2.20.0 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```
@@ -111,6 +111,7 @@ vagrant up
 - [Adding/replacing a node](docs/nodes.md)
 - [Upgrades basics](docs/upgrades.md)
 - [Air-Gap installation](docs/offline-environment.md)
+- [NTP](docs/ntp.md)
 - [Hardening](docs/hardening.md)
 - [Roadmap](docs/roadmap.md)
 
@@ -118,14 +119,15 @@ vagrant up
 
 - **Flatcar Container Linux by Kinvolk**
 - **Debian** Bullseye, Buster, Jessie, Stretch
-- **Ubuntu** 16.04, 18.04, 20.04
-- **CentOS/RHEL** 7, [8](docs/centos8.md)
-- **Fedora** 34, 35
+- **Ubuntu** 16.04, 18.04, 20.04, 22.04
+- **CentOS/RHEL** 7, [8, 9](docs/centos.md#centos-8)
+- **Fedora** 35, 36
 - **Fedora CoreOS** (see [fcos Note](docs/fcos.md))
 - **openSUSE** Leap 15.x/Tumbleweed
-- **Oracle Linux** 7, [8](docs/centos8.md)
-- **Alma Linux** [8](docs/centos8.md)
-- **Rocky Linux** [8](docs/centos8.md)
+- **Oracle Linux** 7, [8, 9](docs/centos.md#centos-8)
+- **Alma Linux** [8, 9](docs/centos.md#centos-8)
+- **Rocky Linux** [8, 9](docs/centos.md#centos-8)
+- **Kylin Linux Advanced Server V10** (experimental: see [kylin linux notes](docs/kylinlinux.md))
 - **Amazon Linux 2** (experimental: see [amazon linux notes](docs/amazonlinux.md))
 
 Note: Upstart/SysV init based OS types are not supported.
@@ -133,27 +135,40 @@ Note: Upstart/SysV init based OS types are not supported.
 ## Supported Components
 
 - Core
-  - [kubernetes](https://github.com/kubernetes/kubernetes) v1.23.7
-  - [etcd](https://github.com/etcd-io/etcd) v3.5.3
+  - [kubernetes](https://github.com/kubernetes/kubernetes) v1.24.6
+  - [etcd](https://github.com/etcd-io/etcd) v3.5.4
   - [docker](https://www.docker.com/) v20.10 (see note)
-  - [containerd](https://containerd.io/) v1.6.4
-  - [cri-o](http://cri-o.io/) v1.22 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS)
+  - [containerd](https://containerd.io/) v1.6.8
+  - [cri-o](http://cri-o.io/) v1.24 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
   - [cni-plugins](https://github.com/containernetworking/plugins) v1.1.1
-  - [calico](https://github.com/projectcalico/calico) v3.22.3
+  - [calico](https://github.com/projectcalico/calico) v3.23.3
   - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
-  - [cilium](https://github.com/cilium/cilium) v1.11.3
-  - [flanneld](https://github.com/flannel-io/flannel) v0.17.0
-  - [kube-ovn](https://github.com/alauda/kube-ovn) v1.9.2
-  - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.4.0
+  - [cilium](https://github.com/cilium/cilium) v1.12.1
+  - [flannel](https://github.com/flannel-io/flannel) v0.19.2
+  - [kube-ovn](https://github.com/alauda/kube-ovn) v1.9.7
+  - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.5.1
   - [multus](https://github.com/intel/multus-cni) v3.8
   - [weave](https://github.com/weaveworks/weave) v2.8.1
+  - [kube-vip](https://github.com/kube-vip/kube-vip) v0.4.2
 - Application
+  - [cert-manager](https://github.com/jetstack/cert-manager) v1.9.1
+  - [coredns](https://github.com/coredns/coredns) v1.8.6
+  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.3.1
+  - [krew](https://github.com/kubernetes-sigs/krew) v0.4.3
+  - [argocd](https://argoproj.github.io/) v2.4.12
+  - [helm](https://helm.sh/) v3.9.4
+  - [metallb](https://metallb.universe.tf/)  v0.12.1
+  - [registry](https://github.com/distribution/distribution) v2.8.1
+- Storage Plugin
   - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.0-k8s1.11
   - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.1-k8s1.11
-  - [cert-manager](https://github.com/jetstack/cert-manager) v1.8.0
-  - [coredns](https://github.com/coredns/coredns) v1.8.6
-  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.2.1
+  - [aws-ebs-csi-plugin](https://github.com/kubernetes-sigs/aws-ebs-csi-driver) v0.5.0
+  - [azure-csi-plugin](https://github.com/kubernetes-sigs/azuredisk-csi-driver) v1.10.0
+  - [cinder-csi-plugin](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/cinder-csi-plugin/using-cinder-csi-plugin.md) v1.22.0
+  - [gcp-pd-csi-plugin](https://github.com/kubernetes-sigs/gcp-compute-persistent-disk-csi-driver) v1.4.0
+  - [local-path-provisioner](https://github.com/rancher/local-path-provisioner) v0.0.22
+  - [local-volume-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner) v2.4.0
 
 ## Container Runtime Notes
 
@@ -162,8 +177,8 @@ Note: Upstart/SysV init based OS types are not supported.
 
 ## Requirements
 
-- **Minimum required version of Kubernetes is v1.21**
-- **Ansible v2.9.x, Jinja 2.11+ and python-netaddr is installed on the machine that will run Ansible commands**
+- **Minimum required version of Kubernetes is v1.22**
+- **Ansible v2.11+, Jinja 2.11+ and python-netaddr is installed on the machine that will run Ansible commands**
 - The target servers must have **access to the Internet** in order to pull docker images. Otherwise, additional configuration is required (See [Offline Environment](docs/offline-environment.md))
 - The target servers are configured to allow **IPv4 forwarding**.
 - If using IPv6 for pods and services, the target servers are configured to allow **IPv6 forwarding**.

RELEASE.md

@@ -9,10 +9,10 @@ The Kubespray Project is released on an as-needed basis. The process is as follo
 5. Create the release note with [Kubernetes Release Notes Generator](https://github.com/kubernetes/release/blob/master/cmd/release-notes/README.md). See the following `Release note creation` section for the details.
 6. An approver creates [new release in GitHub](https://github.com/kubernetes-sigs/kubespray/releases/new) using a version and tag name like `vX.Y.Z` and attaching the release notes
 7. An approver creates a release branch in the form `release-X.Y`
-8. The corresponding version of [quay.io/kubespray/kubespray:vX.Y.Z](https://quay.io/repository/kubespray/kubespray) and [quay.io/kubespray/vagrant:vX.Y.Z](https://quay.io/repository/kubespray/vagrant) docker images are built and tagged
+8. The corresponding version of [quay.io/kubespray/kubespray:vX.Y.Z](https://quay.io/repository/kubespray/kubespray) and [quay.io/kubespray/vagrant:vX.Y.Z](https://quay.io/repository/kubespray/vagrant) container images are built and tagged. See the following `Container image creation` section for the details.
 9. The `KUBESPRAY_VERSION` variable is updated in `.gitlab-ci.yml`
 10. The release issue is closed
-11. An announcement email is sent to `kubernetes-dev@googlegroups.com` with the subject `[ANNOUNCE] Kubespray $VERSION is released`
+11. An announcement email is sent to `dev@kubernetes.io` with the subject `[ANNOUNCE] Kubespray $VERSION is released`
 12. The topic of the #kubespray channel is updated with `vX.Y.Z is released! | ...`
 
 ## Major/minor releases and milestones
@@ -61,3 +61,23 @@ release-notes --start-sha <The start commit-id> --end-sha <The end commit-id> --
 
 If the release note file(/tmp/kubespray-release-note) contains "### Uncategorized" pull requests, those pull requests don't have a valid kind label(`kind/feature`, etc.).
 It is necessary to put a valid label on each pull request and run the above release-notes command again to get a better release note)
+
+## Container image creation
+
+The container image `quay.io/kubespray/kubespray:vX.Y.Z` can be created from Dockerfile of the kubespray root directory:
+
+```shell
+cd kubespray/
+nerdctl build -t quay.io/kubespray/kubespray:vX.Y.Z .
+nerdctl push quay.io/kubespray/kubespray:vX.Y.Z
+```
+
+The container image `quay.io/kubespray/vagrant:vX.Y.Z` can be created from build.sh of test-infra/vagrant-docker/:
+
+```shell
+cd kubespray/test-infra/vagrant-docker/
+./build vX.Y.Z
+```
+
+Please note that the above operation requires the permission to push container images into quay.io/kubespray/.
+If you don't have the permission, please ask it on the #kubespray-dev channel.

SECURITY_CONTACTS

@@ -9,5 +9,7 @@
 #
 # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE
 # INSTRUCTIONS AT https://kubernetes.io/security/
-atoms
 mattymo
+floryut
+oomichi
+cristicalin

Vagrantfile

@@ -28,9 +28,10 @@ SUPPORTED_OS = {
   "centos8-bento"       => {box: "bento/centos-8",             user: "vagrant"},
   "almalinux8"          => {box: "almalinux/8",                user: "vagrant"},
   "almalinux8-bento"    => {box: "bento/almalinux-8",          user: "vagrant"},
-  "fedora34"            => {box: "fedora/34-cloud-base",       user: "vagrant"},
+  "rockylinux8"         => {box: "generic/rocky8",             user: "vagrant"},
   "fedora35"            => {box: "fedora/35-cloud-base",       user: "vagrant"},
-  "opensuse"            => {box: "opensuse/Leap-15.3.x86_64",  user: "vagrant"},
+  "fedora36"            => {box: "fedora/36-cloud-base",       user: "vagrant"},
+  "opensuse"            => {box: "opensuse/Leap-15.4.x86_64",  user: "vagrant"},
   "opensuse-tumbleweed" => {box: "opensuse/Tumbleweed.x86_64", user: "vagrant"},
   "oraclelinux"         => {box: "generic/oracle7",            user: "vagrant"},
   "oraclelinux8"        => {box: "generic/oracle8",            user: "vagrant"},

ansible.cfg

@@ -1,6 +1,6 @@
 [ssh_connection]
 pipelining=True
-ssh_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100 -o UserKnownHostsFile=/dev/null
+ansible_ssh_args = -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=100 -o UserKnownHostsFile=/dev/null
 #control_path = ~/.ssh/ansible-%%r@%%h:%%p
 [defaults]
 # https://github.com/ansible/ansible/issues/56930 (to ignore group names with - and .)
@@ -10,11 +10,11 @@ host_key_checking=False
 gathering = smart
 fact_caching = jsonfile
 fact_caching_connection = /tmp
-fact_caching_timeout = 7200
+fact_caching_timeout = 86400
 stdout_callback = default
 display_skipped_hosts = no
 library = ./library
-callback_whitelist = profile_tasks,ara_default
+callbacks_enabled = profile_tasks,ara_default
 roles_path = roles:$VIRTUAL_ENV/usr/local/share/kubespray/roles:$VIRTUAL_ENV/usr/local/share/ansible/roles:/usr/share/kubespray/roles
 deprecation_warnings=False
 inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo, .creds, .gpg

ansible_version.yml

@@ -3,32 +3,20 @@
   gather_facts: false
   become: no
   vars:
-    minimal_ansible_version: 2.9.0
-    minimal_ansible_version_2_10: 2.10.11
+    minimal_ansible_version: 2.11.0
     maximal_ansible_version: 2.13.0
     ansible_connection: local
   tags: always
   tasks:
     - name: "Check {{ minimal_ansible_version }} <= Ansible version < {{ maximal_ansible_version }}"
       assert:
-        msg: "Ansible must be between {{ minimal_ansible_version }} and {{ maximal_ansible_version }}"
+        msg: "Ansible must be between {{ minimal_ansible_version }} and {{ maximal_ansible_version }} exclusive"
         that:
           - ansible_version.string is version(minimal_ansible_version, ">=")
           - ansible_version.string is version(maximal_ansible_version, "<")
       tags:
         - check
 
-    - name: "Check Ansible version > {{ minimal_ansible_version_2_10 }} when using ansible 2.10"
-      assert:
-        msg: "When using Ansible 2.10, the minimum supported version is {{ minimal_ansible_version_2_10 }}"
-        that:
-          - ansible_version.string is version(minimal_ansible_version_2_10, ">=")
-          - ansible_version.string is version(maximal_ansible_version, "<")
-      when:
-        - ansible_version.string is version('2.10.0', ">=")
-      tags:
-        - check
-
     - name: "Check that python netaddr is installed"
       assert:
         msg: "Python netaddr is not present"

cluster.yml

@@ -35,7 +35,7 @@
     - { role: "container-engine", tags: "container-engine", when: deploy_container_engine }
     - { role: download, tags: download, when: "not skip_downloads" }
 
-- hosts: etcd
+- hosts: etcd:kube_control_plane
   gather_facts: False
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
@@ -59,7 +59,10 @@
       vars:
         etcd_cluster_setup: false
         etcd_events_cluster_setup: false
-      when: etcd_deployment_type != "kubeadm"
+      when:
+        - etcd_deployment_type != "kubeadm"
+        - kube_network_plugin in ["calico", "flannel", "canal", "cilium"] or cilium_deploy_additionally | default(false) | bool
+        - kube_network_plugin != "calico" or calico_datastore == "etcd"
 
 - hosts: k8s_cluster
   gather_facts: False

contrib/dind/run-test-distros.sh

@@ -17,7 +17,7 @@ pass_or_fail() {
 test_distro() {
     local distro=${1:?};shift
     local extra="${*:-}"
-    local prefix="$distro[${extra}]}"
+    local prefix="${distro[${extra}]}"
     ansible-playbook -i hosts dind-cluster.yaml -e node_distro=$distro
     pass_or_fail "$prefix: dind-nodes" || return 1
     (cd ../..
@@ -71,15 +71,15 @@ for spec in ${SPECS}; do
     echo "Loading file=${spec} ..."
     . ${spec} || continue
     : ${DISTROS:?} || continue
-    echo "DISTROS=${DISTROS[@]}"
+    echo "DISTROS:" "${DISTROS[@]}"
     echo "EXTRAS->"
     printf "  %s\n" "${EXTRAS[@]}"
     let n=1
-    for distro in ${DISTROS[@]}; do
+    for distro in "${DISTROS[@]}"; do
         for extra in "${EXTRAS[@]:-NULL}"; do
             # Magic value to let this for run once:
             [[ ${extra} == NULL ]] && unset extra
-            docker rm -f ${NODES[@]}
+            docker rm -f "${NODES[@]}"
             printf -v file_out "%s/%s-%02d.out" ${OUTPUT_DIR} ${spec} $((n++))
             {
                 info "${distro}[${extra}] START: file_out=${file_out}"

contrib/network-storage/glusterfs/roles/glusterfs/README.md

@@ -14,12 +14,16 @@ This role performs basic installation and setup of Gluster, but it does not conf
 
 Available variables are listed below, along with default values (see `defaults/main.yml`):
 
-    glusterfs_default_release: ""
+```yaml
+glusterfs_default_release: ""
+```
 
 You can specify a `default_release` for apt on Debian/Ubuntu by overriding this variable. This is helpful if you need a different package or version for the main GlusterFS packages (e.g. GlusterFS 3.5.x instead of 3.2.x with the `wheezy-backports` default release on Debian Wheezy).
 
-    glusterfs_ppa_use: yes
-    glusterfs_ppa_version: "3.5"
+```yaml
+glusterfs_ppa_use: yes
+glusterfs_ppa_version: "3.5"
+```
 
 For Ubuntu, specify whether to use the official Gluster PPA, and which version of the PPA to use. See Gluster's [Getting Started Guide](https://docs.gluster.org/en/latest/Quick-Start-Guide/Quickstart/) for more info.
 
@@ -29,9 +33,11 @@ None.
 
 ## Example Playbook
 
+```yaml
     - hosts: server
       roles:
         - geerlingguy.glusterfs
+```
 
 For a real-world use example, read through [Simple GlusterFS Setup with Ansible](http://www.jeffgeerling.com/blog/simple-glusterfs-setup-ansible), a blog post by this role's author, which is included in Chapter 8 of [Ansible for DevOps](https://www.ansiblefordevops.com/).
 

contrib/offline/README.md

@@ -45,3 +45,21 @@ temp
 
 In some cases you may want to update some component version, you can declare version variables in ansible inventory file or group_vars,
 then run `./generate_list.sh -i [inventory_file]` to update file.list and images.list.
+
+## manage-offline-files.sh
+
+This script will download all files according to `temp/files.list` and run nginx container to provide offline file download.
+
+Step(1) generate `files.list`
+
+```shell
+./generate_list.sh
+```
+
+Step(2) download files and run nginx container
+
+```shell
+./manage-offline-files.sh
+```
+
+when nginx container is running, it can be accessed through <http://127.0.0.1:8080/>.

contrib/offline/manage-offline-container-images.sh

@@ -15,7 +15,7 @@ function create_container_image_tar() {
 	IMAGES=$(kubectl describe pods --all-namespaces | grep " Image:" | awk '{print $2}' | sort | uniq)
 	# NOTE: etcd and pause cannot be seen as pods.
 	# The pause image is used for --pod-infra-container-image option of kubelet.
-	EXT_IMAGES=$(kubectl cluster-info dump | egrep "quay.io/coreos/etcd:|k8s.gcr.io/pause:" | sed s@\"@@g)
+	EXT_IMAGES=$(kubectl cluster-info dump | egrep "quay.io/coreos/etcd:|registry.k8s.io/pause:" | sed s@\"@@g)
 	IMAGES="${IMAGES} ${EXT_IMAGES}"
 
 	rm -f  ${IMAGE_TAR_FILE}
@@ -46,12 +46,12 @@ function create_container_image_tar() {
 
 		# NOTE: Here removes the following repo parts from each image
 		# so that these parts will be replaced with Kubespray.
-		# - kube_image_repo: "k8s.gcr.io"
+		# - kube_image_repo: "registry.k8s.io"
 		# - gcr_image_repo: "gcr.io"
 		# - docker_image_repo: "docker.io"
 		# - quay_image_repo: "quay.io"
 		FIRST_PART=$(echo ${image} | awk -F"/" '{print $1}')
-		if [ "${FIRST_PART}" = "k8s.gcr.io" ] ||
+		if [ "${FIRST_PART}" = "registry.k8s.io" ] ||
 		   [ "${FIRST_PART}" = "gcr.io" ] ||
 		   [ "${FIRST_PART}" = "docker.io" ] ||
 		   [ "${FIRST_PART}" = "quay.io" ] ||

contrib/offline/manage-offline-files.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+CURRENT_DIR=$( dirname "$(readlink -f "$0")" )
+OFFLINE_FILES_DIR_NAME="offline-files"
+OFFLINE_FILES_DIR="${CURRENT_DIR}/${OFFLINE_FILES_DIR_NAME}"
+OFFLINE_FILES_ARCHIVE="${CURRENT_DIR}/offline-files.tar.gz"
+FILES_LIST=${FILES_LIST:-"${CURRENT_DIR}/temp/files.list"}
+NGINX_PORT=8080
+
+# download files
+if [ ! -f "${FILES_LIST}" ]; then
+    echo "${FILES_LIST} should exist, run ./generate_list.sh first."
+    exit 1
+fi
+
+rm -rf "${OFFLINE_FILES_DIR}"
+rm "${OFFLINE_FILES_ARCHIVE}"
+mkdir  "${OFFLINE_FILES_DIR}"
+
+wget -x -P "${OFFLINE_FILES_DIR}" -i "${FILES_LIST}"
+tar -czvf "${OFFLINE_FILES_ARCHIVE}"  "${OFFLINE_FILES_DIR_NAME}"
+
+[ -n "$NO_HTTP_SERVER" ] && echo "skip to run nginx" && exit 0
+
+# run nginx container server
+if command -v nerdctl 1>/dev/null 2>&1; then
+    runtime="nerdctl"
+elif command -v podman 1>/dev/null 2>&1; then
+    runtime="podman"
+elif command -v docker 1>/dev/null 2>&1; then
+    runtime="docker"
+else
+    echo "No supported container runtime found"
+    exit 1
+fi
+
+sudo "${runtime}" container inspect nginx >/dev/null 2>&1
+if [ $? -ne 0 ]; then
+    sudo "${runtime}" run \
+        --restart=always -d -p ${NGINX_PORT}:80 \
+        --volume "${OFFLINE_FILES_DIR}:/usr/share/nginx/html/download" \
+        --volume "$(pwd)"/nginx.conf:/etc/nginx/nginx.conf \
+        --name nginx nginx:alpine
+fi

contrib/offline/nginx.conf

@@ -0,0 +1,39 @@
+user nginx;
+worker_processes auto;
+error_log /var/log/nginx/error.log;
+pid /run/nginx.pid;
+include /usr/share/nginx/modules/*.conf;
+events {
+    worker_connections 1024;
+}
+http {
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+    access_log  /var/log/nginx/access.log  main;
+    sendfile            on;
+    tcp_nopush          on;
+    tcp_nodelay         on;
+    keepalive_timeout   65;
+    types_hash_max_size 2048;
+    default_type        application/octet-stream;
+    include /etc/nginx/conf.d/*.conf;
+    server {
+        listen       80 default_server;
+        listen       [::]:80 default_server;
+        server_name  _;
+        include /etc/nginx/default.d/*.conf;
+        location / {
+            root    /usr/share/nginx/html/download;
+        autoindex on;
+        autoindex_exact_size off;
+        autoindex_localtime on;
+        }
+        error_page 404 /404.html;
+            location = /40x.html {
+        }
+        error_page 500 502 503 504 /50x.html;
+            location = /50x.html {
+        }
+    }
+}

contrib/terraform/aws/README.md

@@ -36,8 +36,7 @@ terraform apply -var-file=credentials.tfvars
 ```
 
 - Terraform automatically creates an Ansible Inventory file called `hosts` with the created infrastructure in the directory `inventory`
-- Ansible will automatically generate an ssh config file for your bastion hosts. To connect to hosts with ssh using bastion host use generated ssh-bastion.conf.
-  Ansible automatically detects bastion and changes ssh_args  
+- Ansible will automatically generate an ssh config file for your bastion hosts. To connect to hosts with ssh using bastion host use generated `ssh-bastion.conf`. Ansible automatically detects bastion and changes `ssh_args`
 
 ```commandline
 ssh -F ./ssh-bastion.conf user@$ip

contrib/terraform/exoscale/README.md

@@ -31,9 +31,7 @@ The setup looks like following
 
 ## Requirements
 
-* Terraform 0.13.0 or newer
-
-*0.12 also works if you modify the provider block to include version and remove all `versions.tf` files*
+* Terraform 0.13.0 or newer (0.12 also works if you modify the provider block to include version and remove all `versions.tf` files)
 
 ## Quickstart
 

contrib/terraform/openstack/README.md

@@ -294,7 +294,8 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
 Allows a custom definition of worker nodes giving the operator full control over individual node flavor and
 availability zone placement. To enable the use of this mode set the `number_of_k8s_nodes` and
 `number_of_k8s_nodes_no_floating_ip` variables to 0. Then define your desired worker node configuration
-using the `k8s_nodes` variable.
+using the `k8s_nodes` variable. The `az`, `flavor` and `floating_ip` parameters are mandatory.
+The optional parameter `extra_groups` (a comma-delimited string) can be used to define extra inventory group memberships for specific nodes.
 
 For example:
 
@@ -314,6 +315,7 @@ k8s_nodes = {
     "az" = "sto3"
     "flavor" = "83d8b44a-26a0-4f02-a981-079446926445"
     "floating_ip" = true
+    "extra_groups" = "calico_rr"
   }
 }
 ```

contrib/terraform/openstack/kubespray.tf

@@ -96,6 +96,10 @@ module "compute" {
   network_router_id                            = module.network.router_id
   network_id                                   = module.network.network_id
   use_existing_network                         = var.use_existing_network
+
+  depends_on = [
+    module.network.subnet_id
+  ]
 }
 
 output "private_subnet_id" {
@@ -111,7 +115,7 @@ output "router_id" {
 }
 
 output "k8s_master_fips" {
-  value = concat(module.ips.k8s_master_fips, module.ips.k8s_master_no_etcd_fips)
+  value = var.number_of_k8s_masters + var.number_of_k8s_masters_no_etcd > 0 ? concat(module.ips.k8s_master_fips, module.ips.k8s_master_no_etcd_fips) : [for key, value in module.ips.k8s_masters_fips : value.address]
 }
 
 output "k8s_node_fips" {

contrib/terraform/openstack/modules/compute/main.tf

@@ -742,7 +742,7 @@ resource "openstack_compute_instance_v2" "k8s_nodes" {
 
   metadata = {
     ssh_user         = var.ssh_user
-    kubespray_groups = "kube_node,k8s_cluster,%{if each.value.floating_ip == false}no_floating,%{endif}${var.supplementary_node_groups}"
+    kubespray_groups = "kube_node,k8s_cluster,%{if each.value.floating_ip == false}no_floating,%{endif}${var.supplementary_node_groups},${try(each.value.extra_groups, "")}"
     depends_on       = var.network_router_id
     use_access_ip    = var.use_access_ip
   }

contrib/terraform/terraform.py

@@ -212,7 +212,7 @@ def metal_device(resource, tfvars=None):
         'project_id': raw_attrs['project_id'],
         'state': raw_attrs['state'],
         # ansible
-        'ansible_ssh_host': raw_attrs['network.0.address'],
+        'ansible_host': raw_attrs['network.0.address'],
         'ansible_ssh_user': 'root',  # Use root by default in metal
         # generic
         'ipv4_address': raw_attrs['network.0.address'],
@@ -292,16 +292,16 @@ def openstack_host(resource, module_name):
     try:
         if 'metadata.prefer_ipv6' in raw_attrs and raw_attrs['metadata.prefer_ipv6'] == "1":
             attrs.update({
-                'ansible_ssh_host': re.sub("[\[\]]", "", raw_attrs['access_ip_v6']),
+                'ansible_host': re.sub("[\[\]]", "", raw_attrs['access_ip_v6']),
                 'publicly_routable': True,
             })
         else:
             attrs.update({
-                'ansible_ssh_host': raw_attrs['access_ip_v4'],
+                'ansible_host': raw_attrs['access_ip_v4'],
                 'publicly_routable': True,
             })
     except (KeyError, ValueError):
-        attrs.update({'ansible_ssh_host': '', 'publicly_routable': False})
+        attrs.update({'ansible_host': '', 'publicly_routable': False})
 
     # Handling of floating IPs has changed: https://github.com/terraform-providers/terraform-provider-openstack/blob/master/CHANGELOG.md#010-june-21-2017
 
@@ -349,7 +349,7 @@ def iter_host_ips(hosts, ips):
                 'access_ip_v4': ip,
                 'access_ip': ip,
                 'public_ipv4': ip,
-                'ansible_ssh_host': ip,
+                'ansible_host': ip,
             })
 
         if 'use_access_ip' in host[1]['metadata'] and host[1]['metadata']['use_access_ip'] == "0":
@@ -389,7 +389,7 @@ def query_list(hosts):
 def query_hostfile(hosts):
     out = ['## begin hosts generated by terraform.py ##']
     out.extend(
-        '{}\t{}'.format(attrs['ansible_ssh_host'].ljust(16), name)
+        '{}\t{}'.format(attrs['ansible_host'].ljust(16), name)
         for name, attrs, _ in hosts
     )
 

contrib/terraform/upcloud/README.md

@@ -112,12 +112,26 @@ terraform destroy --var-file cluster-settings.tfvars \
     * `size`: The size of the additional disk in GB
     * `tier`: The tier of disk to use (`maxiops` is the only one you can choose atm)
 * `firewall_enabled`: Enable firewall rules
+* `firewall_default_deny_in`: Set the firewall to deny inbound traffic by default. Automatically adds UpCloud DNS server and NTP port allowlisting.
+* `firewall_default_deny_out`: Set the firewall to deny outbound traffic by default.
 * `master_allowed_remote_ips`: List of IP ranges that should be allowed to access API of masters
   * `start_address`: Start of address range to allow
   * `end_address`: End of address range to allow
 * `k8s_allowed_remote_ips`: List of IP ranges that should be allowed SSH access to all nodes
   * `start_address`: Start of address range to allow
   * `end_address`: End of address range to allow
+* `master_allowed_ports`: List of port ranges that should be allowed to access the masters
+  * `protocol`: Protocol *(tcp|udp|icmp)*
+  * `port_range_min`: Start of port range to allow
+  * `port_range_max`: End of port range to allow
+  * `start_address`: Start of address range to allow
+  * `end_address`: End of address range to allow
+* `worker_allowed_ports`: List of port ranges that should be allowed to access the workers
+  * `protocol`: Protocol *(tcp|udp|icmp)*
+  * `port_range_min`: Start of port range to allow
+  * `port_range_max`: End of port range to allow
+  * `start_address`: Start of address range to allow
+  * `end_address`: End of address range to allow
 * `loadbalancer_enabled`: Enable managed load balancer
 * `loadbalancer_plan`: Plan to use for load balancer *(development|production-small)*
 * `loadbalancers`: Ports to load balance and which machines to forward to. Key of this object will be used as the name of the load balancer frontends/backends

contrib/terraform/upcloud/cluster-settings.tfvars

@@ -95,7 +95,9 @@ machines = {
   }
 }
 
-firewall_enabled = false
+firewall_enabled          = false
+firewall_default_deny_in  = false
+firewall_default_deny_out = false
 
 master_allowed_remote_ips = [
   {
@@ -111,6 +113,9 @@ k8s_allowed_remote_ips = [
   }
 ]
 
+master_allowed_ports = []
+worker_allowed_ports = []
+
 loadbalancer_enabled = false
 loadbalancer_plan    = "development"
 loadbalancers = {

contrib/terraform/upcloud/main.tf

@@ -24,8 +24,12 @@ module "kubernetes" {
   ssh_public_keys = var.ssh_public_keys
 
   firewall_enabled          = var.firewall_enabled
+  firewall_default_deny_in  = var.firewall_default_deny_in
+  firewall_default_deny_out = var.firewall_default_deny_out
   master_allowed_remote_ips = var.master_allowed_remote_ips
   k8s_allowed_remote_ips    = var.k8s_allowed_remote_ips
+  master_allowed_ports      = var.master_allowed_ports
+  worker_allowed_ports      = var.worker_allowed_ports
 
   loadbalancer_enabled = var.loadbalancer_enabled
   loadbalancer_plan    = var.loadbalancer_plan

contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf

@@ -228,6 +228,112 @@ resource "upcloud_firewall_rules" "master" {
       source_address_start   = "0.0.0.0"
     }
   }
+
+  dynamic firewall_rule {
+    for_each = var.master_allowed_ports
+
+    content {
+      action                 = "accept"
+      comment                = "Allow access on this port"
+      destination_port_end   = firewall_rule.value.port_range_max
+      destination_port_start = firewall_rule.value.port_range_min
+      direction              = "in"
+      family                 = "IPv4"
+      protocol               = firewall_rule.value.protocol
+      source_address_end     = firewall_rule.value.end_address
+      source_address_start   = firewall_rule.value.start_address
+    }
+  }
+
+  dynamic firewall_rule {
+    for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : []
+
+    content {
+      action                 = "accept"
+      comment                = "UpCloud DNS"
+      destination_port_end   = "53"
+      destination_port_start = "53"
+      direction              = "in"
+      family                 = "IPv4"
+      protocol               = firewall_rule.value
+      source_address_end     = "94.237.40.9"
+      source_address_start   = "94.237.40.9"
+    }
+  }
+
+  dynamic firewall_rule {
+    for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : []
+
+    content {
+      action                 = "accept"
+      comment                = "UpCloud DNS"
+      destination_port_end   = "53"
+      destination_port_start = "53"
+      direction              = "in"
+      family                 = "IPv4"
+      protocol               = firewall_rule.value
+      source_address_end     = "94.237.127.9"
+      source_address_start   = "94.237.127.9"
+    }
+  }
+
+  dynamic firewall_rule {
+    for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : []
+
+    content {
+      action                 = "accept"
+      comment                = "UpCloud DNS"
+      destination_port_end   = "53"
+      destination_port_start = "53"
+      direction              = "in"
+      family                 = "IPv6"
+      protocol               = firewall_rule.value
+      source_address_end     = "2a04:3540:53::1"
+      source_address_start   = "2a04:3540:53::1"
+    }
+  }
+
+  dynamic firewall_rule {
+    for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : []
+
+    content {
+      action                 = "accept"
+      comment                = "UpCloud DNS"
+      destination_port_end   = "53"
+      destination_port_start = "53"
+      direction              = "in"
+      family                 = "IPv6"
+      protocol               = firewall_rule.value
+      source_address_end     = "2a04:3544:53::1"
+      source_address_start   = "2a04:3544:53::1"
+    }
+  }
+
+  dynamic firewall_rule {
+    for_each = var.firewall_default_deny_in ? ["udp"] : []
+
+    content {
+      action                 = "accept"
+      comment                = "NTP Port"
+      destination_port_end   = "123"
+      destination_port_start = "123"
+      direction              = "in"
+      family                 = "IPv4"
+      protocol               = firewall_rule.value
+      source_address_end     = "255.255.255.255"
+      source_address_start   = "0.0.0.0"
+    }
+  }
+
+  firewall_rule {
+    action    = var.firewall_default_deny_in ? "drop" : "accept"
+    direction = "in"
+  }
+
+  firewall_rule {
+    action    = var.firewall_default_deny_out ? "drop" : "accept"
+    direction = "out"
+  }
 }
 
 resource "upcloud_firewall_rules" "k8s" {
@@ -265,6 +371,112 @@ resource "upcloud_firewall_rules" "k8s" {
       source_address_start   = "0.0.0.0"
     }
   }
+
+  dynamic firewall_rule {
+    for_each = var.worker_allowed_ports
+
+    content {
+      action                 = "accept"
+      comment                = "Allow access on this port"
+      destination_port_end   = firewall_rule.value.port_range_max
+      destination_port_start = firewall_rule.value.port_range_min
+      direction              = "in"
+      family                 = "IPv4"
+      protocol               = firewall_rule.value.protocol
+      source_address_end     = firewall_rule.value.end_address
+      source_address_start   = firewall_rule.value.start_address
+    }
+  }
+
+  dynamic firewall_rule {
+    for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : []
+
+    content {
+      action                 = "accept"
+      comment                = "UpCloud DNS"
+      destination_port_end   = "53"
+      destination_port_start = "53"
+      direction              = "in"
+      family                 = "IPv4"
+      protocol               = firewall_rule.value
+      source_address_end     = "94.237.40.9"
+      source_address_start   = "94.237.40.9"
+    }
+  }
+
+  dynamic firewall_rule {
+    for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : []
+
+    content {
+      action                 = "accept"
+      comment                = "UpCloud DNS"
+      destination_port_end   = "53"
+      destination_port_start = "53"
+      direction              = "in"
+      family                 = "IPv4"
+      protocol               = firewall_rule.value
+      source_address_end     = "94.237.127.9"
+      source_address_start   = "94.237.127.9"
+    }
+  }
+
+  dynamic firewall_rule {
+    for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : []
+
+    content {
+      action                 = "accept"
+      comment                = "UpCloud DNS"
+      destination_port_end   = "53"
+      destination_port_start = "53"
+      direction              = "in"
+      family                 = "IPv6"
+      protocol               = firewall_rule.value
+      source_address_end     = "2a04:3540:53::1"
+      source_address_start   = "2a04:3540:53::1"
+    }
+  }
+
+  dynamic firewall_rule {
+    for_each = var.firewall_default_deny_in ? ["tcp", "udp"] : []
+
+    content {
+      action                 = "accept"
+      comment                = "UpCloud DNS"
+      destination_port_end   = "53"
+      destination_port_start = "53"
+      direction              = "in"
+      family                 = "IPv6"
+      protocol               = firewall_rule.value
+      source_address_end     = "2a04:3544:53::1"
+      source_address_start   = "2a04:3544:53::1"
+    }
+  }
+
+  dynamic firewall_rule {
+    for_each = var.firewall_default_deny_in ? ["udp"] : []
+
+    content {
+      action                 = "accept"
+      comment                = "NTP Port"
+      destination_port_end   = "123"
+      destination_port_start = "123"
+      direction              = "in"
+      family                 = "IPv4"
+      protocol               = firewall_rule.value
+      source_address_end     = "255.255.255.255"
+      source_address_start   = "0.0.0.0"
+    }
+  }
+
+  firewall_rule {
+    action    = var.firewall_default_deny_in ? "drop" : "accept"
+    direction = "in"
+  }
+
+  firewall_rule {
+    action    = var.firewall_default_deny_out ? "drop" : "accept"
+    direction = "out"
+  }
 }
 
 resource "upcloud_loadbalancer" "lb" {

contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf

@@ -49,6 +49,34 @@ variable "k8s_allowed_remote_ips" {
   }))
 }
 
+variable "master_allowed_ports" {
+  type = list(object({
+    protocol       = string
+    port_range_min = number
+    port_range_max = number
+    start_address  = string
+    end_address    = string
+  }))
+}
+
+variable "worker_allowed_ports" {
+  type = list(object({
+    protocol       = string
+    port_range_min = number
+    port_range_max = number
+    start_address  = string
+    end_address    = string
+  }))
+}
+
+variable "firewall_default_deny_in" {
+  type = bool
+}
+
+variable "firewall_default_deny_out" {
+  type = bool
+}
+
 variable "loadbalancer_enabled" {
   type = bool
 }

contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf

@@ -3,7 +3,7 @@ terraform {
   required_providers {
     upcloud = {
       source = "UpCloudLtd/upcloud"
-      version = "~>2.4.0"
+      version = "~>2.5.0"
     }
   }
   required_version = ">= 0.13"

contrib/terraform/upcloud/sample-inventory/cluster.tfvars

@@ -95,7 +95,10 @@ machines = {
   }
 }
 
-firewall_enabled = false
+firewall_enabled          = false
+firewall_default_deny_in  = false
+firewall_default_deny_out = false
+
 
 master_allowed_remote_ips = [
   {
@@ -111,6 +114,9 @@ k8s_allowed_remote_ips = [
   }
 ]
 
+master_allowed_ports = []
+worker_allowed_ports = []
+
 loadbalancer_enabled = false
 loadbalancer_plan = "development"
 loadbalancers = {

contrib/terraform/upcloud/variables.tf

@@ -79,6 +79,38 @@ variable "k8s_allowed_remote_ips" {
   default = []
 }
 
+variable "master_allowed_ports" {
+  description = "List of ports to allow on masters"
+  type = list(object({
+    protocol       = string
+    port_range_min = number
+    port_range_max = number
+    start_address  = string
+    end_address    = string
+  }))
+}
+
+variable "worker_allowed_ports" {
+  description = "List of ports to allow on workers"
+  type = list(object({
+    protocol       = string
+    port_range_min = number
+    port_range_max = number
+    start_address  = string
+    end_address    = string
+  }))
+}
+
+variable "firewall_default_deny_in" {
+  description = "Add firewall policies that deny all inbound traffic by default"
+  default     = false
+}
+
+variable "firewall_default_deny_out" {
+  description = "Add firewall policies that deny all outbound traffic by default"
+  default     = false
+}
+
 variable "loadbalancer_enabled" {
   description = "Enable load balancer"
   default     = false

contrib/terraform/upcloud/versions.tf

@@ -3,7 +3,7 @@ terraform {
   required_providers {
     upcloud = {
       source  = "UpCloudLtd/upcloud"
-      version = "~>2.4.0"
+      version = "~>2.5.0"
     }
   }
   required_version = ">= 0.13"

contrib/terraform/vsphere/README.md

@@ -35,9 +35,7 @@ This setup assumes that the DHCP is disabled in the vSphere cluster and IP addre
 
 ## Requirements
 
-* Terraform 0.13.0 or newer
-
-*0.12 also works if you modify the provider block to include version and remove all `versions.tf` files*
+* Terraform 0.13.0 or newer (0.12 also works if you modify the provider block to include version and remove all `versions.tf` files)
 
 ## Quickstart
 

docs/_sidebar.md

@@ -35,6 +35,7 @@
   * [OpenSUSE](docs/opensuse.md)
   * [RedHat Enterprise Linux](docs/rhel.md)
   * [CentOS/OracleLinux/AlmaLinux/Rocky Linux](docs/centos.md)
+  * [Kylin Linux Advanced Server V10](docs/kylinlinux.md)
   * [Amazon Linux 2](docs/amazonlinux.md)
 * CRI
   * [Containerd](docs/containerd.md)
@@ -50,6 +51,7 @@
   * [DNS Stack](docs/dns-stack.md)
   * [Kubernetes reliability](docs/kubernetes-reliability.md)
   * [Local Registry](docs/kubernetes-apps/registry.md)
+  * [NTP](docs/ntp.md)
 * External Storage Provisioners
   * [RBD Provisioner](docs/kubernetes-apps/rbd_provisioner.md)
   * [CEPHFS Provisioner](docs/kubernetes-apps/cephfs_provisioner.md)

docs/ansible.md

@@ -3,7 +3,7 @@
 ## Installing Ansible
 
 Kubespray supports multiple ansible versions and ships different `requirements.txt` files for them.
-Depending on your available python version you may be limited in chooding which ansible version to use.
+Depending on your available python version you may be limited in choosing which ansible version to use.
 
 It is recommended to deploy the ansible version used by kubespray into a python virtual environment.
 
@@ -13,7 +13,7 @@ KUBESPRAYDIR=kubespray
 ANSIBLE_VERSION=2.12
 virtualenv  --python=$(which python3) $VENVDIR
 source $VENVDIR/bin/activate
-cd $KUESPRAYDIR
+cd $KUBESPRAYDIR
 pip install -U -r requirements-$ANSIBLE_VERSION.txt
 test -f requirements-$ANSIBLE_VERSION.yml && \
   ansible-galaxy role install -r requirements-$ANSIBLE_VERSION.yml && \
@@ -26,8 +26,6 @@ Based on the table below and the available python version for your ansible host
 
 | Ansible Version | Python Version |
 | --------------- | -------------- |
-| 2.9             | 2.7,3.5-3.8    |
-| 2.10            | 2.7,3.5-3.8    |
 | 2.11            | 2.7,3.5-3.9    |
 | 2.12            | 3.8-3.10       |
 

docs/aws-ebs-csi.md

@@ -27,7 +27,7 @@ Check the associated storage class (if you enabled persistent_volumes):
 ```ShellSession
 $ kubectl get storageclass
 NAME         PROVISIONER                AGE
-ebs-sc   ebs.csi.aws.com   45s
+ebs-sc       ebs.csi.aws.com            45s
 ```
 
 You can run a PVC and an example Pod using this file `ebs-pod.yml`:
@@ -71,8 +71,8 @@ You should see the PVC provisioned and bound:
 
 ```ShellSession
 $ kubectl get pvc
-NAME                   STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
-ebs-claim   Bound    pvc-0034cb9e-1ddd-4b3f-bb9e-0b5edbf5194c   1Gi        RWO            ebs-sc         50s
+NAME          STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
+ebs-claim     Bound    pvc-0034cb9e-1ddd-4b3f-bb9e-0b5edbf5194c   1Gi        RWO            ebs-sc         50s
 ```
 
 And the volume mounted to the example Pod (wait until the Pod is Running):

docs/azure-csi.md

@@ -57,19 +57,28 @@ The name of the network security group your instances are in, can be retrieved v
 These will have to be generated first:
 
 - Create an Azure AD Application with:
-`az ad app create --display-name kubespray --identifier-uris http://kubespray --homepage http://kubespray.com --password CLIENT_SECRET`
+
+  ```ShellSession
+  az ad app create --display-name kubespray --identifier-uris http://kubespray --homepage http://kubespray.com --password CLIENT_SECRET
+  ```
 
 Display name, identifier-uri, homepage and the password can be chosen
 
 Note the AppId in the output.
 
 - Create Service principal for the application with:
-`az ad sp create --id AppId`
+
+  ```ShellSession
+  az ad sp create --id AppId
+  ```
 
 This is the AppId from the last command
 
 - Create the role assignment with:
-`az role assignment create --role "Owner" --assignee http://kubespray --subscription SUBSCRIPTION_ID`
+
+  ```ShellSession
+  az role assignment create --role "Owner" --assignee http://kubespray --subscription SUBSCRIPTION_ID
+  ```
 
 azure\_csi\_aad\_client\_id must be set to the AppId, azure\_csi\_aad\_client\_secret is your chosen secret.
 

docs/azure.md

@@ -71,14 +71,27 @@ The name of the resource group that contains the route table.  Defaults to `azur
 These will have to be generated first:
 
 - Create an Azure AD Application with:
-`az ad app create --display-name kubernetes --identifier-uris http://kubernetes --homepage http://example.com --password CLIENT_SECRET`
+
+  ```ShellSession
+   az ad app create --display-name kubernetes --identifier-uris http://kubernetes --homepage http://example.com --password CLIENT_SECRET
+  ```
+
 display name, identifier-uri, homepage and the password can be chosen
 Note the AppId in the output.
+
 - Create Service principal for the application with:
-`az ad sp create --id AppId`
+
+  ```ShellSession
+  az ad sp create --id AppId
+  ```
+
 This is the AppId from the last command
+
 - Create the role assignment with:
-`az role assignment create --role "Owner" --assignee http://kubernetes --subscription SUBSCRIPTION_ID`
+
+  ```ShellSession
+  az role assignment create --role "Owner" --assignee http://kubernetes --subscription SUBSCRIPTION_ID
+  ```
 
 azure\_aad\_client\_id must be set to the AppId, azure\_aad\_client\_secret is your chosen secret.
 

docs/bootstrap-os.md

@@ -48,11 +48,13 @@ The `kubespray-defaults` role is expected to be run before this role.
 
 Remember to disable fact gathering since Python might not be present on hosts.
 
-    - hosts: all
-      gather_facts: false  # not all hosts might be able to run modules yet
-      roles:
-         - kubespray-defaults
-         - bootstrap-os
+```yaml
+- hosts: all
+  gather_facts: false  # not all hosts might be able to run modules yet
+  roles:
+    - kubespray-defaults
+    - bootstrap-os
+```
 
 ## License
 

docs/calico.md

@@ -72,9 +72,14 @@ calico_pool_cidr_ipv6: fd85:ee78:d8a6:8607::1:0000/112
 
 In some cases you may want to route the pods subnet and so NAT is not needed on the nodes.
 For instance if you have a cluster spread on different locations and you want your pods to talk each other no matter where they are located.
-The following variables need to be set:
-`peer_with_router` to enable the peering with the datacenter's border router (default value: false).
-you'll need to edit the inventory and add a hostvar `local_as` by node.
+The following variables need to be set as follow:
+
+```yml
+peer_with_router: true  # enable the peering with the datacenter's border router (default value: false).
+nat_outgoing: false  # (optional) NAT outgoing (default value: true).
+```
+
+And you'll need to edit the inventory and add a hostvar `local_as` by node.
 
 ```ShellSession
 node1 ansible_ssh_host=95.54.0.12 local_as=xxxxxx
@@ -124,8 +129,7 @@ You need to edit your inventory and add:
 * `calico_rr` group with nodes in it. `calico_rr` can be combined with
   `kube_node` and/or `kube_control_plane`. `calico_rr` group also must be a child
    group of `k8s_cluster` group.
-* `cluster_id` by route reflector node/group (see details
-[here](https://hub.docker.com/r/calico/routereflector/))
+* `cluster_id` by route reflector node/group (see details [here](https://hub.docker.com/r/calico/routereflector/))
 
 Here's an example of Kubespray inventory with standalone route reflectors:
 
@@ -172,6 +176,8 @@ node5
 
 [rack0:vars]
 cluster_id="1.0.0.1"
+calico_rr_id=rr1
+calico_group_id=rr1
 ```
 
 The inventory above will deploy the following topology assuming that calico's
@@ -212,7 +218,7 @@ calico_node_readinessprobe_timeout: 10
 
 Calico supports two types of encapsulation: [VXLAN and IP in IP](https://docs.projectcalico.org/v3.11/networking/vxlan-ipip). VXLAN is the more mature implementation and enabled by default, please check your environment if you need *IP in IP* encapsulation.
 
-*IP in IP* and *VXLAN* is mutualy exclusive modes.
+*IP in IP* and *VXLAN* is mutually exclusive modes.
 
 Kubespray defaults have changed after version 2.18 from auto-enabling `ipip` mode to auto-enabling `vxlan`. This was done to facilitate wider deployment scenarios including those where vxlan acceleration is provided by the underlying network devices.
 
@@ -245,14 +251,14 @@ calico_network_backend: 'bird'
 
 If you would like to migrate from the old IP in IP with `bird` network backends default to the new VXLAN based encapsulation you need to perform this change before running an upgrade of your cluster; the `cluster.yml` and `upgrade-cluster.yml` playbooks will refuse to continue if they detect incompatible settings.
 
-Execute the following sters on one of the control plane nodes, ensure the cluster in healthy before proceeding.
+Execute the following steps on one of the control plane nodes, ensure the cluster in healthy before proceeding.
 
 ```shell
 calicoctl.sh patch felixconfig default -p '{"spec":{"vxlanEnabled":true}}'
 calicoctl.sh patch ippool default-pool -p '{"spec":{"ipipMode":"Never", "vxlanMode":"Always"}}'
 ```
 
-**Note:** if you created multiple ippools you will need to patch all of them individually to change their encapsulation. The kubespray playbooks only handle the default ippool creaded by kubespray.
+**Note:** if you created multiple ippools you will need to patch all of them individually to change their encapsulation. The kubespray playbooks only handle the default ippool created by kubespray.
 
 Wait for the `vxlan.calico` interfaces to be created on all cluster nodes and traffic to be routed through it then you can disable `ipip`.
 
@@ -315,6 +321,13 @@ calico_ipam_host_local: true
 
 Refer to Project Calico section [Using host-local IPAM](https://docs.projectcalico.org/reference/cni-plugin/configuration#using-host-local-ipam) for further information.
 
+### Optional : Disable CNI logging to disk
+
+Calico CNI plugin logs to /var/log/calico/cni/cni.log and to stderr.
+stderr of CNI plugins can be found in the logs of container runtime.
+
+You can disable Calico CNI logging to disk by setting `calico_cni_log_file_path: false`.
+
 ## eBPF Support
 
 Calico supports eBPF for its data plane see [an introduction to the Calico eBPF Dataplane](https://www.projectcalico.org/introducing-the-calico-ebpf-dataplane/) for further information.
@@ -362,7 +375,7 @@ use_localhost_as_kubeapi_loadbalancer: true
 
 ### Tunneled versus Direct Server Return
 
-By default Calico usese Tunneled service mode but it can use direct server return (DSR) in order to optimize the return path for a service.
+By default Calico uses Tunneled service mode but it can use direct server return (DSR) in order to optimize the return path for a service.
 
 To configure DSR:
 
@@ -388,7 +401,7 @@ Please see [Calico eBPF troubleshooting guide](https://docs.projectcalico.org/ma
 
 ## Wireguard Encryption
 
-Calico supports using Wireguard for encryption. Please see the docs on [encryptiong cluster pod traffic](https://docs.projectcalico.org/security/encrypt-cluster-pod-traffic).
+Calico supports using Wireguard for encryption. Please see the docs on [encrypt cluster pod traffic](https://docs.projectcalico.org/security/encrypt-cluster-pod-traffic).
 
 To enable wireguard support:
 

docs/centos.md

@@ -7,9 +7,9 @@ Kubespray supports multiple ansible versions but only the default (5.x) gets wid
 
 ## CentOS 8
 
-CentOS 8 / Oracle Linux 8 / AlmaLinux 8 / Rocky Linux 8 ship only with iptables-nft (ie without iptables-legacy similar to RHEL8)
+CentOS 8 / Oracle Linux 8,9 / AlmaLinux 8,9 / Rocky Linux 8,9 ship only with iptables-nft (ie without iptables-legacy similar to RHEL8)
 The only tested configuration for now is using Calico CNI
-You need to add `calico_iptables_backend: "NFT"` or `calico_iptables_backend: "Auto"` to your configuration.
+You need to add `calico_iptables_backend: "NFT"` to your configuration.
 
 If you have containers that are using iptables in the host network namespace (`hostNetwork=true`),
 you need to ensure they are using iptables-nft.

docs/ci.md

@@ -8,17 +8,19 @@ To generate this Matrix run `./tests/scripts/md-table/main.py`
 |---| --- | --- | --- | --- | --- | --- | --- | --- |
 almalinux8 |  :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: |
 amazon |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-centos7 |  :white_check_mark: | :x: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: |
+centos7 |  :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: |
 debian10 |  :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian9 |  :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
-fedora34 |  :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: |
 fedora35 |  :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
+fedora36 |  :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: |
 opensuse |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-oracle7 |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+rockylinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+rockylinux9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu16 |  :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: |
 ubuntu18 |  :white_check_mark: | :x: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: |
 ubuntu20 |  :white_check_mark: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
+ubuntu22 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 
 ## crio
 
@@ -30,13 +32,15 @@ centos7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian10 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora34 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora35 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora36 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 opensuse |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-oracle7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+rockylinux9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu16 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu18 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu20 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu22 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 
 ## docker
 
@@ -44,14 +48,16 @@ ubuntu20 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 |---| --- | --- | --- | --- | --- | --- | --- | --- |
 almalinux8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-centos7 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
+centos7 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian10 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian11 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora34 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
-fedora35 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora35 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora36 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
 opensuse |  :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
-oracle7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+rockylinux8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+rockylinux9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu16 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
 ubuntu18 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu20 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu22 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |

docs/cilium.md

@@ -56,7 +56,7 @@ cilium_operator_extra_volume_mounts:
 ## Choose Cilium version
 
 ```yml
-cilium_version: v1.11.3
+cilium_version: v1.12.1
 ```
 
 ## Add variable to config
@@ -153,3 +153,32 @@ cilium_hubble_metrics:
 ```
 
 [More](https://docs.cilium.io/en/v1.9/operations/metrics/#hubble-exported-metrics)
+
+## Upgrade considerations
+
+### Rolling-restart timeouts
+
+Cilium relies on the kernel's BPF support, which is extremely fast at runtime but incurs a compilation penalty on initialization and update.
+
+As a result, the Cilium DaemonSet pods can take a significant time to start, which scales with the number of nodes and endpoints in your cluster.
+
+As part of cluster.yml, this DaemonSet is restarted, and Kubespray's [default timeouts for this operation](../roles/network_plugin/cilium/defaults/main.yml)
+are not appropriate for large clusters.
+
+This means that you will likely want to update these timeouts to a value more in-line with your cluster's number of nodes and their respective CPU performance.
+This is configured by the following values:
+
+```yaml
+# Configure how long to wait for the Cilium DaemonSet to be ready again
+cilium_rolling_restart_wait_retries_count: 30
+cilium_rolling_restart_wait_retries_delay_seconds: 10
+```
+
+The total time allowed (count * delay) should be at least `($number_of_nodes_in_cluster * $cilium_pod_start_time)` for successful rolling updates. There are no
+drawbacks to making it higher and giving yourself a time buffer to accommodate transient slowdowns.
+
+Note: To find the `$cilium_pod_start_time` for your cluster, you can simply restart a Cilium pod on a node of your choice and look at how long it takes for it
+to become ready.
+
+Note 2: The default CPU requests/limits for Cilium pods is set to a very conservative 100m:500m which will likely yield very slow startup for Cilium pods. You
+probably want to significantly increase the CPU limit specifically if short bursts of CPU from Cilium are acceptable to you.

docs/containerd.md

@@ -39,4 +39,65 @@ containerd_registries:
 image_command_tool: crictl
 ```
 
+### Containerd Runtimes
+
+Containerd supports multiple runtime configurations that can be used with
+[RuntimeClass] Kubernetes feature. See [runtime classes in containerd] for the
+details of containerd configuration.
+
+In kubespray, the default runtime name is "runc", and it can be configured with the `containerd_runc_runtime` dictionary:
+
+```yaml
+containerd_runc_runtime:
+  name: runc
+  type: "io.containerd.runc.v2"
+  engine: ""
+  root: ""
+  options:
+    systemdCgroup: "false"
+    binaryName: /usr/local/bin/my-runc
+  base_runtime_spec: cri-base.json
+```
+
+Further runtimes can be configured with `containerd_additional_runtimes`, which
+is a list of such dictionaries.
+
+Default runtime can be changed by setting `containerd_default_runtime`.
+
+#### base_runtime_spec
+
+`base_runtime_spec` key in a runtime dictionary can be used to explicitly
+specify a runtime spec json file. We ship the default one which is generated
+with `ctr oci spec > /etc/containerd/cri-base.json`. It will be used if you set
+`base_runtime_spec: cri-base.json`. The main advantage of doing so is the presence of
+`rlimits` section in this configuration, which will restrict the maximum number
+of file descriptors(open files) per container to 1024.
+
+You can tune many more [settings][runtime-spec] by supplying your own file name and content with `containerd_base_runtime_specs`:
+
+```yaml
+containerd_base_runtime_specs:
+  cri-spec-custom.json: |
+    {
+      "ociVersion": "1.0.2-dev",
+      "process": {
+        "user": {
+          "uid": 0,
+    ...
+```
+
+The files in this dict will be placed in containerd config directory,
+`/etc/containerd` by default. The files can then be referenced by filename in a
+runtime:
+
+```yaml
+containerd_runc_runtime:
+  name: runc
+  base_runtime_spec: cri-spec-custom.json
+  ...
+```
+
 [containerd]: https://containerd.io/
+[RuntimeClass]: https://kubernetes.io/docs/concepts/containers/runtime-class/
+[runtime classes in containerd]: https://github.com/containerd/containerd/blob/main/docs/cri/config.md#runtime-classes
+[runtime-spec]: https://github.com/opencontainers/runtime-spec

docs/debian.md

@@ -3,34 +3,39 @@
 Debian Jessie installation Notes:
 
 - Add
-  
-  ```GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1"```
-  
-  to /etc/default/grub. Then update with
-  
-  ```ShellSession
-   sudo update-grub
-   sudo update-grub2
-   sudo reboot
+
+  ```ini
+  GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1"
   ```
-  
+
+  to `/etc/default/grub`. Then update with
+
+  ```ShellSession
+  sudo update-grub
+  sudo update-grub2
+  sudo reboot
+  ```
+
 - Add the [backports](https://backports.debian.org/Instructions/) which contain Systemd 2.30 and update Systemd.
-  
-  ```apt-get -t jessie-backports install systemd```
-  
+
+  ```ShellSession
+  apt-get -t jessie-backports install systemd
+  ```
+
   (Necessary because the default Systemd version (2.15) does not support the "Delegate" directive in service files)
-  
+
 - Add the Ansible repository and install Ansible to get a proper version
 
   ```ShellSession
   sudo add-apt-repository ppa:ansible/ansible
   sudo apt-get update
   sudo apt-get install ansible
-
   ```
 
 - Install Jinja2 and Python-Netaddr
 
-  ```sudo apt-get install python-jinja2=2.8-1~bpo8+1 python-netaddr```
-  
+  ```ShellSession
+  sudo apt-get install python-jinja2=2.8-1~bpo8+1 python-netaddr
+  ```
+
 Now you can continue with [Preparing your deployment](getting-started.md#starting-custom-deployment)

docs/dns-stack.md

@@ -19,6 +19,14 @@ ndots value to be used in ``/etc/resolv.conf``
 It is important to note that multiple search domains combined with high ``ndots``
 values lead to poor performance of DNS stack, so please choose it wisely.
 
+## dns_timeout
+
+timeout value to be used in ``/etc/resolv.conf``
+
+## dns_attempts
+
+attempts value to be used in ``/etc/resolv.conf``
+
 ### searchdomains
 
 Custom search domains to be added in addition to the cluster search domains (``default.svc.{{ dns_domain }}, svc.{{ dns_domain }}``).
@@ -26,6 +34,8 @@ Custom search domains to be added in addition to the cluster search domains (``d
 Most Linux systems limit the total number of search domains to 6 and the total length of all search domains
 to 256 characters. Depending on the length of ``dns_domain``, you're limited to less than the total limit.
 
+`remove_default_searchdomains: true` will remove the default cluster search domains.
+
 Please note that ``resolvconf_mode: docker_dns`` will automatically add your systems search domains as
 additional search domains. Please take this into the accounts for the limits.
 
@@ -62,6 +72,13 @@ coredns_external_zones:
   nameservers:
   - 192.168.0.53
   cache: 0
+- zones:
+  - mydomain.tld
+  nameservers:
+  - 10.233.0.3
+  cache: 5
+  rewrite:
+  - name stop website.tld website.namespace.svc.cluster.local
 ```
 
 or as INI
@@ -263,7 +280,8 @@ nodelocaldns_secondary_skew_seconds: 5
 
 * the ``searchdomains`` have a limitation of a 6 names and 256 chars
   length. Due to default ``svc, default.svc`` subdomains, the actual
-  limits are a 4 names and 239 chars respectively.
+  limits are a 4 names and 239 chars respectively. If `remove_default_searchdomains: true`
+  added you are back to 6 names.
 
 * the ``nameservers`` have a limitation of a 3 servers, although there
   is a way to mitigate that with the ``upstream_dns_servers``,

docs/docker.md

@@ -8,13 +8,7 @@ Using the docker container manager:
 container_manager: docker
 ```
 
-Using `cri-dockerd` instead of `dockershim`:
-
-```yaml
-cri_dockerd_enabled: false
-```
-
-*Note:* The `cri_dockerd_enabled: true` setting will become the default in a future kubespray release once kubespray 1.24+ is supported and `dockershim` is removed. At that point, changing this option will be deprecated and silently ignored.
+*Note:* `cri-dockerd` has replaced `dockershim` across supported kubernetes version in kubespray 2.20.
 
 Enabling the `overlay2` graph driver:
 

docs/fcos.md

@@ -54,7 +54,7 @@ Prepare ignition and serve via http (a.e. python -m http.server )
 
 ### create guest
 
-```shell script
+```ShellSeasion
 machine_name=myfcos1
 ignition_url=http://mywebserver/fcos.ign
 

docs/gcp-lb.md

@@ -2,15 +2,19 @@
 
 Google Cloud Platform can be used for creation of Kubernetes Service Load Balancer.
 
-This feature is able to deliver by adding parameters to kube-controller-manager and kubelet. You need specify:
+This feature is able to deliver by adding parameters to `kube-controller-manager` and `kubelet`. You need specify:
 
+```ShellSession
     --cloud-provider=gce
     --cloud-config=/etc/kubernetes/cloud-config
+```
 
-To get working it in kubespray, you need to add tag to GCE instances and specify it in kubespray group vars and also set cloud_provider to gce. So for example, in file group_vars/all/gcp.yml:
+To get working it in kubespray, you need to add tag to GCE instances and specify it in kubespray group vars and also set `cloud_provider` to `gce`. So for example, in file `group_vars/all/gcp.yml`:
 
+```yaml
     cloud_provider: gce
     gce_node_tags: k8s-lb
+```
 
-When you will setup it and create SVC in Kubernetes with type=LoadBalancer, cloud provider will create public IP and will set firewall.
+When you will setup it and create SVC in Kubernetes with `type=LoadBalancer`, cloud provider will create public IP and will set firewall.
 Note: Cloud provider run under VM service account, so this account needs to have correct permissions to be able to create all GCP resources.

docs/ha-mode.md

@@ -36,12 +36,6 @@ The following diagram shows how traffic to the apiserver is directed.
 
 ![Image](figures/loadbalancer_localhost.png?raw=true)
 
-  Note: Kubernetes master nodes still use insecure localhost access because
-  there are bugs in Kubernetes <1.5.0 in using TLS auth on master role
-  services. This makes backends receiving unencrypted traffic and may be a
-  security issue when interconnecting different nodes, or maybe not, if those
-  belong to the isolated management network without external access.
-
 A user may opt to use an external loadbalancer (LB) instead. An external LB
 provides access for external clients, while the internal LB accepts client
 connections only to the localhost.
@@ -129,11 +123,6 @@ Kubespray has nothing to do with it, this is informational only.
 As you can see, the masters' internal API endpoints are always
 contacted via the local bind IP, which is `https://bip:sp`.
 
-**Note** that for some cases, like healthchecks of applications deployed by
-Kubespray, the masters' APIs are accessed via the insecure endpoint, which
-consists of the local `kube_apiserver_insecure_bind_address` and
-`kube_apiserver_insecure_port`.
-
 ## Optional configurations
 
 ### ETCD with a LB

docs/hardening.md

@@ -74,7 +74,6 @@ kube_kubeadm_scheduler_extra_args:
 etcd_deployment_type: kubeadm
 
 ## kubelet
-kubelet_authorization_mode_webhook: true
 kubelet_authentication_token_webhook: true
 kube_read_only_port: 0
 kubelet_rotate_server_certificates: true
@@ -83,7 +82,24 @@ kubelet_event_record_qps: 1
 kubelet_rotate_certificates: true
 kubelet_streaming_connection_idle_timeout: "5m"
 kubelet_make_iptables_util_chains: true
-kubelet_feature_gates: ["RotateKubeletServerCertificate=true"]
+kubelet_feature_gates: ["RotateKubeletServerCertificate=true","SeccompDefault=true"]
+kubelet_seccomp_default: true
+kubelet_systemd_hardening: true
+# In case you have multiple interfaces in your
+# control plane nodes and you want to specify the right
+# IP addresses, kubelet_secure_addresses allows you
+# to specify the IP from which the kubelet
+# will receive the packets.
+kubelet_secure_addresses: "192.168.10.110 192.168.10.111 192.168.10.112"
+
+# additional configurations
+kube_owner: root
+kube_cert_group: root
+
+# create a default Pod Security Configuration and deny running of insecure pods
+# kube_system namespace is exempted by default
+kube_pod_security_use_default: true
+kube_pod_security_default_enforce: restricted
 ```
 
 Let's take a deep look to the resultant **kubernetes** configuration:
@@ -93,6 +109,8 @@ Let's take a deep look to the resultant **kubernetes** configuration:
 * The `encryption-provider-config` provide encryption at rest. This means that the `kube-apiserver` encrypt data that is going to be stored before they reach `etcd`. So the data is completely unreadable from `etcd` (in case an attacker is able to exploit this).
 * The `rotateCertificates` in `KubeletConfiguration` is set to `true` along with `serverTLSBootstrap`. This could be used in alternative to `tlsCertFile` and `tlsPrivateKeyFile` parameters. Additionally it automatically generates certificates by itself, but you need to manually approve them or at least using an operator to do this (for more details, please take a look here: <https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/>).
 * If you are installing **kubernetes** in an AppArmor-based OS (eg. Debian/Ubuntu) you can enable the `AppArmor` feature gate uncommenting the lines with the comment `# AppArmor-based OS` on top.
+* The `kubelet_systemd_hardening`, both with `kubelet_secure_addresses` setup a minimal firewall on the system. To better understand how these variables work, here's an explanatory image:
+  ![kubelet hardening](img/kubelet-hardening.png)
 
 Once you have the file properly filled, you can run the **Ansible** command to start the installation:
 

docs/integration.md

@@ -6,84 +6,100 @@
      * List of all forked repos could be retrieved from github page of original project.
 
 2. Add **forked repo** as submodule to desired folder in your existent ansible repo (for example 3d/kubespray):
-  ```git submodule add https://github.com/YOUR_GITHUB/kubespray.git kubespray```
-  Git will create `.gitmodules` file in your existent ansible repo:
+
+   ```ShellSession
+   git submodule add https://github.com/YOUR_GITHUB/kubespray.git kubespray
+   ```
+
+   Git will create `.gitmodules` file in your existent ansible repo:
 
    ```ini
    [submodule "3d/kubespray"]
-         path = 3d/kubespray
-         url = https://github.com/YOUR_GITHUB/kubespray.git
+          path = 3d/kubespray
+          url = https://github.com/YOUR_GITHUB/kubespray.git
    ```
 
 3. Configure git to show submodule status:
-```git config --global status.submoduleSummary true```
+
+   ```ShellSession
+   git config --global status.submoduleSummary true
+   ```
 
 4. Add *original* kubespray repo as upstream:
-```cd kubespray && git remote add upstream https://github.com/kubernetes-sigs/kubespray.git```
+
+   ```ShellSession
+   cd kubespray && git remote add upstream https://github.com/kubernetes-sigs/kubespray.git
+   ```
 
 5. Sync your master branch with upstream:
 
    ```ShellSession
-      git checkout master
-      git fetch upstream
-      git merge upstream/master
-      git push origin master
+   git checkout master
+   git fetch upstream
+   git merge upstream/master
+   git push origin master
    ```
 
 6. Create a new branch which you will use in your working environment:
-```git checkout -b work```
+
+   ```ShellSession
+   git checkout -b work
+   ```
+
     ***Never*** use master branch of your repository for your commits.
 
 7. Modify path to library and roles in your ansible.cfg file (role naming should be unique, you may have to rename your existent roles if they have same names as kubespray project),
    if you had roles in your existing ansible project before, you can add the path to those separated with `:`:
 
-8. ```ini
+   ```ini
    ...
    library       = ./library/:3d/kubespray/library/
    roles_path    = ./roles/:3d/kubespray/roles/
    ...
    ```
 
-9. Copy and modify configs from kubespray `group_vars` folder to corresponding `group_vars` folder in your existent project.
-You could rename *all.yml* config to something else, i.e. *kubespray.yml* and create corresponding group in your inventory file, which will include all hosts groups related to kubernetes setup.
+8. Copy and modify configs from kubespray `group_vars` folder to corresponding `group_vars` folder in your existent project.
 
-10. Modify your ansible inventory file by adding mapping of your existent groups (if any) to kubespray naming.
+   You could rename *all.yml* config to something else, i.e. *kubespray.yml* and create corresponding group in your inventory file, which will include all hosts groups related to kubernetes setup.
+
+9. Modify your ansible inventory file by adding mapping of your existent groups (if any) to kubespray naming.
     For example:
 
-    ```ini
-      ...
-      #Kargo groups:
-      [kube_node:children]
-      kubenode
+   ```ini
+   ...
+   #Kubespray groups:
+   [kube_node:children]
+   kubenode
 
-      [k8s_cluster:children]
-      kubernetes
+   [k8s_cluster:children]
+   kubernetes
 
-      [etcd:children]
-      kubemaster
-      kubemaster-ha
+   [etcd:children]
+   kubemaster
+   kubemaster-ha
 
-      [kube_control_plane:children]
-      kubemaster
-      kubemaster-ha
+   [kube_control_plane:children]
+   kubemaster
+   kubemaster-ha
 
-      [kubespray:children]
-      kubernetes
-      ```
+   [kubespray:children]
+   kubernetes
+   ```
 
-      * Last entry here needed to apply kubespray.yml config file, renamed from all.yml of kubespray project.
+* Last entry here needed to apply kubespray.yml config file, renamed from all.yml of kubespray project.
 
-11. Now you can include kubespray tasks in you existent playbooks by including cluster.yml file:
+10. Now you can include kubespray tasks in you existent playbooks by including cluster.yml file:
 
-     ```yml
-     - name: Include kubespray tasks
-       include: 3d/kubespray/cluster.yml
-     ```
+    ```yml
+    - name: Import kubespray playbook
+      ansible.builtin.import_playbook: 3d/kubespray/cluster.yml
+    ```
 
-     Or your could copy separate tasks from cluster.yml into your ansible repository.
+    Or your could copy separate tasks from cluster.yml into your ansible repository.
 
-12. Commit changes to your ansible repo. Keep in mind, that submodule folder is just a link to the git commit hash of your forked repo.
-When you update your "work" branch you need to commit changes to ansible repo as well.
+11. Commit changes to your ansible repo. Keep in mind, that submodule folder is just a link to the git commit hash of your forked repo.
+
+    When you update your "work" branch you need to commit changes to ansible repo as well.
 Other members of your team should use ```git submodule sync```, ```git submodule update --init``` to get actual code from submodule.
 
 ## Contributing
@@ -95,37 +111,78 @@ If you made useful changes or fixed a bug in existent kubespray repo, use this f
 2. Change working directory to git submodule directory (3d/kubespray).
 
 3. Setup desired user.name and user.email for submodule.
-If kubespray is only one submodule in your repo you could use something like:
-```git submodule foreach --recursive 'git config user.name "First Last" && git config user.email "your-email-address@used.for.cncf"'```
+
+   If kubespray is only one submodule in your repo you could use something like:
+
+   ```ShellSession
+   git submodule foreach --recursive 'git config user.name "First Last" && git config user.email "your-email-address@used.for.cncf"'
+   ```
 
 4. Sync with upstream master:
 
    ```ShellSession
-    git fetch upstream
-    git merge upstream/master
-    git push origin master
-     ```
+   git fetch upstream
+   git merge upstream/master
+   git push origin master
+   ```
 
 5. Create new branch for the specific fixes that you want to contribute:
-```git checkout -b fixes-name-date-index```
-Branch name should be self explaining to you, adding date and/or index will help you to track/delete your old PRs.
+
+   ```ShellSession
+   git checkout -b fixes-name-date-index
+   ```
+
+   Branch name should be self explaining to you, adding date and/or index will help you to track/delete your old PRs.
 
 6. Find git hash of your commit in "work" repo and apply it to newly created "fix" repo:
 
-     ```ShellSession
-     git cherry-pick <COMMIT_HASH>
-     ```
+   ```ShellSession
+   git cherry-pick <COMMIT_HASH>
+   ```
 
-7. If you have several temporary-stage commits - squash them using [```git rebase -i```](https://eli.thegreenplace.net/2014/02/19/squashing-github-pull-requests-into-a-single-commit)
-Also you could use interactive rebase (```git rebase -i HEAD~10```) to delete commits which you don't want to contribute into original repo.
+7. If you have several temporary-stage commits - squash them using [git rebase -i](https://eli.thegreenplace.net/2014/02/19/squashing-github-pull-requests-into-a-single-commit)
+
+   Also you could use interactive rebase
+
+   ```ShellSession
+   git rebase -i HEAD~10
+   ```
+
+   to delete commits which you don't want to contribute into original repo.
 
 8. When your changes is in place, you need to check upstream repo one more time because it could be changed during your work.
-Check that you're on correct branch:
-```git status```
-And pull changes from upstream (if any):
-```git pull --rebase upstream master```
 
-9. Now push your changes to your **fork** repo with ```git push```. If your branch doesn't exists on github, git will propose you to use something like ```git push --set-upstream origin fixes-name-date-index```.
+   Check that you're on correct branch:
 
-10. Open you forked repo in browser, on the main page you will see proposition to create pull request for your newly created branch. Check proposed diff of your PR. If something is wrong you could safely delete "fix" branch on github using ```git push origin --delete fixes-name-date-index```, ```git branch -D fixes-name-date-index``` and start whole process from the beginning.
-If everything is fine - add description about your changes (what they do and why they're needed) and confirm pull request creation.
+   ```ShellSession
+   git status
+   ```
+
+   And pull changes from upstream (if any):
+
+   ```ShellSession
+   git pull --rebase upstream master
+   ```
+
+9. Now push your changes to your **fork** repo with
+
+   ```ShellSession
+   git push
+   ```
+
+   If your branch doesn't exists on github, git will propose you to use something like
+
+   ```ShellSession
+   git push --set-upstream origin fixes-name-date-index
+   ```
+
+10. Open you forked repo in browser, on the main page you will see proposition to create pull request for your newly created branch. Check proposed diff of your PR. If something is wrong you could safely delete "fix" branch on github using
+
+    ```ShellSession
+    git push origin --delete fixes-name-date-index
+    git branch -D fixes-name-date-index
+    ```
+
+    and start whole process from the beginning.
+
+    If everything is fine - add description about your changes (what they do and why they're needed) and confirm pull request creation.

docs/kube-vip.md

@@ -2,6 +2,14 @@
 
 kube-vip provides Kubernetes clusters with a virtual IP and load balancer for both the control plane (for building a highly-available cluster) and Kubernetes Services of type LoadBalancer without relying on any external hardware or software.
 
+## Prerequisites
+
+You have to configure `kube_proxy_strict_arp` when the kube_proxy_mode is `ipvs` and kube-vip ARP is enabled.
+
+```yaml
+kube_proxy_strict_arp: true
+```
+
 ## Install
 
 You have to explicitly enable the kube-vip extension:
@@ -11,7 +19,7 @@ kube_vip_enabled: true
 ```
 
 You also need to enable
-[kube-vip as HA, Load Balancer, or both](https://kube-vip.chipzoller.dev/docs/installation/static/#kube-vip-as-ha-load-balancer-or-both):
+[kube-vip as HA, Load Balancer, or both](https://kube-vip.io/docs/installation/static/#kube-vip-as-ha-load-balancer-or-both):
 
 ```yaml
 # HA for control-plane, requires a VIP
@@ -28,16 +36,16 @@ kube_vip_services_enabled: false
 ```
 
 > Note: When using `kube-vip` as LoadBalancer for services,
-[additionnal manual steps](https://kube-vip.chipzoller.dev/docs/usage/cloud-provider/)
+[additional manual steps](https://kube-vip.io/docs/usage/cloud-provider/)
 are needed.
 
-If using [ARP mode](https://kube-vip.chipzoller.dev/docs/installation/static/#arp) :
+If using [ARP mode](https://kube-vip.io/docs/installation/static/#arp) :
 
 ```yaml
 kube_vip_arp_enabled: true
 ```
 
-If using [BGP mode](https://kube-vip.chipzoller.dev/docs/installation/static/#bgp) :
+If using [BGP mode](https://kube-vip.io/docs/installation/static/#bgp) :
 
 ```yaml
 kube_vip_bgp_enabled: true

docs/kubernetes-apps/registry.md

@@ -29,8 +29,7 @@ use Kubernetes's `PersistentVolume` abstraction. The following template is
 expanded by `salt` in the GCE cluster turnup, but can easily be adapted to
 other situations:
 
-<!-- BEGIN MUNGE: EXAMPLE registry-pv.yaml.in -->
-``` yaml
+```yaml
 kind: PersistentVolume
 apiVersion: v1
 metadata:
@@ -46,7 +45,6 @@ spec:
     fsType: "ext4"
 {% endif %}
 ```
-<!-- END MUNGE: EXAMPLE registry-pv.yaml.in -->
 
 If, for example, you wanted to use NFS you would just need to change the
 `gcePersistentDisk` block to `nfs`. See
@@ -68,8 +66,7 @@ Now that the Kubernetes cluster knows that some storage exists, you can put a
 claim on that storage. As with the `PersistentVolume` above, you can start
 with the `salt` template:
 
-<!-- BEGIN MUNGE: EXAMPLE registry-pvc.yaml.in -->
-``` yaml
+```yaml
 kind: PersistentVolumeClaim
 apiVersion: v1
 metadata:
@@ -82,7 +79,6 @@ spec:
     requests:
       storage: {{ pillar['cluster_registry_disk_size'] }}
 ```
-<!-- END MUNGE: EXAMPLE registry-pvc.yaml.in -->
 
 This tells Kubernetes that you want to use storage, and the `PersistentVolume`
 you created before will be bound to this claim (unless you have other
@@ -93,8 +89,7 @@ gives you the right to use this storage until you release the claim.
 
 Now we can run a Docker registry:
 
-<!-- BEGIN MUNGE: EXAMPLE registry-rc.yaml -->
-``` yaml
+```yaml
 apiVersion: v1
 kind: ReplicationController
 metadata:
@@ -138,7 +133,6 @@ spec:
         persistentVolumeClaim:
           claimName: kube-registry-pvc
 ```
-<!-- END MUNGE: EXAMPLE registry-rc.yaml -->
 
 *Note:* that if you have set multiple replicas, make sure your CSI driver has support for the `ReadWriteMany` accessMode.
 
@@ -146,8 +140,7 @@ spec:
 
 Now that we have a registry `Pod` running, we can expose it as a Service:
 
-<!-- BEGIN MUNGE: EXAMPLE registry-svc.yaml -->
-``` yaml
+```yaml
 apiVersion: v1
 kind: Service
 metadata:
@@ -164,7 +157,6 @@ spec:
     port: 5000
     protocol: TCP
 ```
-<!-- END MUNGE: EXAMPLE registry-svc.yaml -->
 
 ## Expose the registry on each node
 
@@ -172,8 +164,7 @@ Now that we have a running `Service`, we need to expose it onto each Kubernetes
 `Node` so that Docker will see it as `localhost`. We can load a `Pod` on every
 node by creating following daemonset.
 
-<!-- BEGIN MUNGE: EXAMPLE ../../saltbase/salt/kube-registry-proxy/kube-registry-proxy.yaml -->
-``` yaml
+```yaml
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -207,7 +198,6 @@ spec:
           containerPort: 80
           hostPort: 5000
 ```
-<!-- END MUNGE: EXAMPLE ../../saltbase/salt/kube-registry-proxy/kube-registry-proxy.yaml -->
 
 When modifying replication-controller, service and daemon-set definitions, take
 care to ensure *unique* identifiers for the rc-svc couple and the daemon-set.
@@ -219,7 +209,7 @@ This ensures that port 5000 on each node is directed to the registry `Service`.
 You should be able to verify that it is running by hitting port 5000 with a web
 browser and getting a 404 error:
 
-``` console
+```ShellSession
 $ curl localhost:5000
 404 page not found
 ```
@@ -229,7 +219,7 @@ $ curl localhost:5000
 To use an image hosted by this registry, simply say this in your `Pod`'s
 `spec.containers[].image` field:
 
-``` yaml
+```yaml
     image: localhost:5000/user/container
 ```
 
@@ -241,7 +231,7 @@ building locally and want to push to your cluster.
 You can use `kubectl` to set up a port-forward from your local node to a
 running Pod:
 
-``` console
+```ShellSession
 $ POD=$(kubectl get pods --namespace kube-system -l k8s-app=registry \
             -o template --template '{{range .items}}{{.metadata.name}} {{.status.phase}}{{"\n"}}{{end}}' \
             | grep Running | head -1 | cut -f1 -d' ')

docs/kylinlinux.md

@@ -0,0 +1,11 @@
+# Kylin Linux
+
+Kylin Linux is supported with docker and containerd runtimes.
+
+**Note:** that Kylin Linux is not currently covered in kubespray CI and
+support for it is currently considered experimental.
+
+At present, only `Kylin Linux Advanced Server V10 (Sword)` has been adapted, which can support the deployment of aarch64 and x86_64 platforms.
+
+There are no special considerations for using Kylin Linux as the target OS
+for Kubespray deployments.

docs/metallb.md

@@ -2,7 +2,7 @@
 
 MetalLB hooks into your Kubernetes cluster, and provides a network load-balancer implementation.
 It allows you to create Kubernetes services of type "LoadBalancer" in clusters that don't run on a cloud provider, and thus cannot simply hook into 3rd party products to provide load-balancers.
-The default operationg mode of MetalLB is in ["Layer2"](https://metallb.universe.tf/concepts/layer2/) but it can also operate in ["BGP"](https://metallb.universe.tf/concepts/bgp/) mode.
+The default operating mode of MetalLB is in ["Layer2"](https://metallb.universe.tf/concepts/layer2/) but it can also operate in ["BGP"](https://metallb.universe.tf/concepts/bgp/) mode.
 
 ## Prerequisites
 
@@ -19,6 +19,7 @@ You have to explicitly enable the MetalLB extension and set an IP address range
 ```yaml
 metallb_enabled: true
 metallb_speaker_enabled: true
+metallb_avoid_buggy_ips: true
 metallb_ip_range:
   - 10.5.0.0/16
 ```
@@ -69,16 +70,17 @@ metallb_peers:
 
 When using calico >= 3.18 you can replace MetalLB speaker by calico Service LoadBalancer IP advertisement.
 See [calico service IPs advertisement documentation](https://docs.projectcalico.org/archive/v3.18/networking/advertise-service-ips#advertise-service-load-balancer-ip-addresses).
-In this scenarion you should disable the MetalLB speaker and configure the `calico_advertise_service_loadbalancer_ips` to match your `metallb_ip_range`
+In this scenario you should disable the MetalLB speaker and configure the `calico_advertise_service_loadbalancer_ips` to match your `metallb_ip_range`
 
 ```yaml
 metallb_speaker_enabled: false
+metallb_avoid_buggy_ips: true
 metallb_ip_range:
   - 10.5.0.0/16
 calico_advertise_service_loadbalancer_ips: "{{ metallb_ip_range }}"
 ```
 
-If you have additional loadbalancer IP pool in `metallb_additional_address_pools`, ensure to add them to the list.
+If you have additional loadbalancer IP pool in `metallb_additional_address_pools` , ensure to add them to the list.
 
 ```yaml
 metallb_speaker_enabled: false
@@ -90,11 +92,13 @@ metallb_additional_address_pools:
       - 10.6.0.0/16
     protocol: "bgp"
     auto_assign: false
+    avoid_buggy_ips: true
   kube_service_pool_2:
     ip_range:
       - 10.10.0.0/16
     protocol: "bgp"
     auto_assign: false
+    avoid_buggy_ips: true
 calico_advertise_service_loadbalancer_ips:
   - 10.5.0.0/16
   - 10.6.0.0/16

docs/nodes.md

@@ -124,7 +124,7 @@ to
 With the old node still in the inventory, run `remove-node.yml`. You need to pass `-e node=node-1` to the playbook to limit the execution to the node being removed.
 If the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars.
 
-### 3) Edit cluster-info configmap in kube-system namespace
+### 3) Edit cluster-info configmap in kube-public namespace
 
 `kubectl  edit cm -n kube-public cluster-info`
 

docs/ntp.md

@@ -0,0 +1,50 @@
+# NTP synchronization
+
+The Network Time Protocol (NTP) is a networking protocol for clock synchronization between computer systems. Time synchronization is important to Kubernetes and Etcd.
+
+## Enable the NTP
+
+To start the ntpd(or chrony) service and enable it at system boot. There are related specific variables:
+
+```ShellSession
+ntp_enabled: true
+```
+
+The NTP service would be enabled and sync time automatically.
+
+## Customize the NTP configure file
+
+In the Air-Gap environment, the node cannot access the NTP server by internet. So the node can use the customized ntp server by configuring ntp file.
+
+```ShellSession
+ntp_enabled: true
+ntp_manage_config: true
+ntp_servers:
+  - "0.your-ntp-server.org iburst"
+  - "1.your-ntp-server.org iburst"
+  - "2.your-ntp-server.org iburst"
+  - "3.your-ntp-server.org iburst"
+```
+
+## Setting the TimeZone
+
+The timezone can also be set by the `ntp_timezone` , eg: "Etc/UTC","Asia/Shanghai". If not set, the timezone will not change.
+
+```ShellSession
+ntp_enabled: true
+ntp_timezone: Etc/UTC
+```
+
+## Advanced Configure
+
+Enable `tinker panic` is useful when running NTP in a VM environment to avoiding clock drift on VMs. It only takes effect when ntp_manage_config is true.
+
+```ShellSession
+ntp_tinker_panic: true
+```
+
+Force sync time immediately by NTP after the ntp installed, which is useful in newly installed system.
+
+```ShellSession
+ntp_force_sync_immediately: true
+```

docs/openstack.md

@@ -34,52 +34,6 @@ Otherwise [cinder](https://wiki.openstack.org/wiki/Cinder) won't work as expecte
 
 Unless you are using calico or kube-router you can now run the playbook.
 
-**Additional step needed when using calico or kube-router:**
-
-Being L3 CNI, calico and kube-router do not encapsulate all packages with the hosts' ip addresses. Instead the packets will be routed with the PODs ip addresses directly.
-
-OpenStack will filter and drop all packets from ips it does not know to prevent spoofing.
-
-In order to make L3 CNIs work on OpenStack you will need to tell OpenStack to allow pods packets by allowing the network they use.
-
-First you will need the ids of your OpenStack instances that will run kubernetes:
-
-  ```bash
-  openstack server list --project YOUR_PROJECT
-  +--------------------------------------+--------+----------------------------------+--------+-------------+
-  | ID                                   | Name   | Tenant ID                        | Status | Power State |
-  +--------------------------------------+--------+----------------------------------+--------+-------------+
-  | e1f48aad-df96-4bce-bf61-62ae12bf3f95 | k8s-1  | fba478440cb2444a9e5cf03717eb5d6f | ACTIVE | Running     |
-  | 725cd548-6ea3-426b-baaa-e7306d3c8052 | k8s-2  | fba478440cb2444a9e5cf03717eb5d6f | ACTIVE | Running     |
-  ```
-
-Then you can use the instance ids to find the connected [neutron](https://wiki.openstack.org/wiki/Neutron) ports (though they are now configured through using OpenStack):
-
-  ```bash
-  openstack port list -c id -c device_id --project YOUR_PROJECT
-  +--------------------------------------+--------------------------------------+
-  | id                                   | device_id                            |
-  +--------------------------------------+--------------------------------------+
-  | 5662a4e0-e646-47f0-bf88-d80fbd2d99ef | e1f48aad-df96-4bce-bf61-62ae12bf3f95 |
-  | e5ae2045-a1e1-4e99-9aac-4353889449a7 | 725cd548-6ea3-426b-baaa-e7306d3c8052 |
-  ```
-
-Given the port ids on the left, you can set the two `allowed-address`(es) in OpenStack. Note that you have to allow both `kube_service_addresses` (default `10.233.0.0/18`) and `kube_pods_subnet` (default `10.233.64.0/18`.)
-
-  ```bash
-  # allow kube_service_addresses and kube_pods_subnet network
-  openstack port set 5662a4e0-e646-47f0-bf88-d80fbd2d99ef --allowed-address ip-address=10.233.0.0/18 --allowed-address ip-address=10.233.64.0/18
-  openstack port set e5ae2045-a1e1-4e99-9aac-4353889449a7 --allowed-address ip-address=10.233.0.0/18 --allowed-address ip-address=10.233.64.0/18
-  ```
-
-If all the VMs in the tenant correspond to Kubespray deployment, you can "sweep run" above with:
-
-  ```bash
-  openstack port list --device-owner=compute:nova -c ID -f value | xargs -tI@ openstack port set @ --allowed-address ip-address=10.233.0.0/18 --allowed-address ip-address=10.233.64.0/18
-  ```
-
-Now you can finally run the playbook.
-
 ## The external cloud provider
 
 The in-tree cloud provider is deprecated and will be removed in a future version of Kubernetes. The target release for removing all remaining in-tree cloud providers is set to 1.21.
@@ -156,3 +110,49 @@ The new cloud provider is configured to have Octavia by default in Kubespray.
 
 - Run `source path/to/your/openstack-rc` to read your OpenStack credentials like `OS_AUTH_URL`, `OS_USERNAME`, `OS_PASSWORD`, etc. Those variables are used for accessing OpenStack from the external cloud provider.
 - Run the `cluster.yml` playbook
+
+## Additional step needed when using calico or kube-router
+
+Being L3 CNI, calico and kube-router do not encapsulate all packages with the hosts' ip addresses. Instead the packets will be routed with the PODs ip addresses directly.
+
+OpenStack will filter and drop all packets from ips it does not know to prevent spoofing.
+
+In order to make L3 CNIs work on OpenStack you will need to tell OpenStack to allow pods packets by allowing the network they use.
+
+First you will need the ids of your OpenStack instances that will run kubernetes:
+
+  ```bash
+  openstack server list --project YOUR_PROJECT
+  +--------------------------------------+--------+----------------------------------+--------+-------------+
+  | ID                                   | Name   | Tenant ID                        | Status | Power State |
+  +--------------------------------------+--------+----------------------------------+--------+-------------+
+  | e1f48aad-df96-4bce-bf61-62ae12bf3f95 | k8s-1  | fba478440cb2444a9e5cf03717eb5d6f | ACTIVE | Running     |
+  | 725cd548-6ea3-426b-baaa-e7306d3c8052 | k8s-2  | fba478440cb2444a9e5cf03717eb5d6f | ACTIVE | Running     |
+  ```
+
+Then you can use the instance ids to find the connected [neutron](https://wiki.openstack.org/wiki/Neutron) ports (though they are now configured through using OpenStack):
+
+  ```bash
+  openstack port list -c id -c device_id --project YOUR_PROJECT
+  +--------------------------------------+--------------------------------------+
+  | id                                   | device_id                            |
+  +--------------------------------------+--------------------------------------+
+  | 5662a4e0-e646-47f0-bf88-d80fbd2d99ef | e1f48aad-df96-4bce-bf61-62ae12bf3f95 |
+  | e5ae2045-a1e1-4e99-9aac-4353889449a7 | 725cd548-6ea3-426b-baaa-e7306d3c8052 |
+  ```
+
+Given the port ids on the left, you can set the two `allowed-address`(es) in OpenStack. Note that you have to allow both `kube_service_addresses` (default `10.233.0.0/18`) and `kube_pods_subnet` (default `10.233.64.0/18`.)
+
+  ```bash
+  # allow kube_service_addresses and kube_pods_subnet network
+  openstack port set 5662a4e0-e646-47f0-bf88-d80fbd2d99ef --allowed-address ip-address=10.233.0.0/18 --allowed-address ip-address=10.233.64.0/18
+  openstack port set e5ae2045-a1e1-4e99-9aac-4353889449a7 --allowed-address ip-address=10.233.0.0/18 --allowed-address ip-address=10.233.64.0/18
+  ```
+
+If all the VMs in the tenant correspond to Kubespray deployment, you can "sweep run" above with:
+
+  ```bash
+  openstack port list --device-owner=compute:nova -c ID -f value | xargs -tI@ openstack port set @ --allowed-address ip-address=10.233.0.0/18 --allowed-address ip-address=10.233.64.0/18
+  ```
+
+Now you can finally run the playbook.

docs/setting-up-your-first-cluster.md

@@ -252,11 +252,7 @@ Ansible will now execute the playbook, this can take up to 20 minutes.
 We will leverage a kubeconfig file from one of the controller nodes to access
  the cluster as administrator from our local workstation.
 
-> In this simplified set-up, we did not include a load balancer that usually
- sits on top of the
-three controller nodes for a high available API server endpoint. In this
- simplified tutorial we connect directly to one of the three
- controllers.
+> In this simplified set-up, we did not include a load balancer that usually sits on top of the three controller nodes for a high available API server endpoint. In this simplified tutorial we connect directly to one of the three controllers.
 
 First, we need to edit the permission of the kubeconfig file on one of the
 controller nodes:

docs/vagrant.md

@@ -58,7 +58,7 @@ see [download documentation](/docs/downloads.md).
 
 The following is an example of setting up and running kubespray using `vagrant`.
 For repeated runs, you could save the script to a file in the root of the
-kubespray and run it by executing 'source <name_of_the_file>.
+kubespray and run it by executing `source <name_of_the_file>`.
 
 ```ShellSession
 # use virtualenv to install all python requirements

docs/vars.md

@@ -15,7 +15,7 @@ Some variables of note include:
 
 * *calico_version* - Specify version of Calico to use
 * *calico_cni_version* - Specify version of Calico CNI plugin to use
-* *docker_version* - Specify version of Docker to used (should be quoted
+* *docker_version* - Specify version of Docker to use (should be quoted
   string). Must match one of the keys defined for *docker_versioned_pkg*
   in `roles/container-engine/docker/vars/*.yml`.
 * *containerd_version* - Specify version of containerd to use when setting `container_manager` to `containerd`
@@ -28,6 +28,7 @@ Some variables of note include:
 * *kube_proxy_mode* - Changes k8s proxy mode to iptables mode
 * *kube_version* - Specify a given Kubernetes version
 * *searchdomains* - Array of DNS domains to search when looking up hostnames
+* *remove_default_searchdomains* - Boolean that removes the default searchdomain
 * *nameservers* - Array of nameservers to use for DNS lookup
 * *preinstall_selinux_state* - Set selinux state, permitted values are permissive, enforcing and disabled.
 
@@ -81,7 +82,7 @@ following default cluster parameters:
   raise an assertion in playbooks if the `kubelet_max_pods` var also isn't adjusted accordingly
   (assertion not applicable to calico which doesn't use this as a hard limit, see
   [Calico IP block sizes](https://docs.projectcalico.org/reference/resources/ippool#block-sizes).
-  
+
 * *enable_dual_stack_networks* - Setting this to true will provision both IPv4 and IPv6 networking for pods and services.
 
 * *kube_service_addresses_ipv6* - Subnet for cluster IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1000/116``). Must not overlap with ``kube_pods_subnet_ipv6``.
@@ -99,7 +100,7 @@ following default cluster parameters:
 
 * *coredns_k8s_external_zone* - Zone that will be used when CoreDNS k8s_external plugin is enabled
   (default is k8s_external.local)
-  
+
 * *enable_coredns_k8s_endpoint_pod_names* - If enabled, it configures endpoint_pod_names option for kubernetes plugin.
   on the CoreDNS service.
 
@@ -166,6 +167,7 @@ variables to match your requirements.
   addition to Kubespray deployed DNS
 * *nameservers* - Array of DNS servers configured for use by hosts
 * *searchdomains* - Array of up to 4 search domains
+* *remove_default_searchdomains* - Boolean. If enabled, `searchdomains` variable can hold 6 search domains.
 * *dns_etchosts* - Content of hosts file for coredns and nodelocaldns
 
 For more information, see [DNS
@@ -175,25 +177,46 @@ Stack](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/dns-stack.m
 
 * *docker_options* - Commonly used to set
   ``--insecure-registry=myregistry.mydomain:5000``
+
 * *docker_plugins* - This list can be used to define [Docker plugins](https://docs.docker.com/engine/extend/) to install.
-* *containerd_default_runtime* - Sets the default Containerd runtime used by the Kubernetes CRI plugin.
+
+* *containerd_default_runtime* - If defined, changes the default Containerd runtime used by the Kubernetes CRI plugin.
+
 * *containerd_additional_runtimes* - Sets the additional Containerd runtimes used by the Kubernetes CRI plugin.
   [Default config](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/container-engine/containerd/defaults/main.yml) can be overriden in inventory vars.
+
 * *http_proxy/https_proxy/no_proxy/no_proxy_exclude_workers/additional_no_proxy* - Proxy variables for deploying behind a
   proxy. Note that no_proxy defaults to all internal cluster IPs and hostnames
   that correspond to each node.
+  
 * *kubelet_cgroup_driver* - Allows manual override of the cgroup-driver option for Kubelet.
   By default autodetection is used to match container manager configuration.
   `systemd` is the preferred driver for `containerd` though it can have issues with `cgroups v1` and `kata-containers` in which case you may want to change to `cgroupfs`.
+
 * *kubelet_rotate_certificates* - Auto rotate the kubelet client certificates by requesting new certificates
   from the kube-apiserver when the certificate expiration approaches.
+
 * *kubelet_rotate_server_certificates* - Auto rotate the kubelet server certificates by requesting new certificates
   from the kube-apiserver when the certificate expiration approaches.
   **Note** that server certificates are **not** approved automatically. Approve them manually
   (`kubectl get csr`, `kubectl certificate approve`) or implement custom approving controller like
   [kubelet-rubber-stamp](https://github.com/kontena/kubelet-rubber-stamp).
+
 * *kubelet_streaming_connection_idle_timeout* - Set the maximum time a streaming connection can be idle before the connection is automatically closed.
+
 * *kubelet_make_iptables_util_chains* - If `true`, causes the kubelet ensures a set of `iptables` rules are present on host.
+
+* *kubelet_systemd_hardening* - If `true`, provides kubelet systemd service with security features for isolation.
+
+  **N.B.** To enable this feature, ensure you are using the **`cgroup v2`** on your system. Check it out with command: `sudo ls -l /sys/fs/cgroup/*.slice`. If directory does not exists, enable this with the following guide: [enable cgroup v2](https://rootlesscontaine.rs/getting-started/common/cgroup2/#enabling-cgroup-v2).
+
+  * *kubelet_secure_addresses* - By default *kubelet_systemd_hardening* set the **control plane** `ansible_host` IPs as the `kubelet_secure_addresses`. In case you have multiple interfaces in your control plane nodes and the `kube-apiserver` is not bound to the default interface, you can override them with this variable.
+    Example:
+  
+    The **control plane** node may have 2 interfaces with the following IP addresses: `eth0:10.0.0.110`, `eth1:192.168.1.110`.
+  
+    By default the `kubelet_secure_addresses` is set with the `10.0.0.110` the ansible control host uses `eth0` to  connect to the machine. In case you want to use `eth1` as the outgoing interface on which `kube-apiserver` connects to the `kubelet`s, you should override the variable in this way: `kubelet_secure_addresses: "192.168.1.110"`.
+
 * *node_labels* - Labels applied to nodes via kubelet --node-labels parameter.
   For example, labels can be set in the inventory as variables or more widely in group_vars.
   *node_labels* can only be defined as a dict:

docs/vsphere-csi.md

@@ -31,12 +31,13 @@ You need to source the vSphere credentials you use to deploy your machines that
 | vsphere_csi_controller_replicas             | TRUE     | integer |                            | 1                         | Number of pods Kubernetes should deploy for the CSI controller                                                      |
 | vsphere_csi_liveness_probe_image_tag        | TRUE     | string  |                            | "v2.2.0"                  | CSI liveness probe image tag to use                                                                                 |
 | vsphere_csi_provisioner_image_tag           | TRUE     | string  |                            | "v2.1.0"                  | CSI provisioner image tag to use                                                                                    |
-| vsphere_csi_node_driver_registrar_image_tag | TRUE     | string  |                            | "v1.1.0"                  | CSI node driver registrat image tag to use                                                                          |
+| vsphere_csi_node_driver_registrar_image_tag | TRUE     | string  |                            | "v1.1.0"                  | CSI node driver registrar image tag to use                                                                          |
 | vsphere_csi_driver_image_tag                | TRUE     | string  |                            | "v1.0.2"                  | CSI driver image tag to use                                                                                         |
 | vsphere_csi_resizer_tag                     | TRUE     | string  |                            | "v1.1.0"                  | CSI resizer image tag to use
 | vsphere_csi_aggressive_node_drain           | FALSE    | boolean |                            | false                     | Enable aggressive node drain strategy                                                                               |
 | vsphere_csi_aggressive_node_unreachable_timeout            | FALSE     | int  | 300   |                           | Timeout till node will be drained when it in an unreachable state                                                           |
 | vsphere_csi_aggressive_node_not_ready_timeout              | FALSE     | int  | 300   |                           | Timeout till node will be drained when it in not-ready state                                                                |
+| vsphere_csi_namespace                       | TRUE     | string  |                            | "kube-system"       | vSphere CSI namespace to use; kube-system for backward compatibility, should be change to vmware-system-csi on the long run |
 
 ## Usage example
 

inventory/sample/group_vars/all/all.yml

@@ -120,3 +120,16 @@ kube_webhook_token_auth_url_skip_tls_verify: false
 # kube_webhook_token_auth_url: https://...
 ## base64-encoded string of the webhook's CA certificate
 # kube_webhook_token_auth_ca_data: "LS0t..."
+
+## NTP Settings
+# Start the ntpd or chrony service and enable it at system boot.
+ntp_enabled: false
+ntp_manage_config: false
+ntp_servers:
+  - "0.pool.ntp.org iburst"
+  - "1.pool.ntp.org iburst"
+  - "2.pool.ntp.org iburst"
+  - "3.pool.ntp.org iburst"
+
+## Used to control no_log attribute
+unsafe_show_logs: false

inventory/sample/group_vars/all/docker.yml

@@ -57,6 +57,3 @@ docker_rpm_keepcache: 1
 ## A string of extra options to pass to the docker daemon.
 ## This string should be exactly as you wish it to appear.
 # docker_options: ""
-
-## Use CRI-DockerD instead of dockershim
-# cri_dockerd_enabled: false

inventory/sample/group_vars/all/offline.yml

@@ -18,44 +18,48 @@
 # quay_image_repo: "{{ registry_host }}"
 
 ## Kubernetes components
-# kubeadm_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubeadm"
-# kubectl_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubectl"
-# kubelet_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubelet"
+# kubeadm_download_url: "{{ files_repo }}/storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/{{ image_arch }}/kubeadm"
+# kubectl_download_url: "{{ files_repo }}/storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubectl"
+# kubelet_download_url: "{{ files_repo }}/storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubelet"
 
 ## CNI Plugins
-# cni_download_url: "{{ files_repo }}/kubernetes/cni/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
+# cni_download_url: "{{ files_repo }}/github.com/containernetworking/plugins/releases/download/{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
 
 ## cri-tools
-# crictl_download_url: "{{ files_repo }}/kubernetes/cri-tools/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
+# crictl_download_url: "{{ files_repo }}/github.com/kubernetes-sigs/cri-tools/releases/download/{{ crictl_version }}/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
 
 ## [Optional] etcd: only if you **DON'T** use etcd_deployment=host
-# etcd_download_url: "{{ files_repo }}/kubernetes/etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz"
+# etcd_download_url: "{{ files_repo }}/github.com/etcd-io/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
 
 # [Optional] Calico: If using Calico network plugin
-# calicoctl_download_url: "{{ files_repo }}/kubernetes/calico/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
+# calicoctl_download_url: "{{ files_repo }}/github.com/projectcalico/calico/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
+# calicoctl_alternate_download_url: "{{ files_repo }}/github.com/projectcalico/calicoctl/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
 # [Optional] Calico with kdd: If using Calico network plugin with kdd datastore
-# calico_crds_download_url: "{{ files_repo }}/kubernetes/calico/{{ calico_version }}.tar.gz"
+# calico_crds_download_url: "{{ files_repo }}/github.com/projectcalico/calico/archive/{{ calico_version }}.tar.gz"
 
 # [Optional] Flannel: If using Falnnel network plugin
 # flannel_cni_download_url: "{{ files_repo }}/kubernetes/flannel/{{ flannel_cni_version }}/flannel-{{ image_arch }}"
 
 # [Optional] helm: only if you set helm_enabled: true
-# helm_download_url: "{{ files_repo }}/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz"
+# helm_download_url: "{{ files_repo }}/get.helm.sh/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz"
 
 # [Optional] crun: only if you set crun_enabled: true
-# crun_download_url: "{{ files_repo }}/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}"
+# crun_download_url: "{{ files_repo }}/github.com/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}"
 
 # [Optional] kata: only if you set kata_containers_enabled: true
-# kata_containers_download_url: "{{ files_repo }}/kata-containers/runtime/releases/download/{{ kata_containers_version }}/kata-static-{{ kata_containers_version }}-{{ ansible_architecture }}.tar.xz"
+# kata_containers_download_url: "{{ files_repo }}/github.com/kata-containers/kata-containers/releases/download/{{ kata_containers_version }}/kata-static-{{ kata_containers_version }}-{{ ansible_architecture }}.tar.xz"
+
+# [Optional] cri-dockerd: only if you set container_manager: docker
+# cri_dockerd_download_url: "{{ files_repo }}/github.com/Mirantis/cri-dockerd/releases/download/v{{ cri_dockerd_version }}/cri-dockerd-{{ cri_dockerd_version }}.{{ image_arch }}.tgz"
 
 # [Optional] cri-o: only if you set container_manager: crio
 # crio_download_base: "download.opensuse.org/repositories/devel:kubic:libcontainers:stable"
 # crio_download_crio: "http://{{ crio_download_base }}:/cri-o:/"
 
 # [Optional] runc,containerd: only if you set container_runtime: containerd
-# runc_download_url: "{{ files_repo }}/{{ runc_version }}/runc.{{ image_arch }}"
-# containerd_download_url: "{{ files_repo }}/containerd/v{{ containerd_version }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"
-# nerdctl_download_url: "{{ files_repo }}/nerdctl/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
+# runc_download_url: "{{ files_repo }}/github.com/opencontainers/runc/releases/download/{{ runc_version }}/runc.{{ image_arch }}"
+# containerd_download_url: "{{ files_repo }}/github.com/containerd/containerd/releases/download/v{{ containerd_version }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"
+# nerdctl_download_url: "{{ files_repo }}/github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
 
 ## CentOS/Redhat/AlmaLinux
 ### For EL7, base and extras repo must be available, for EL8, baseos and appstream
@@ -78,8 +82,8 @@
 # docker_debian_repo_base_url: "{{ debian_repo }}/docker-ce"
 # docker_debian_repo_gpgkey: "{{ debian_repo }}/docker-ce/gpg"
 ### Containerd
-# containerd_debian_repo_base_url: "{{ ubuntu_repo }}/containerd"
-# containerd_debian_repo_gpgkey: "{{ ubuntu_repo }}/containerd/gpg"
+# containerd_debian_repo_base_url: "{{ debian_repo }}/containerd"
+# containerd_debian_repo_gpgkey: "{{ debian_repo }}/containerd/gpg"
 # containerd_debian_repo_repokey: 'YOURREPOKEY'
 
 ## Ubuntu

inventory/sample/group_vars/all/upcloud.yml

@@ -7,13 +7,18 @@
 # upcloud_csi_provisioner_image_tag: "v3.1.0"
 # upcloud_csi_attacher_image_tag: "v3.4.0"
 # upcloud_csi_resizer_image_tag: "v1.4.0"
-# upcloud_csi_plugin_image_tag: "v0.2.1"
+# upcloud_csi_plugin_image_tag: "v0.3.3"
 # upcloud_csi_node_image_tag: "v2.5.0"
 # upcloud_tolerations: []
 ## Storage class options
-# expand_persistent_volumes: true
-# parameters:
-#   tier: maxiops # or hdd
 # storage_classes:
 #   - name: standard
 #     is_default: true
+#     expand_persistent_volumes: true
+#     parameters:
+#       tier: maxiops
+#   - name: hdd
+#     is_default: false
+#     expand_persistent_volumes: true
+#     parameters:
+#       tier: hdd

inventory/sample/group_vars/all/vsphere.yml

@@ -14,16 +14,16 @@
 ## gcr.io/cloud-provider-vsphere/cpi/release/manager
 # external_vsphere_cloud_controller_image_tag: "latest"
 ## gcr.io/cloud-provider-vsphere/csi/release/syncer
-# vsphere_syncer_image_tag: "v2.4.0"
-## k8s.gcr.io/sig-storage/csi-attacher
-# vsphere_csi_attacher_image_tag: "v3.3.0"
+# vsphere_syncer_image_tag: "v2.5.1"
+## registry.k8s.io/sig-storage/csi-attacher
+# vsphere_csi_attacher_image_tag: "v3.4.0"
 ## gcr.io/cloud-provider-vsphere/csi/release/driver
-# vsphere_csi_controller: "v2.4.0"
-## k8s.gcr.io/sig-storage/livenessprobe
-# vsphere_csi_liveness_probe_image_tag: "v2.4.0"
-## k8s.gcr.io/sig-storage/csi-provisioner
-# vsphere_csi_provisioner_image_tag: "v3.0.0"
-## k8s.gcr.io/sig-storage/csi-resizer
+# vsphere_csi_controller: "v2.5.1"
+## registry.k8s.io/sig-storage/livenessprobe
+# vsphere_csi_liveness_probe_image_tag: "v2.6.0"
+## registry.k8s.io/sig-storage/csi-provisioner
+# vsphere_csi_provisioner_image_tag: "v3.1.0"
+## registry.k8s.io/sig-storage/csi-resizer
 ## makes sense only for vSphere version >=7.0
 # vsphere_csi_resizer_tag: "v1.3.0"
 

inventory/sample/group_vars/k8s_cluster/addons.yml

@@ -27,7 +27,7 @@ local_path_provisioner_enabled: false
 # local_path_provisioner_claim_root: /opt/local-path-provisioner/
 # local_path_provisioner_debug: false
 # local_path_provisioner_image_repo: "rancher/local-path-provisioner"
-# local_path_provisioner_image_tag: "v0.0.21"
+# local_path_provisioner_image_tag: "v0.0.22"
 # local_path_provisioner_helper_image_repo: "busybox"
 # local_path_provisioner_helper_image_tag: "latest"
 
@@ -165,7 +165,8 @@ metallb_speaker_enabled: true
 # metallb_ip_range:
 #   - "10.5.0.50-10.5.0.99"
 # metallb_pool_name: "loadbalanced"
-# matallb_auto_assign: true
+# metallb_auto_assign: true
+# metallb_avoid_buggy_ips: false
 # metallb_speaker_nodeselector:
 #   kubernetes.io/os: "linux"
 # metallb_controller_nodeselector:
@@ -198,6 +199,7 @@ metallb_speaker_enabled: true
 #       - "10.5.1.50-10.5.1.99"
 #     protocol: "layer2"
 #     auto_assign: false
+#     avoid_buggy_ips: false
 # metallb_protocol: "bgp"
 # metallb_peers:
 #   - peer_address: 192.0.2.1
@@ -207,12 +209,11 @@ metallb_speaker_enabled: true
 #     peer_asn: 64513
 #     my_asn: 4200000000
 
-
 argocd_enabled: false
-# argocd_version: v2.1.6
+# argocd_version: v2.4.12
 # argocd_namespace: argocd
 # Default password:
-#   - https://argoproj.github.io/argo-cd/getting_started/#4-login-using-the-cli
+#   - https://argo-cd.readthedocs.io/en/stable/getting_started/#4-login-using-the-cli
 #   ---
 #   The initial password is autogenerated to be the pod name of the Argo CD API server. This can be retrieved with the command:
 #   kubectl get pods -n argocd -l app.kubernetes.io/name=argocd-server -o name | cut -d'/' -f 2

inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml

@@ -17,7 +17,7 @@ kube_token_dir: "{{ kube_config_dir }}/tokens"
 kube_api_anonymous_auth: true
 
 ## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.23.7
+kube_version: v1.24.6
 
 # Where the binaries will be downloaded.
 # Note: ensure that you've enough disk space (about 1G)
@@ -25,6 +25,9 @@ local_release_dir: "/tmp/releases"
 # Random shifts for retrying failed ops like pushing/downloading
 retry_stagger: 5
 
+# This is the user that owns tha cluster installation.
+kube_owner: kube
+
 # This is the group that the cert creation scripts chgrp the
 # cert files to. Not really changeable...
 kube_cert_group: kube-cert
@@ -116,16 +119,13 @@ kube_network_node_prefix_ipv6: 120
 # The port the API Server will be listening on.
 kube_apiserver_ip: "{{ kube_service_addresses|ipaddr('net')|ipaddr(1)|ipaddr('address') }}"
 kube_apiserver_port: 6443  # (https)
-# kube_apiserver_insecure_port: 8080  # (http)
-# Set to 0 to disable insecure port - Requires RBAC in authorization_modes and kube_api_anonymous_auth: true
-kube_apiserver_insecure_port: 0  # (disabled)
 
 # Kube-proxy proxyMode configuration.
 # Can be ipvs, iptables
 kube_proxy_mode: ipvs
 
 # configure arp_ignore and arp_announce to avoid answering ARP queries from kube-ipvs0 interface
-# must be set to true for MetalLB to work
+# must be set to true for MetalLB, kube-vip(ARP enabled) to work
 kube_proxy_strict_arp: false
 
 # A string slice of values which specify the addresses to use for NodePorts.
@@ -160,6 +160,14 @@ kube_encrypt_secret_data: false
 cluster_name: cluster.local
 # Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods
 ndots: 2
+# dns_timeout: 2
+# dns_attempts: 2
+# Custom search domains to be added in addition to the default cluster search domains
+# searchdomains:
+#   - svc.{{ cluster_name }}
+#   - default.svc.{{ cluster_name  }}
+# Remove default cluster search domains (``default.svc.{{ dns_domain }}, svc.{{ dns_domain }}``).
+# remove_default_searchdomains: false
 # Can be coredns, coredns_dual, manual or none
 dns_mode: coredns
 # Set manual server if using a custom cluster DNS server
@@ -185,6 +193,13 @@ nodelocaldns_secondary_skew_seconds: 5
 #   nameservers:
 #   - 192.168.0.53
 #   cache: 0
+# - zones:
+#   - mydomain.tld
+#   nameservers:
+#   - 10.233.0.3
+#   cache: 5
+#   rewrite:
+#   - name website.tld website.namespace.svc.cluster.local
 # Enable k8s_external plugin for CoreDNS
 enable_coredns_k8s_external: false
 coredns_k8s_external_zone: k8s_external.local
@@ -216,16 +231,9 @@ k8s_image_pull_policy: IfNotPresent
 # audit log for kubernetes
 kubernetes_audit: false
 
-# dynamic kubelet configuration
-# Note: Feature DynamicKubeletConfig is deprecated in 1.22 and will not move to GA.
-# It is planned to be removed from Kubernetes in the version 1.23.
-# Please use alternative ways to update kubelet configuration.
-dynamic_kubelet_configuration: false
-
 # define kubelet config dir for dynamic kubelet
 # kubelet_config_dir:
 default_kubelet_config_dir: "{{ kube_config_dir }}/dynamic_kubelet_dir"
-dynamic_kubelet_configuration_dir: "{{ kubelet_config_dir | default(default_kubelet_config_dir) }}"
 
 # pod security policy (RBAC must be enabled either by having 'RBAC' in authorization_modes or kubeadm enabled)
 podsecuritypolicy_enabled: false
@@ -252,9 +260,11 @@ podsecuritypolicy_enabled: false
 ## Uncomment to override default values
 # system_memory_reserved: 512Mi
 # system_cpu_reserved: 500m
+# system_ephemeral_storage_reserved: 2Gi
 ## Reservation for master hosts
 # system_master_memory_reserved: 256Mi
 # system_master_cpu_reserved: 250m
+# system_master_ephemeral_storage_reserved: 2Gi
 
 ## Eviction Thresholds to avoid system OOMs
 # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#eviction-thresholds
@@ -292,7 +302,7 @@ persistent_volumes_enabled: false
 # nvidia_driver_install_centos_container: atzedevries/nvidia-centos-driver-installer:2
 # nvidia_driver_install_ubuntu_container: gcr.io/google-containers/ubuntu-nvidia-driver-installer@sha256:7df76a0f0a17294e86f691c81de6bbb7c04a1b4b3d4ea4e7e2cccdc42e1f6d63
 ## NVIDIA GPU device plugin image.
-# nvidia_gpu_device_plugin_container: "k8s.gcr.io/nvidia-gpu-device-plugin@sha256:0842734032018be107fa2490c98156992911e3e1f2a21e059ff0105b07dd8e9e"
+# nvidia_gpu_device_plugin_container: "registry.k8s.io/nvidia-gpu-device-plugin@sha256:0842734032018be107fa2490c98156992911e3e1f2a21e059ff0105b07dd8e9e"
 
 ## Support tls min version, Possible values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
 # tls_min_version: ""

inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml

@@ -19,7 +19,7 @@ calico_cni_name: k8s-pod-network
 # calico_pool_name: "default-pool"
 
 # add default ippool blockSize (defaults kube_network_node_prefix)
-# calico_pool_blocksize: 24
+calico_pool_blocksize: 26
 
 # add default ippool CIDR (must be inside kube_pods_subnet, defaults to kube_pods_subnet otherwise)
 # calico_pool_cidr: 1.2.3.4/5
@@ -60,7 +60,7 @@ calico_cni_name: k8s-pod-network
 # - x.x.x.x/24
 # - y.y.y.y/32
 
-# Adveritse Service LoadBalancer IPs
+# Advertise Service LoadBalancer IPs
 # calico_advertise_service_loadbalancer_ips:
 # - x.x.x.x/24
 # - y.y.y.y/16
@@ -99,7 +99,7 @@ calico_cni_name: k8s-pod-network
 # calico_vxlan_vni: 4096
 # calico_vxlan_port: 4789
 
-# Cenable eBPF mode
+# Enable eBPF mode
 # calico_bpf_enabled: false
 
 # If you want to use non default IP_AUTODETECTION_METHOD, IP6_AUTODETECTION_METHOD for calico node set this option to one of:

inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml

@@ -1,19 +1,86 @@
-# see roles/network_plugin/cilium/defaults/main.yml
+---
+# cilium_version: "v1.12.1"
 
-# cilium_version: "v1.11.3"
-# cilium_identity_allocation_mode: kvstore # kvstore or crd
+# Log-level
+# cilium_debug: false
 
-# For adding and mounting extra volumes to the cilium operator
-# cilium_operator_extra_volumes: []
-# cilium_operator_extra_volume_mounts: []
+# cilium_mtu: ""
+# cilium_enable_ipv4: true
+# cilium_enable_ipv6: false
 
-# Name of the cluster. Only relevant when building a mesh of clusters.
-# cilium_cluster_name: default
+# Cilium agent health port
+# cilium_agent_health_port: "9879"
+
+# Identity allocation mode selects how identities are shared between cilium
+# nodes by setting how they are stored. The options are "crd" or "kvstore".
+# - "crd" stores identities in kubernetes as CRDs (custom resource definition).
+#   These can be queried with:
+#     `kubectl get ciliumid`
+# - "kvstore" stores identities in an etcd kvstore.
+# - In order to support External Workloads, "crd" is required
+#   - Ref: https://docs.cilium.io/en/stable/gettingstarted/external-workloads/#setting-up-support-for-external-workloads-beta
+# - KVStore operations are only required when cilium-operator is running with any of the below options:
+#   - --synchronize-k8s-services
+#   - --synchronize-k8s-nodes
+#   - --identity-allocation-mode=kvstore
+#   - Ref: https://docs.cilium.io/en/stable/internals/cilium_operator/#kvstore-operations
+# cilium_identity_allocation_mode: kvstore
+
+# Etcd SSL dirs
+# cilium_cert_dir: /etc/cilium/certs
+# kube_etcd_cacert_file: ca.pem
+# kube_etcd_cert_file: cert.pem
+# kube_etcd_key_file: cert-key.pem
+
+# Limits for apps
+# cilium_memory_limit: 500M
+# cilium_cpu_limit: 500m
+# cilium_memory_requests: 64M
+# cilium_cpu_requests: 100m
+
+# Overlay Network Mode
+# cilium_tunnel_mode: vxlan
+# Optional features
+# cilium_enable_prometheus: false
+# Enable if you want to make use of hostPort mappings
+# cilium_enable_portmap: false
+# Monitor aggregation level (none/low/medium/maximum)
+# cilium_monitor_aggregation: medium
+# The monitor aggregation flags determine which TCP flags which, upon the
+# first observation, cause monitor notifications to be generated.
+#
+# Only effective when monitor aggregation is set to "medium" or higher.
+# cilium_monitor_aggregation_flags: "all"
+# Kube Proxy Replacement mode (strict/probe/partial)
+# cilium_kube_proxy_replacement: probe
+
+# If upgrading from Cilium < 1.5, you may want to override some of these options
+# to prevent service disruptions. See also:
+# http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action
+# cilium_preallocate_bpf_maps: false
+
+# `cilium_tofqdns_enable_poller` is deprecated in 1.8, removed in 1.9
+# cilium_tofqdns_enable_poller: false
+
+# `cilium_enable_legacy_services` is deprecated in 1.6, removed in 1.9
+# cilium_enable_legacy_services: false
 
 # Unique ID of the cluster. Must be unique across all conneted clusters and
 # in the range of 1 and 255. Only relevant when building a mesh of clusters.
 # This value is not defined by default
-# cluster-id:
+# cilium_cluster_id:
+
+# Deploy cilium even if kube_network_plugin is not cilium.
+# This enables to deploy cilium alongside another CNI to replace kube-proxy.
+# cilium_deploy_additionally: false
+
+# Auto direct nodes routes can be used to advertise pods routes in your cluster
+# without any tunelling (with `cilium_tunnel_mode` sets to `disabled`).
+# This works only if you have a L2 connectivity between all your nodes.
+# You wil also have to specify the variable `cilium_native_routing_cidr` to
+# make this work. Please refer to the cilium documentation for more
+# information about this kind of setups.
+# cilium_auto_direct_node_routes: false
 
 # Allows to explicitly specify the IPv4 CIDR for native routing.
 # When specified, Cilium assumes networking for this CIDR is preconfigured and
@@ -30,7 +97,6 @@
 # Allows to explicitly specify the IPv6 CIDR for native routing.
 # cilium_native_routing_cidr_ipv6: ""
 
-# Encryption
 # Enable transparent network encryption.
 # cilium_encryption_enabled: false
 
@@ -40,8 +106,140 @@
 
 # Enable encryption for pure node to node traffic.
 # This option is only effective when `cilium_encryption_type` is set to `ipsec`.
-# cilium_ipsec_node_encryption: "false"
+# cilium_ipsec_node_encryption: false
 
-# Enables the fallback to the user-space implementation.
+# If your kernel or distribution does not support WireGuard, Cilium agent can be configured to fall back on the user-space implementation.
+# When this flag is enabled and Cilium detects that the kernel has no native support for WireGuard,
+# it will fallback on the wireguard-go user-space implementation of WireGuard.
 # This option is only effective when `cilium_encryption_type` is set to `wireguard`.
-# cilium_wireguard_userspace_fallback: "false"
+# cilium_wireguard_userspace_fallback: false
+
+# IP Masquerade Agent
+# https://docs.cilium.io/en/stable/concepts/networking/masquerading/
+# By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded
+# cilium_ip_masq_agent_enable: false
+
+### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded
+# cilium_non_masquerade_cidrs:
+#   - 10.0.0.0/8
+#   - 172.16.0.0/12
+#   - 192.168.0.0/16
+#   - 100.64.0.0/10
+#   - 192.0.0.0/24
+#   - 192.0.2.0/24
+#   - 192.88.99.0/24
+#   - 198.18.0.0/15
+#   - 198.51.100.0/24
+#   - 203.0.113.0/24
+#   - 240.0.0.0/4
+### Indicates whether to masquerade traffic to the link local prefix.
+### If the masqLinkLocal is not set or set to false, then 169.254.0.0/16 is appended to the non-masquerade CIDRs list.
+# cilium_masq_link_local: false
+### A time interval at which the agent attempts to reload config from disk
+# cilium_ip_masq_resync_interval: 60s
+
+# Hubble
+### Enable Hubble without install
+# cilium_enable_hubble: false
+### Enable Hubble Metrics
+# cilium_enable_hubble_metrics: false
+### if cilium_enable_hubble_metrics: true
+# cilium_hubble_metrics: {}
+# - dns
+# - drop
+# - tcp
+# - flow
+# - icmp
+# - http
+### Enable Hubble install
+# cilium_hubble_install: false
+### Enable auto generate certs if cilium_hubble_install: true
+# cilium_hubble_tls_generate: false
+
+# IP address management mode for v1.9+.
+# https://docs.cilium.io/en/v1.9/concepts/networking/ipam/
+# cilium_ipam_mode: kubernetes
+
+# Extra arguments for the Cilium agent
+# cilium_agent_custom_args: []
+
+# For adding and mounting extra volumes to the cilium agent
+# cilium_agent_extra_volumes: []
+# cilium_agent_extra_volume_mounts: []
+
+# cilium_agent_extra_env_vars: []
+
+# cilium_operator_replicas: 2
+
+# The address at which the cillium operator bind health check api
+# cilium_operator_api_serve_addr: "127.0.0.1:9234"
+
+## A dictionary of extra config variables to add to cilium-config, formatted like:
+##  cilium_config_extra_vars:
+##    var1: "value1"
+##    var2: "value2"
+# cilium_config_extra_vars: {}
+
+# For adding and mounting extra volumes to the cilium operator
+# cilium_operator_extra_volumes: []
+# cilium_operator_extra_volume_mounts: []
+
+# Extra arguments for the Cilium Operator
+# cilium_operator_custom_args: []
+
+# Name of the cluster. Only relevant when building a mesh of clusters.
+# cilium_cluster_name: default
+
+# Make Cilium take ownership over the `/etc/cni/net.d` directory on the node, renaming all non-Cilium CNI configurations to `*.cilium_bak`.
+# This ensures no Pods can be scheduled using other CNI plugins during Cilium agent downtime.
+# Available for Cilium v1.10 and up.
+# cilium_cni_exclusive: true
+
+# Configure the log file for CNI logging with retention policy of 7 days.
+# Disable CNI file logging by setting this field to empty explicitly.
+# Available for Cilium v1.12 and up.
+# cilium_cni_log_file: "/var/run/cilium/cilium-cni.log"
+
+# -- Configure cgroup related configuration
+# -- Enable auto mount of cgroup2 filesystem.
+# When `cilium_cgroup_auto_mount` is enabled, cgroup2 filesystem is mounted at
+# `cilium_cgroup_host_root` path on the underlying host and inside the cilium agent pod.
+# If users disable `cilium_cgroup_auto_mount`, it's expected that users have mounted
+# cgroup2 filesystem at the specified `cilium_cgroup_auto_mount` volume, and then the
+# volume will be mounted inside the cilium agent pod at the same path.
+# Available for Cilium v1.11 and up
+# cilium_cgroup_auto_mount: true
+# -- Configure cgroup root where cgroup2 filesystem is mounted on the host
+# cilium_cgroup_host_root: "/run/cilium/cgroupv2"
+
+# Specifies the ratio (0.0-1.0) of total system memory to use for dynamic
+# sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.
+# cilium_bpf_map_dynamic_size_ratio: "0.0"
+
+# -- Enables masquerading of IPv4 traffic leaving the node from endpoints.
+# Available for Cilium v1.10 and up
+# cilium_enable_ipv4_masquerade: true
+# -- Enables masquerading of IPv6 traffic leaving the node from endpoints.
+# Available for Cilium v1.10 and up
+# cilium_enable_ipv6_masquerade: true
+
+# -- Enable native IP masquerade support in eBPF
+# cilium_enable_bpf_masquerade: false
+
+# -- Configure whether direct routing mode should route traffic via
+# host stack (true) or directly and more efficiently out of BPF (false) if
+# the kernel supports it. The latter has the implication that it will also
+# bypass netfilter in the host namespace.
+# cilium_enable_host_legacy_routing: true
+
+# -- Enable use of the remote node identity.
+# ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity
+# cilium_enable_remote_node_identity: true
+
+# -- Enable the use of well-known identities.
+# cilium_enable_well_known_identities: false
+
+# cilium_enable_bpf_clock_probe: true
+
+# -- Whether to enable CNP status updates.
+# cilium_disable_cnp_status_updates: true

library/kube.py

@@ -314,7 +314,7 @@ def main():
             wait=dict(default=False, type='bool'),
             all=dict(default=False, type='bool'),
             log_level=dict(default=0, type='int'),
-            state=dict(default='present', choices=['present', 'absent', 'latest', 'reloaded', 'stopped']),
+            state=dict(default='present', choices=['present', 'absent', 'latest', 'reloaded', 'stopped', 'exists']),
             recursive=dict(default=False, type='bool'),
             ),
             mutually_exclusive=[['filename', 'list']]
@@ -339,6 +339,11 @@ def main():
     elif state == 'latest':
         result = manager.replace()
 
+    elif state == 'exists':
+        result = manager.exists()
+        module.exit_json(changed=changed,
+                     msg='%s' % result)
+
     else:
         module.fail_json(msg='Unrecognized state %s.' % state)
 

requirements-2.10.txt

@@ -1,10 +0,0 @@
-ansible==3.4.0
-ansible-base==2.10.15
-cryptography==2.8
-jinja2==2.11.3
-netaddr==0.7.19
-pbr==5.4.4
-jmespath==0.9.5
-ruamel.yaml==0.16.10
-ruamel.yaml.clib==0.2.6
-MarkupSafe==1.1.1

requirements-2.9.txt

@@ -1,10 +0,0 @@
-ansible==2.9.27
-cryptography==2.8
-jinja2==2.11.3
-netaddr==0.7.19
-pbr==5.4.4
-jmespath==0.9.5
-ruamel.yaml==0.16.10
-ruamel.yaml.clib==0.2.6 ; python_version >= '3.5'
-ruamel.yaml.clib==0.2.2 ; python_version < '3.5'
-MarkupSafe==1.1.1

requirements-2.9.yml

@@ -1,4 +0,0 @@
----
-collections:
-- name: community.general
-  version: '<3.0'

roles/adduser/defaults/main.yml

@@ -1,4 +1,5 @@
 ---
+kube_owner: kube
 kube_cert_group: kube-cert
 etcd_data_dir: "/var/lib/etcd"
 
@@ -6,13 +7,13 @@ addusers:
   etcd:
     name: etcd
     comment: "Etcd user"
-    createhome: no
+    create_home: no
     system: yes
     shell: /sbin/nologin
   kube:
     name: kube
     comment: "Kubernetes user"
-    createhome: no
+    create_home: no
     system: yes
     shell: /sbin/nologin
     group: "{{ kube_cert_group }}"
@@ -23,4 +24,4 @@ adduser:
   comment: "{{ user.comment|default(None) }}"
   shell: "{{ user.shell|default(None) }}"
   system: "{{ user.system|default(None) }}"
-  createhome: "{{ user.createhome|default(None) }}"
+  create_home: "{{ user.create_home|default(None) }}"

roles/adduser/molecule/default/molecule.yml

@@ -17,7 +17,7 @@ provisioner:
   name: ansible
   config_options:
     defaults:
-      callback_whitelist: profile_tasks
+      callbacks_enabled: profile_tasks
       timeout: 120
   lint:
     name: ansible-lint

roles/adduser/tasks/main.yml

@@ -7,9 +7,10 @@
 - name: User | Create User
   user:
     comment: "{{ user.comment|default(omit) }}"
-    createhome: "{{ user.createhome|default(omit) }}"
+    create_home: "{{ user.create_home|default(omit) }}"
     group: "{{ user.group|default(user.name) }}"
     home: "{{ user.home|default(omit) }}"
     shell: "{{ user.shell|default(omit) }}"
     name: "{{ user.name }}"
     system: "{{ user.system|default(omit) }}"
+  when: user.name != "root"

roles/adduser/vars/coreos.yml

@@ -5,4 +5,4 @@ addusers:
     shell: /sbin/nologin
     system: yes
     group: "{{ kube_cert_group }}"
-    createhome: no
+    create_home: no

roles/adduser/vars/debian.yml

@@ -2,14 +2,14 @@
 addusers:
   - name: etcd
     comment: "Etcd user"
-    createhome: yes
+    create_home: yes
     home: "{{ etcd_data_dir }}"
     system: yes
     shell: /sbin/nologin
 
   - name: kube
     comment: "Kubernetes user"
-    createhome: no
+    create_home: no
     system: yes
     shell: /sbin/nologin
     group: "{{ kube_cert_group }}"

roles/adduser/vars/redhat.yml

@@ -2,14 +2,14 @@
 addusers:
   - name: etcd
     comment: "Etcd user"
-    createhome: yes
+    create_home: yes
     home: "{{ etcd_data_dir }}"
     system: yes
     shell: /sbin/nologin
 
   - name: kube
     comment: "Kubernetes user"
-    createhome: no
+    create_home: no
     system: yes
     shell: /sbin/nologin
     group: "{{ kube_cert_group }}"

roles/bastion-ssh-config/molecule/default/molecule.yml

@@ -17,7 +17,7 @@ provisioner:
   name: ansible
   config_options:
     defaults:
-      callback_whitelist: profile_tasks
+      callbacks_enabled: profile_tasks
       timeout: 120
   lint:
     name: ansible-lint

roles/bootstrap-os/defaults/main.yml

@@ -25,3 +25,8 @@ override_system_hostname: true
 is_fedora_coreos: false
 
 skip_http_proxy_on_os_packages: false
+
+# If this is true, debug information will be displayed but
+# may contain some private data, so it is recommended to set it to false
+# in the production environment.
+unsafe_show_logs: false

roles/bootstrap-os/files/bootstrap.sh

@@ -2,23 +2,39 @@
 set -e
 
 BINDIR="/opt/bin"
-PYPY_VERSION=7.3.2
-PYPI_URL="https://downloads.python.org/pypy/pypy3.6-v${PYPY_VERSION}-linux64.tar.bz2"
-PYPI_HASH=d7a91f179076aaa28115ffc0a81e46c6a787785b2bc995c926fe3b02f0e9ad83
+if [[ -e $BINDIR/.bootstrapped ]]; then
+  exit 0
+fi
+
+ARCH=$(uname -m)
+case $ARCH in
+  "x86_64")
+    PYPY_ARCH=linux64
+    PYPI_HASH=46818cb3d74b96b34787548343d266e2562b531ddbaf330383ba930ff1930ed5
+    ;;
+  "aarch64")
+    PYPY_ARCH=aarch64
+    PYPI_HASH=2e1ae193d98bc51439642a7618d521ea019f45b8fb226940f7e334c548d2b4b9
+    ;;
+  *)
+    echo "Unsupported Architecture: ${ARCH}"
+    exit 1
+esac
+
+PYTHON_VERSION=3.9
+PYPY_VERSION=7.3.9
+PYPY_FILENAME="pypy${PYTHON_VERSION}-v${PYPY_VERSION}-${PYPY_ARCH}"
+PYPI_URL="https://downloads.python.org/pypy/${PYPY_FILENAME}.tar.bz2"
 
 mkdir -p $BINDIR
 
 cd $BINDIR
 
-if [[ -e $BINDIR/.bootstrapped ]]; then
-  exit 0
-fi
-
 TAR_FILE=pyp.tar.bz2
 wget -O "${TAR_FILE}" "${PYPI_URL}"
 echo "${PYPI_HASH} ${TAR_FILE}" | sha256sum -c -
 tar -xjf "${TAR_FILE}" && rm "${TAR_FILE}"
-mv -n "pypy3.6-v${PYPY_VERSION}-linux64" pypy3
+mv -n "${PYPY_FILENAME}" pypy3
 
 ln -s ./pypy3/bin/pypy3 python
 $BINDIR/python --version

roles/bootstrap-os/molecule/default/molecule.yml

@@ -41,7 +41,7 @@ provisioner:
   name: ansible
   config_options:
     defaults:
-      callback_whitelist: profile_tasks
+      callbacks_enabled: profile_tasks
       timeout: 120
   lint:
     name: ansible-lint

roles/bootstrap-os/tasks/bootstrap-opensuse.yml

@@ -1,5 +1,9 @@
 ---
 # OpenSUSE ships with Python installed
+- name: Gather host facts to get ansible_distribution_version ansible_distribution_major_version
+  setup:
+    gather_subset: '!all'
+    filter: ansible_distribution_*version
 
 - name: Check that /etc/sysconfig/proxy file exists
   stat:
@@ -59,6 +63,17 @@
     state: present
     update_cache: true
   become: true
+  when:
+    - ansible_distribution_version is version('15.4', '<')
+
+- name: Install python3-cryptography
+  zypper:
+    name: python3-cryptography
+    state: present
+    update_cache: true
+  become: true
+  when:
+    - ansible_distribution_version is version('15.4', '>=')
 
 # Nerdctl needs some basic packages to get an environment up
 - name: Install basic dependencies

roles/bootstrap-os/tasks/bootstrap-redhat.yml

@@ -17,7 +17,7 @@
   when: not skip_http_proxy_on_os_packages
 
 - name: Add proxy to RHEL subscription-manager if http_proxy is defined
-  command: /sbin/subscription-manager config --server.proxy_hostname={{ http_proxy | regex_replace(':\\d+$') }} --server.proxy_port={{ http_proxy | regex_replace('^.*:') }}
+  command: /sbin/subscription-manager config --server.proxy_hostname={{ http_proxy | regex_replace(':\d+$') }} --server.proxy_port={{ http_proxy | regex_replace('^.*:') }}
   become: true
   when:
     - not skip_http_proxy_on_os_packages
@@ -65,7 +65,7 @@
   notify: RHEL auto-attach subscription
   ignore_errors: true  # noqa ignore-errors
   become: true
-  no_log: true
+  no_log: "{{ not (unsafe_show_logs|bool) }}"
   when:
     - rh_subscription_username is defined
     - rh_subscription_status.changed

roles/bootstrap-os/tasks/main.yml

@@ -7,7 +7,7 @@
   check_mode: false
 
 - include_tasks: bootstrap-centos.yml
-  when: '''ID="centos"'' in os_release.stdout_lines or ''ID="ol"'' in os_release.stdout_lines or ''ID="almalinux"'' in os_release.stdout_lines or ''ID="rocky"'' in os_release.stdout_lines'
+  when: '''ID="centos"'' in os_release.stdout_lines or ''ID="ol"'' in os_release.stdout_lines or ''ID="almalinux"'' in os_release.stdout_lines or ''ID="rocky"'' in os_release.stdout_lines or ''ID="kylin"'' in os_release.stdout_lines'
 
 - include_tasks: bootstrap-amazon.yml
   when: '''ID="amzn"'' in os_release.stdout_lines'
@@ -84,7 +84,7 @@
     or is_fedora_coreos
     or ansible_distribution == "Fedora")
 
-- name: "Install ceph-commmon package"
+- name: Install ceph-commmon package
   package:
     name:
       - ceph-common

roles/container-engine/containerd/defaults/main.yml

@@ -4,7 +4,7 @@ containerd_state_dir: "/run/containerd"
 containerd_systemd_dir: "/etc/systemd/system/containerd.service.d"
 containerd_oom_score: 0
 
-containerd_default_runtime: "runc"
+# containerd_default_runtime: "runc"
 # containerd_snapshotter: "native"
 
 containerd_runc_runtime:
@@ -12,6 +12,7 @@ containerd_runc_runtime:
   type: "io.containerd.runc.v2"
   engine: ""
   root: ""
+  # base_runtime_spec: cri-base.json # use this to limit number of file descriptors per container
   options:
     systemdCgroup: "{{ containerd_use_systemd_cgroup | ternary('true', 'false') }}"
 
@@ -22,6 +23,9 @@ containerd_additional_runtimes: []
 #    engine: ""
 #    root: ""
 
+containerd_base_runtime_specs:
+  cri-base.json: "{{ lookup('file', 'cri-base.json') }}"
+
 containerd_grpc_max_recv_message_size: 16777216
 containerd_grpc_max_send_message_size: 16777216
 
@@ -46,3 +50,9 @@ containerd_registry_auth: []
 #  - registry: 10.0.0.2:5000
 #    username: user
 #    password: pass
+
+# Configure containerd service
+containerd_limit_proc_num: "infinity"
+containerd_limit_core: "infinity"
+containerd_limit_open_file_num: "infinity"
+containerd_limit_mem_lock: "infinity"

roles/container-engine/containerd/files/cri-base.json

@@ -0,0 +1,214 @@
+{
+    "ociVersion": "1.0.2-dev",
+    "process": {
+        "user": {
+            "uid": 0,
+            "gid": 0
+        },
+        "cwd": "/",
+        "capabilities": {
+            "bounding": [
+                "CAP_CHOWN",
+                "CAP_DAC_OVERRIDE",
+                "CAP_FSETID",
+                "CAP_FOWNER",
+                "CAP_MKNOD",
+                "CAP_NET_RAW",
+                "CAP_SETGID",
+                "CAP_SETUID",
+                "CAP_SETFCAP",
+                "CAP_SETPCAP",
+                "CAP_NET_BIND_SERVICE",
+                "CAP_SYS_CHROOT",
+                "CAP_KILL",
+                "CAP_AUDIT_WRITE"
+            ],
+            "effective": [
+                "CAP_CHOWN",
+                "CAP_DAC_OVERRIDE",
+                "CAP_FSETID",
+                "CAP_FOWNER",
+                "CAP_MKNOD",
+                "CAP_NET_RAW",
+                "CAP_SETGID",
+                "CAP_SETUID",
+                "CAP_SETFCAP",
+                "CAP_SETPCAP",
+                "CAP_NET_BIND_SERVICE",
+                "CAP_SYS_CHROOT",
+                "CAP_KILL",
+                "CAP_AUDIT_WRITE"
+            ],
+            "inheritable": [
+                "CAP_CHOWN",
+                "CAP_DAC_OVERRIDE",
+                "CAP_FSETID",
+                "CAP_FOWNER",
+                "CAP_MKNOD",
+                "CAP_NET_RAW",
+                "CAP_SETGID",
+                "CAP_SETUID",
+                "CAP_SETFCAP",
+                "CAP_SETPCAP",
+                "CAP_NET_BIND_SERVICE",
+                "CAP_SYS_CHROOT",
+                "CAP_KILL",
+                "CAP_AUDIT_WRITE"
+            ],
+            "permitted": [
+                "CAP_CHOWN",
+                "CAP_DAC_OVERRIDE",
+                "CAP_FSETID",
+                "CAP_FOWNER",
+                "CAP_MKNOD",
+                "CAP_NET_RAW",
+                "CAP_SETGID",
+                "CAP_SETUID",
+                "CAP_SETFCAP",
+                "CAP_SETPCAP",
+                "CAP_NET_BIND_SERVICE",
+                "CAP_SYS_CHROOT",
+                "CAP_KILL",
+                "CAP_AUDIT_WRITE"
+            ]
+        },
+        "rlimits": [
+            {
+                "type": "RLIMIT_NOFILE",
+                "hard": 1024,
+                "soft": 1024
+            }
+        ],
+        "noNewPrivileges": true
+    },
+    "root": {
+        "path": "rootfs"
+    },
+    "mounts": [
+        {
+            "destination": "/proc",
+            "type": "proc",
+            "source": "proc",
+            "options": [
+                "nosuid",
+                "noexec",
+                "nodev"
+            ]
+        },
+        {
+            "destination": "/dev",
+            "type": "tmpfs",
+            "source": "tmpfs",
+            "options": [
+                "nosuid",
+                "strictatime",
+                "mode=755",
+                "size=65536k"
+            ]
+        },
+        {
+            "destination": "/dev/pts",
+            "type": "devpts",
+            "source": "devpts",
+            "options": [
+                "nosuid",
+                "noexec",
+                "newinstance",
+                "ptmxmode=0666",
+                "mode=0620",
+                "gid=5"
+            ]
+        },
+        {
+            "destination": "/dev/shm",
+            "type": "tmpfs",
+            "source": "shm",
+            "options": [
+                "nosuid",
+                "noexec",
+                "nodev",
+                "mode=1777",
+                "size=65536k"
+            ]
+        },
+        {
+            "destination": "/dev/mqueue",
+            "type": "mqueue",
+            "source": "mqueue",
+            "options": [
+                "nosuid",
+                "noexec",
+                "nodev"
+            ]
+        },
+        {
+            "destination": "/sys",
+            "type": "sysfs",
+            "source": "sysfs",
+            "options": [
+                "nosuid",
+                "noexec",
+                "nodev",
+                "ro"
+            ]
+        },
+        {
+            "destination": "/run",
+            "type": "tmpfs",
+            "source": "tmpfs",
+            "options": [
+                "nosuid",
+                "strictatime",
+                "mode=755",
+                "size=65536k"
+            ]
+        }
+    ],
+    "linux": {
+        "resources": {
+            "devices": [
+                {
+                    "allow": false,
+                    "access": "rwm"
+                }
+            ]
+        },
+        "cgroupsPath": "/default",
+        "namespaces": [
+            {
+                "type": "pid"
+            },
+            {
+                "type": "ipc"
+            },
+            {
+                "type": "uts"
+            },
+            {
+                "type": "mount"
+            },
+            {
+                "type": "network"
+            }
+        ],
+        "maskedPaths": [
+            "/proc/acpi",
+            "/proc/asound",
+            "/proc/kcore",
+            "/proc/keys",
+            "/proc/latency_stats",
+            "/proc/timer_list",
+            "/proc/timer_stats",
+            "/proc/sched_debug",
+            "/sys/firmware",
+            "/proc/scsi"
+        ],
+        "readonlyPaths": [
+            "/proc/bus",
+            "/proc/fs",
+            "/proc/irq",
+            "/proc/sys",
+            "/proc/sysrq-trigger"
+        ]
+    }
+}

roles/container-engine/containerd/handlers/reset.yml

@@ -0,0 +1 @@
+---