CI: upgrade vagrant to 2.2.19 (#8264) (#8267)

Closes https://github.com/kubernetes-sigs/kubespray/issues/8028

Signed-off-by: Iago Santos <iago.santos.pardo@adfinis.com>

.ansible-lint

@@ -18,3 +18,13 @@ skip_list:
   # While it can be useful to have these metadata available, they are also available in the existing documentation.
   # (Disabled in May 2019)
   - '701'
+
+  # [role-name] "meta/main.yml" Role name role-name does not match ``^+$`` pattern
+  # Meta roles in Kubespray don't need proper names
+  # (Disabled in June 2021)
+  - 'role-name'
+
+  # [var-naming] "defaults/main.yml" File defines variable 'apiVersion' that violates variable naming standards
+  # In Kubespray we use variables that use camelCase to match their k8s counterparts
+  # (Disabled in June 2021)
+  - 'var-naming'

.gitlab-ci.yml

@@ -8,7 +8,7 @@ stages:
   - deploy-special
 
 variables:
-  KUBESPRAY_VERSION: v2.15.1
+  KUBESPRAY_VERSION: v2.16.0
   FAILFASTCI_NAMESPACE: 'kargo-ci'
   GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray'
   ANSIBLE_FORCE_COLOR: "true"
@@ -31,12 +31,13 @@ variables:
   ANSIBLE_LOG_LEVEL: "-vv"
   RECOVER_CONTROL_PLANE_TEST: "false"
   RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:],kube_control_plane[1:]"
-  TERRAFORM_14_VERSION: 0.14.10
+  TERRAFORM_14_VERSION: 0.14.11
-  TERRAFORM_13_VERSION: 0.13.6
+  TERRAFORM_15_VERSION: 0.15.5
 
 before_script:
   - ./tests/scripts/rebase.sh
   - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+  - python -m pip uninstall -y ansible
   - python -m pip install -r tests/requirements.txt
   - mkdir -p /.ssh
 
@@ -51,6 +52,7 @@ before_script:
 
 .testcases: &testcases
   <<: *job
+  retry: 1
   before_script:
     - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
     - ./tests/scripts/rebase.sh

.gitlab-ci/lint.yml

@@ -14,7 +14,7 @@ vagrant-validate:
   stage: unit-tests
   tags: [light]
   variables:
-    VAGRANT_VERSION: 2.2.15
+    VAGRANT_VERSION: 2.2.19
   script:
     - ./tests/scripts/vagrant-validate.sh
   except: ['triggers', 'master']
@@ -53,6 +53,7 @@ tox-inventory-builder:
     - ./tests/scripts/rebase.sh
     - apt-get update && apt-get install -y python3-pip
     - update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+    - python -m pip uninstall -y ansible
     - python -m pip install -r tests/requirements.txt
   script:
     - pip3 install tox

.gitlab-ci/packet.yml

@@ -91,6 +91,11 @@ packet_debian10-containerd:
   variables:
     MITOGEN_ENABLE: "true"
 
+packet_debian11-calico:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
 packet_centos7-calico-ha-once-localhost:
   stage: deploy-part2
   extends: .packet_pr
@@ -111,7 +116,7 @@ packet_centos8-calico:
   extends: .packet_pr
   when: on_success
 
-packet_fedora32-weave:
+packet_fedora34-weave:
   stage: deploy-part2
   extends: .packet_pr
   when: on_success
@@ -177,15 +182,18 @@ packet_fedora33-calico:
   stage: deploy-part2
   extends: .packet_periodic
   when: on_success
-  variables:
+
-    MITOGEN_ENABLE: "true"
+packet_fedora34-calico-selinux:
+  stage: deploy-part2
+  extends: .packet_periodic
+  when: on_success
 
 packet_amazon-linux-2-aio:
   stage: deploy-part2
   extends: .packet_pr
   when: manual
 
-packet_fedora32-kube-ovn-containerd:
+packet_fedora34-kube-ovn-containerd:
   stage: deploy-part2
   extends: .packet_periodic
   when: on_success
@@ -201,6 +209,14 @@ packet_centos7-weave-upgrade-ha:
     UPGRADE_TEST: basic
     MITOGEN_ENABLE: "false"
 
+# Calico HA Wireguard
+packet_ubuntu20-calico-ha-wireguard:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: manual
+  variables:
+    MITOGEN_ENABLE: "true"
+
 packet_debian9-calico-upgrade:
   stage: deploy-part3
   extends: .packet_pr

.gitlab-ci/terraform.yml

@@ -12,13 +12,13 @@
     # Prepare inventory
     - cp contrib/terraform/$PROVIDER/sample-inventory/cluster.tfvars .
     - ln -s contrib/terraform/$PROVIDER/hosts
-    - terraform init contrib/terraform/$PROVIDER
+    - terraform -chdir="contrib/terraform/$PROVIDER" init
     # Copy SSH keypair
     - mkdir -p ~/.ssh
     - echo "$PACKET_PRIVATE_KEY" | base64 -d > ~/.ssh/id_rsa
     - chmod 400 ~/.ssh/id_rsa
     - echo "$PACKET_PUBLIC_KEY" | base64 -d > ~/.ssh/id_rsa.pub
-    - mkdir -p group_vars
+    - mkdir -p contrib/terraform/$PROVIDER/group_vars
     # Random subnet to avoid routing conflicts
     - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
 
@@ -28,8 +28,8 @@
   tags: [light]
   only: ['master', /^pr-.*$/]
   script:
-    - terraform validate -var-file=cluster.tfvars contrib/terraform/$PROVIDER
+    - terraform -chdir="contrib/terraform/$PROVIDER" validate
-    - terraform fmt -check -diff contrib/terraform/$PROVIDER
+    - terraform -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
 
 .terraform_apply:
   extends: .terraform_install
@@ -53,44 +53,44 @@
     # Cleanup regardless of exit code
     - chronic ./tests/scripts/testcases_cleanup.sh
 
-tf-0.13.x-validate-openstack:
+tf-0.15.x-validate-openstack:
   extends: .terraform_validate
   variables:
-    TF_VERSION: $TERRAFORM_13_VERSION
+    TF_VERSION: $TERRAFORM_15_VERSION
     PROVIDER: openstack
     CLUSTER: $CI_COMMIT_REF_NAME
 
-tf-0.13.x-validate-packet:
+tf-0.15.x-validate-packet:
   extends: .terraform_validate
   variables:
-    TF_VERSION: $TERRAFORM_13_VERSION
+    TF_VERSION: $TERRAFORM_15_VERSION
     PROVIDER: packet
     CLUSTER: $CI_COMMIT_REF_NAME
 
-tf-0.13.x-validate-aws:
+tf-0.15.x-validate-aws:
   extends: .terraform_validate
   variables:
-    TF_VERSION: $TERRAFORM_13_VERSION
+    TF_VERSION: $TERRAFORM_15_VERSION
     PROVIDER: aws
     CLUSTER: $CI_COMMIT_REF_NAME
 
-tf-0.13.x-validate-exoscale:
+tf-0.15.x-validate-exoscale:
   extends: .terraform_validate
   variables:
-    TF_VERSION: $TERRAFORM_13_VERSION
+    TF_VERSION: $TERRAFORM_15_VERSION
     PROVIDER: exoscale
 
-tf-0.13.x-validate-vsphere:
+tf-0.15.x-validate-vsphere:
   extends: .terraform_validate
   variables:
-    TF_VERSION: $TERRAFORM_13_VERSION
+    TF_VERSION: $TERRAFORM_15_VERSION
     PROVIDER: vsphere
     CLUSTER: $CI_COMMIT_REF_NAME
 
-tf-0.13.x-validate-upcloud:
+tf-0.15.x-validate-upcloud:
   extends: .terraform_validate
   variables:
-    TF_VERSION: $TERRAFORM_13_VERSION
+    TF_VERSION: $TERRAFORM_15_VERSION
     PROVIDER: upcloud
     CLUSTER: $CI_COMMIT_REF_NAME
 
@@ -207,9 +207,10 @@ tf-elastx_ubuntu18-calico:
   extends: .terraform_apply
   stage: deploy-part3
   when: on_success
+  allow_failure: true
   variables:
     <<: *elastx_variables
-    TF_VERSION: $TERRAFORM_14_VERSION
+    TF_VERSION: $TERRAFORM_15_VERSION
     PROVIDER: openstack
     CLUSTER: $CI_COMMIT_REF_NAME
     ANSIBLE_TIMEOUT: "60"
@@ -235,44 +236,45 @@ tf-elastx_ubuntu18-calico:
     TF_VAR_image: ubuntu-18.04-server-latest
     TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
 
+# OVH voucher expired, commenting job until things are sorted  out
 
-tf-ovh_cleanup:
+# tf-ovh_cleanup:
-  stage: unit-tests
+#  stage: unit-tests
-  tags: [light]
+#  tags: [light]
-  image: python
+#  image: python
-  environment: ovh
+#  environment: ovh
-  variables:
+#  variables:
-    <<: *ovh_variables
+#    <<: *ovh_variables
-  before_script:
+#  before_script:
-    - pip install -r scripts/openstack-cleanup/requirements.txt
+#    - pip install -r scripts/openstack-cleanup/requirements.txt
-  script:
+#  script:
-    - ./scripts/openstack-cleanup/main.py
+#    - ./scripts/openstack-cleanup/main.py
 
-tf-ovh_ubuntu18-calico:
+# tf-ovh_ubuntu18-calico:
-  extends: .terraform_apply
+#  extends: .terraform_apply
-  when: on_success
+#  when: on_success
-  environment: ovh
+#  environment: ovh
-  variables:
+#  variables:
-    <<: *ovh_variables
+#    <<: *ovh_variables
-    TF_VERSION: $TERRAFORM_14_VERSION
+#    TF_VERSION: $TERRAFORM_14_VERSION
-    PROVIDER: openstack
+#    PROVIDER: openstack
-    CLUSTER: $CI_COMMIT_REF_NAME
+#    CLUSTER: $CI_COMMIT_REF_NAME
-    ANSIBLE_TIMEOUT: "60"
+#    ANSIBLE_TIMEOUT: "60"
-    SSH_USER: ubuntu
+#    SSH_USER: ubuntu
-    TF_VAR_number_of_k8s_masters: "0"
+#    TF_VAR_number_of_k8s_masters: "0"
-    TF_VAR_number_of_k8s_masters_no_floating_ip: "1"
+#    TF_VAR_number_of_k8s_masters_no_floating_ip: "1"
-    TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
+#    TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
-    TF_VAR_number_of_etcd: "0"
+#    TF_VAR_number_of_etcd: "0"
-    TF_VAR_number_of_k8s_nodes: "0"
+#    TF_VAR_number_of_k8s_nodes: "0"
-    TF_VAR_number_of_k8s_nodes_no_floating_ip: "1"
+#    TF_VAR_number_of_k8s_nodes_no_floating_ip: "1"
-    TF_VAR_number_of_gfs_nodes_no_floating_ip: "0"
+#    TF_VAR_number_of_gfs_nodes_no_floating_ip: "0"
-    TF_VAR_number_of_bastions: "0"
+#    TF_VAR_number_of_bastions: "0"
-    TF_VAR_number_of_k8s_masters_no_etcd: "0"
+#    TF_VAR_number_of_k8s_masters_no_etcd: "0"
-    TF_VAR_use_neutron: "0"
+#    TF_VAR_use_neutron: "0"
-    TF_VAR_floatingip_pool: "Ext-Net"
+#    TF_VAR_floatingip_pool: "Ext-Net"
-    TF_VAR_external_net: "6011fbc9-4cbf-46a4-8452-6890a340b60b"
+#    TF_VAR_external_net: "6011fbc9-4cbf-46a4-8452-6890a340b60b"
-    TF_VAR_network_name: "Ext-Net"
+#    TF_VAR_network_name: "Ext-Net"
-    TF_VAR_flavor_k8s_master: "defa64c3-bd46-43b4-858a-d93bbae0a229"    # s1-8
+#    TF_VAR_flavor_k8s_master: "defa64c3-bd46-43b4-858a-d93bbae0a229"    # s1-8
-    TF_VAR_flavor_k8s_node: "defa64c3-bd46-43b4-858a-d93bbae0a229"      # s1-8
+#    TF_VAR_flavor_k8s_node: "defa64c3-bd46-43b4-858a-d93bbae0a229"      # s1-8
-    TF_VAR_image: "Ubuntu 18.04"
+#    TF_VAR_image: "Ubuntu 18.04"
-    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
+#    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'

.gitlab-ci/vagrant.yml

@@ -11,6 +11,7 @@ molecule_tests:
     - tests/scripts/rebase.sh
     - apt-get update && apt-get install -y python3-pip
     - update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+    - python -m pip uninstall -y ansible
     - python -m pip install -r tests/requirements.txt
     - ./tests/scripts/vagrant_clean.sh
   script:
@@ -31,6 +32,7 @@ molecule_tests:
   before_script:
     - apt-get update && apt-get install -y python3-pip
     - update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+    - python -m pip uninstall -y ansible
     - python -m pip install -r tests/requirements.txt
     - ./tests/scripts/vagrant_clean.sh
   script:

Dockerfile

@@ -4,7 +4,7 @@ FROM ubuntu:bionic-20200807
 RUN apt update -y \
     && apt install -y \
     libssl-dev python3-dev sshpass apt-transport-https jq moreutils \
-    ca-certificates curl gnupg2 software-properties-common python3-pip rsync \
+    ca-certificates curl gnupg2 software-properties-common python3-pip rsync git \
     && rm -rf /var/lib/apt/lists/*
 RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
     && add-apt-repository \
@@ -14,17 +14,20 @@ RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
     && apt update -y && apt-get install --no-install-recommends -y docker-ce \
     && rm -rf /var/lib/apt/lists/*
 
+# Some tools like yamllint need this
+# Pip needs this as well at the moment to install ansible
+# (and potentially other packages)
+# See: https://github.com/pypa/pip/issues/10219
+ENV LANG=C.UTF-8
+
 WORKDIR /kubespray
 COPY . .
-RUN /usr/bin/python3 -m pip install pip -U \
+RUN /usr/bin/python3 -m pip install --no-cache-dir pip -U \
-    && /usr/bin/python3 -m pip install -r tests/requirements.txt \
+    && /usr/bin/python3 -m pip install --no-cache-dir -r tests/requirements.txt \
-    && python3 -m pip install -r requirements.txt \
+    && python3 -m pip install --no-cache-dir -r requirements.txt \
     && update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 
 RUN KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main.yaml) \
     && curl -LO https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/amd64/kubectl \
     && chmod a+x kubectl \
     && mv kubectl /usr/local/bin/kubectl
-
-# Some tools like yamllint need this
-ENV LANG=C.UTF-8

README.md

@@ -5,7 +5,7 @@
 If you have questions, check the documentation at [kubespray.io](https://kubespray.io) and join us on the [kubernetes slack](https://kubernetes.slack.com), channel **\#kubespray**.
 You can get your invite [here](http://slack.k8s.io/)
 
-- Can be deployed on **[AWS](docs/aws.md), GCE, [Azure](docs/azure.md), [OpenStack](docs/openstack.md), [vSphere](docs/vsphere.md), [Packet](docs/packet.md) (bare metal), Oracle Cloud Infrastructure (Experimental), or Baremetal**
+- Can be deployed on **[AWS](docs/aws.md), GCE, [Azure](docs/azure.md), [OpenStack](docs/openstack.md), [vSphere](docs/vsphere.md), [Equinix Metal](docs/equinix-metal.md) (bare metal), Oracle Cloud Infrastructure (Experimental), or Baremetal**
 - **Highly available** cluster
 - **Composable** (Choice of the network plugin for instance)
 - Supports most popular **Linux distributions**
@@ -32,7 +32,7 @@ CONFIG_FILE=inventory/mycluster/hosts.yaml python3 contrib/inventory_builder/inv
 
 # Review and change parameters under ``inventory/mycluster/group_vars``
 cat inventory/mycluster/group_vars/all/all.yml
-cat inventory/mycluster/group_vars/k8s_cluster/k8s_cluster.yml
+cat inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
 
 # Deploy Kubespray with Ansible Playbook - run the playbook as root
 # The option `--become` is required, as for example writing SSL keys in /etc/,
@@ -57,10 +57,10 @@ A simple way to ensure you get all the correct version of Ansible is to use the
 You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mounts/) to get the inventory and ssh key into the container, like this:
 
 ```ShellSession
-docker pull quay.io/kubespray/kubespray:v2.15.1
+docker pull quay.io/kubespray/kubespray:v2.16.0
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
   --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.15.1 bash
+  quay.io/kubespray/kubespray:v2.16.0 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```
@@ -105,7 +105,7 @@ vagrant up
 - [AWS](docs/aws.md)
 - [Azure](docs/azure.md)
 - [vSphere](docs/vsphere.md)
-- [Packet Host](docs/packet.md)
+- [Equinix Metal](docs/equinix-metal.md)
 - [Large deployments](docs/large-deployments.md)
 - [Adding/replacing a node](docs/nodes.md)
 - [Upgrades basics](docs/upgrades.md)
@@ -115,48 +115,48 @@ vagrant up
 ## Supported Linux Distributions
 
 - **Flatcar Container Linux by Kinvolk**
-- **Debian** Buster, Jessie, Stretch, Wheezy
+- **Debian** Bullseye, Buster, Jessie, Stretch
 - **Ubuntu** 16.04, 18.04, 20.04
 - **CentOS/RHEL** 7, [8](docs/centos8.md)
-- **Fedora** 32, 33
+- **Fedora** 33, 34
-- **Fedora CoreOS** (experimental: see [fcos Note](docs/fcos.md))
+- **Fedora CoreOS** (see [fcos Note](docs/fcos.md))
 - **openSUSE** Leap 15.x/Tumbleweed
 - **Oracle Linux** 7, [8](docs/centos8.md)
 - **Alma Linux** [8](docs/centos8.md)
-- **Amazon Linux 2** (experimental: see [amazon linux notes](docs/amazonlinux.md)
+- **Amazon Linux 2** (experimental: see [amazon linux notes](docs/amazonlinux.md))
 
 Note: Upstart/SysV init based OS types are not supported.
 
 ## Supported Components
 
 - Core
-  - [kubernetes](https://github.com/kubernetes/kubernetes) v1.20.7
+  - [kubernetes](https://github.com/kubernetes/kubernetes) v1.21.6
   - [etcd](https://github.com/coreos/etcd) v3.4.13
-  - [docker](https://www.docker.com/) v19.03 (see note)
+  - [docker](https://www.docker.com/) v20.10 (see note)
-  - [containerd](https://containerd.io/) v1.4.4
+  - [containerd](https://containerd.io/) v1.4.9
-  - [cri-o](http://cri-o.io/) v1.20 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS)
+  - [cri-o](http://cri-o.io/) v1.21 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
   - [cni-plugins](https://github.com/containernetworking/plugins) v0.9.1
-  - [calico](https://github.com/projectcalico/calico) v3.17.4
+  - [calico](https://github.com/projectcalico/calico) v3.19.2
   - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
-  - [cilium](https://github.com/cilium/cilium) v1.8.9
+  - [cilium](https://github.com/cilium/cilium) v1.9.10
-  - [flanneld](https://github.com/coreos/flannel) v0.13.0
+  - [flanneld](https://github.com/flannel-io/flannel) v0.14.0
-  - [kube-ovn](https://github.com/alauda/kube-ovn) v1.6.2
+  - [kube-ovn](https://github.com/alauda/kube-ovn) v1.7.2
-  - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.2.2
+  - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.3.0
-  - [multus](https://github.com/intel/multus-cni) v3.7.0
+  - [multus](https://github.com/intel/multus-cni) v3.7.2
   - [ovn4nfv](https://github.com/opnfv/ovn4nfv-k8s-plugin) v1.1.0
   - [weave](https://github.com/weaveworks/weave) v2.8.1
 - Application
   - [ambassador](https://github.com/datawire/ambassador): v1.5
   - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.0-k8s1.11
   - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.1-k8s1.11
-  - [cert-manager](https://github.com/jetstack/cert-manager) v0.16.1
+  - [cert-manager](https://github.com/jetstack/cert-manager) v1.0.4
-  - [coredns](https://github.com/coredns/coredns) v1.7.0
+  - [coredns](https://github.com/coredns/coredns) v1.8.0
-  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v0.43.0
+  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.0.0
 
 ## Container Runtime Notes
 
-- The list of available docker version is 18.09, 19.03 and 20.10. The recommended docker version is 19.03. The kubelet might break on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. yum versionlock plugin or apt pin).
+- The list of available docker version is 18.09, 19.03 and 20.10. The recommended docker version is 20.10. The kubelet might break on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. yum versionlock plugin or apt pin).
 - The cri-o version should be aligned with the respective kubernetes version (i.e. kube_version=1.20.x, crio_version=1.20)
 
 ## Requirements
@@ -239,6 +239,6 @@ See also [Network checker](docs/netcheck.md).
 
 [![Build graphs](https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/badges/master/pipeline.svg)](https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/pipelines)
 
-CI/end-to-end tests sponsored by: [CNCF](https://cncf.io), [Packet](https://www.packet.com/), [OVHcloud](https://www.ovhcloud.com/), [ELASTX](https://elastx.se/).
+CI/end-to-end tests sponsored by: [CNCF](https://cncf.io), [Equinix Metal](https://metal.equinix.com/), [OVHcloud](https://www.ovhcloud.com/), [ELASTX](https://elastx.se/).
 
 See the [test matrix](docs/test_cases.md) for details.

Vagrantfile

@@ -26,8 +26,8 @@ SUPPORTED_OS = {
   "centos-bento"        => {box: "bento/centos-7.6",           user: "vagrant"},
   "centos8"             => {box: "centos/8",                   user: "vagrant"},
   "centos8-bento"       => {box: "bento/centos-8",             user: "vagrant"},
-  "fedora32"            => {box: "fedora/32-cloud-base",       user: "vagrant"},
   "fedora33"            => {box: "fedora/33-cloud-base",       user: "vagrant"},
+  "fedora34"            => {box: "fedora/34-cloud-base",       user: "vagrant"},
   "opensuse"            => {box: "bento/opensuse-leap-15.2",   user: "vagrant"},
   "opensuse-tumbleweed" => {box: "opensuse/Tumbleweed.x86_64", user: "vagrant"},
   "oraclelinux"         => {box: "generic/oracle7",            user: "vagrant"},

ansible_version.yml

@@ -4,8 +4,10 @@
   become: no
   vars:
     minimal_ansible_version: 2.9.0
+    minimal_ansible_version_2_10: 2.10.11
     maximal_ansible_version: 2.11.0
     ansible_connection: local
+  tags: always
   tasks:
     - name: "Check {{ minimal_ansible_version }} <= Ansible version < {{ maximal_ansible_version }}"
       assert:
@@ -16,6 +18,17 @@
       tags:
         - check
 
+    - name: "Check Ansible version > {{ minimal_ansible_version_2_10 }} when using ansible 2.10"
+      assert:
+        msg: "When using Ansible 2.10, the minimum supported version is {{ minimal_ansible_version_2_10 }}"
+        that:
+          - ansible_version.string is version(minimal_ansible_version_2_10, ">=")
+          - ansible_version.string is version(maximal_ansible_version, "<")
+      when:
+        - ansible_version.string is version('2.10.0', ">=")
+      tags:
+        - check
+
     - name: "Check that python netaddr is installed"
       assert:
         msg: "Python netaddr is not present"

cluster.yml

@@ -86,8 +86,8 @@
   roles:
     - { role: kubespray-defaults }
     - { role: kubernetes/kubeadm, tags: kubeadm}
-    - { role: network_plugin, tags: network }
     - { role: kubernetes/node-label, tags: node-label }
+    - { role: network_plugin, tags: network }
 
 - hosts: calico_rr
   gather_facts: False
@@ -116,13 +116,6 @@
     - { role: kubernetes-apps/policy_controller, tags: policy-controller }
     - { role: kubernetes-apps/ingress_controller, tags: ingress-controller }
     - { role: kubernetes-apps/external_provisioner, tags: external-provisioner }
-
-- hosts: kube_control_plane
-  gather_facts: False
-  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
-  environment: "{{ proxy_disable_env }}"
-  roles:
-    - { role: kubespray-defaults }
     - { role: kubernetes-apps, tags: apps }
 
 - hosts: k8s_cluster

contrib/aws_inventory/kubespray-aws-inventory.py

@@ -69,7 +69,7 @@ class SearchEC2Tags(object):
 
         hosts[group].append(dns_name)
         hosts['_meta']['hostvars'][dns_name] = ansible_host
-        
+
     hosts['k8s_cluster'] = {'children':['kube_control_plane', 'kube_node']}
     print(json.dumps(hosts, sort_keys=True, indent=2))
 

contrib/azurerm/roles/generate-inventory/tasks/main.yml

@@ -12,3 +12,4 @@
   template:
     src: inventory.j2
     dest: "{{ playbook_dir }}/inventory"
+    mode: 0644

contrib/azurerm/roles/generate-inventory_2/tasks/main.yml

@@ -22,8 +22,10 @@
   template:
     src: inventory.j2
     dest: "{{ playbook_dir }}/inventory"
+    mode: 0644
 
 - name: Generate Load Balancer variables
   template:
     src: loadbalancer_vars.j2
     dest: "{{ playbook_dir }}/loadbalancer_vars.yml"
+    mode: 0644

contrib/azurerm/roles/generate-templates/tasks/main.yml

@@ -8,11 +8,13 @@
     path: "{{ base_dir }}"
     state: directory
     recurse: true
+    mode: 0755
 
 - name: Store json files in base_dir
   template:
     src: "{{ item }}"
     dest: "{{ base_dir }}/{{ item }}"
+    mode: 0644
   with_items:
     - network.json
     - storage.json

contrib/dind/roles/dind-cluster/tasks/main.yaml

@@ -35,6 +35,7 @@
       path-exclude=/usr/share/doc/*
       path-include=/usr/share/doc/*/copyright
     dest: /etc/dpkg/dpkg.cfg.d/01_nodoc
+    mode: 0644
   when:
     - ansible_os_family == 'Debian'
 
@@ -63,6 +64,7 @@
   copy:
     content: "{{ distro_user }} ALL=(ALL) NOPASSWD:ALL"
     dest: "/etc/sudoers.d/{{ distro_user }}"
+    mode: 0640
 
 - name: Add my pubkey to "{{ distro_user }}" user authorized keys
   authorized_key:

contrib/inventory_builder/inventory.py

@@ -48,7 +48,7 @@ ROLES = ['all', 'kube_control_plane', 'kube_node', 'etcd', 'k8s_cluster',
          'calico_rr']
 PROTECTED_NAMES = ROLES
 AVAILABLE_COMMANDS = ['help', 'print_cfg', 'print_ips', 'print_hostnames',
-                      'load']
+                      'load', 'add']
 _boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
                    '0': False, 'no': False, 'false': False, 'off': False}
 yaml = YAML()
@@ -82,22 +82,35 @@ class KubesprayInventory(object):
     def __init__(self, changed_hosts=None, config_file=None):
         self.config_file = config_file
         self.yaml_config = {}
-        if self.config_file:
+        loadPreviousConfig = False
+        # See whether there are any commands to process
+        if changed_hosts and changed_hosts[0] in AVAILABLE_COMMANDS:
+            if changed_hosts[0] == "add":
+                loadPreviousConfig = True
+                changed_hosts = changed_hosts[1:]
+            else:
+                self.parse_command(changed_hosts[0], changed_hosts[1:])
+                sys.exit(0)
+
+        # If the user wants to remove a node, we need to load the config anyway
+        if changed_hosts and changed_hosts[0][0] == "-":
+            loadPreviousConfig = True
+
+        if self.config_file and loadPreviousConfig:  # Load previous YAML file
             try:
                 self.hosts_file = open(config_file, 'r')
-                self.yaml_config = yaml.load_all(self.hosts_file)
+                self.yaml_config = yaml.load(self.hosts_file)
-            except OSError:
+            except OSError as e:
-                pass
+                # I am assuming we are catching "cannot open file" exceptions
-
+                print(e)
-        if changed_hosts and changed_hosts[0] in AVAILABLE_COMMANDS:
+                sys.exit(1)
-            self.parse_command(changed_hosts[0], changed_hosts[1:])
-            sys.exit(0)
 
         self.ensure_required_groups(ROLES)
 
         if changed_hosts:
             changed_hosts = self.range2ips(changed_hosts)
-            self.hosts = self.build_hostnames(changed_hosts)
+            self.hosts = self.build_hostnames(changed_hosts,
+                                              loadPreviousConfig)
             self.purge_invalid_hosts(self.hosts.keys(), PROTECTED_NAMES)
             self.set_all(self.hosts)
             self.set_k8s_cluster()
@@ -158,17 +171,29 @@ class KubesprayInventory(object):
         except IndexError:
             raise ValueError("Host name must end in an integer")
 
-    def build_hostnames(self, changed_hosts):
+    # Keeps already specified hosts,
+    # and adds or removes the hosts provided as an argument
+    def build_hostnames(self, changed_hosts, loadPreviousConfig=False):
         existing_hosts = OrderedDict()
         highest_host_id = 0
-        try:
+        # Load already existing hosts from the YAML
-            for host in self.yaml_config['all']['hosts']:
+        if loadPreviousConfig:
-                existing_hosts[host] = self.yaml_config['all']['hosts'][host]
+            try:
-                host_id = self.get_host_id(host)
+                for host in self.yaml_config['all']['hosts']:
-                if host_id > highest_host_id:
+                    # Read configuration of an existing host
-                    highest_host_id = host_id
+                    hostConfig = self.yaml_config['all']['hosts'][host]
-        except Exception:
+                    existing_hosts[host] = hostConfig
-            pass
+                    # If the existing host seems
+                    # to have been created automatically, detect its ID
+                    if host.startswith(HOST_PREFIX):
+                        host_id = self.get_host_id(host)
+                        if host_id > highest_host_id:
+                            highest_host_id = host_id
+            except Exception as e:
+                # I am assuming we are catching automatically
+                # created hosts without IDs
+                print(e)
+                sys.exit(1)
 
         # FIXME(mattymo): Fix condition where delete then add reuses highest id
         next_host_id = highest_host_id + 1
@@ -176,6 +201,7 @@ class KubesprayInventory(object):
 
         all_hosts = existing_hosts.copy()
         for host in changed_hosts:
+            # Delete the host from config the hostname/IP has a "-" prefix
             if host[0] == "-":
                 realhost = host[1:]
                 if self.exists_hostname(all_hosts, realhost):
@@ -184,6 +210,8 @@ class KubesprayInventory(object):
                 elif self.exists_ip(all_hosts, realhost):
                     self.debug("Marked {0} for deletion.".format(realhost))
                     self.delete_host_by_ip(all_hosts, realhost)
+            # Host/Argument starts with a digit,
+            # then we assume its an IP address
             elif host[0].isdigit():
                 if ',' in host:
                     ip, access_ip = host.split(',')
@@ -203,11 +231,15 @@ class KubesprayInventory(object):
                     next_host = subprocess.check_output(cmd, shell=True)
                     next_host = next_host.strip().decode('ascii')
                 else:
+                    # Generates a hostname because we have only an IP address
                     next_host = "{0}{1}".format(HOST_PREFIX, next_host_id)
                     next_host_id += 1
+                # Uses automatically generated node name
+                # in case we dont provide it.
                 all_hosts[next_host] = {'ansible_host': access_ip,
                                         'ip': ip,
                                         'access_ip': access_ip}
+            # Host/Argument starts with a letter, then we assume its a hostname
             elif host[0].isalpha():
                 if ',' in host:
                     try:
@@ -226,6 +258,7 @@ class KubesprayInventory(object):
                                        'access_ip': access_ip}
         return all_hosts
 
+    # Expand IP ranges into individual addresses
     def range2ips(self, hosts):
         reworked_hosts = []
 
@@ -394,9 +427,11 @@ help - Display this message
 print_cfg - Write inventory file to stdout
 print_ips - Write a space-delimited list of IPs from "all" group
 print_hostnames - Write a space-delimited list of Hostnames from "all" group
+add - Adds specified hosts into an already existing inventory
 
 Advanced usage:
-Add another host after initial creation: inventory.py 10.10.1.5
+Create new or overwrite old inventory file: inventory.py 10.10.1.5
+Add another host after initial creation: inventory.py add 10.10.1.6
 Add range of hosts: inventory.py 10.10.1.3-10.10.1.5
 Add hosts with different ip and access ip: inventory.py 10.0.0.1,192.168.10.1 10.0.0.2,192.168.10.2 10.0.0.3,192.168.10.3
 Add hosts with a specific hostname, ip, and optional access ip: first,10.0.0.1,192.168.10.1 second,10.0.0.2 last,10.0.0.3
@@ -430,6 +465,7 @@ def main(argv=None):
     if not argv:
         argv = sys.argv[1:]
     KubesprayInventory(argv, CONFIG_FILE)
+    return 0
 
 
 if __name__ == "__main__":

contrib/inventory_builder/tests/test_inventory.py

@@ -67,23 +67,14 @@ class TestInventory(unittest.TestCase):
             self.assertRaisesRegex(ValueError, "Host name must end in an",
                                    self.inv.get_host_id, hostname)
 
-    def test_build_hostnames_add_one(self):
-        changed_hosts = ['10.90.0.2']
-        expected = OrderedDict([('node1',
-                                 {'ansible_host': '10.90.0.2',
-                                  'ip': '10.90.0.2',
-                                  'access_ip': '10.90.0.2'})])
-        result = self.inv.build_hostnames(changed_hosts)
-        self.assertEqual(expected, result)
-
     def test_build_hostnames_add_duplicate(self):
         changed_hosts = ['10.90.0.2']
-        expected = OrderedDict([('node1',
+        expected = OrderedDict([('node3',
                                  {'ansible_host': '10.90.0.2',
                                   'ip': '10.90.0.2',
                                   'access_ip': '10.90.0.2'})])
         self.inv.yaml_config['all']['hosts'] = expected
-        result = self.inv.build_hostnames(changed_hosts)
+        result = self.inv.build_hostnames(changed_hosts, True)
         self.assertEqual(expected, result)
 
     def test_build_hostnames_add_two(self):
@@ -99,6 +90,30 @@ class TestInventory(unittest.TestCase):
         result = self.inv.build_hostnames(changed_hosts)
         self.assertEqual(expected, result)
 
+    def test_build_hostnames_add_three(self):
+        changed_hosts = ['10.90.0.2', '10.90.0.3', '10.90.0.4']
+        expected = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'}),
+            ('node3', {'ansible_host': '10.90.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '10.90.0.4'})])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_one(self):
+        changed_hosts = ['10.90.0.2']
+        expected = OrderedDict([('node1',
+                                 {'ansible_host': '10.90.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '10.90.0.2'})])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
     def test_build_hostnames_delete_first(self):
         changed_hosts = ['-10.90.0.2']
         existing_hosts = OrderedDict([
@@ -113,7 +128,24 @@ class TestInventory(unittest.TestCase):
             ('node2', {'ansible_host': '10.90.0.3',
                        'ip': '10.90.0.3',
                        'access_ip': '10.90.0.3'})])
-        result = self.inv.build_hostnames(changed_hosts)
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_delete_by_hostname(self):
+        changed_hosts = ['-node1']
+        existing_hosts = OrderedDict([
+            ('node1', {'ansible_host': '10.90.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '10.90.0.2'}),
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        self.inv.yaml_config['all']['hosts'] = existing_hosts
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '10.90.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '10.90.0.3'})])
+        result = self.inv.build_hostnames(changed_hosts, True)
         self.assertEqual(expected, result)
 
     def test_exists_hostname_positive(self):
@@ -313,7 +345,7 @@ class TestInventory(unittest.TestCase):
         self.assertRaisesRegex(Exception, "Range of ip_addresses isn't valid",
                                self.inv.range2ips, host_range)
 
-    def test_build_hostnames_different_ips_add_one(self):
+    def test_build_hostnames_create_with_one_different_ips(self):
         changed_hosts = ['10.90.0.2,192.168.0.2']
         expected = OrderedDict([('node1',
                                  {'ansible_host': '192.168.0.2',
@@ -322,17 +354,7 @@ class TestInventory(unittest.TestCase):
         result = self.inv.build_hostnames(changed_hosts)
         self.assertEqual(expected, result)
 
-    def test_build_hostnames_different_ips_add_duplicate(self):
+    def test_build_hostnames_create_with_two_different_ips(self):
-        changed_hosts = ['10.90.0.2,192.168.0.2']
-        expected = OrderedDict([('node1',
-                                 {'ansible_host': '192.168.0.2',
-                                  'ip': '10.90.0.2',
-                                  'access_ip': '192.168.0.2'})])
-        self.inv.yaml_config['all']['hosts'] = expected
-        result = self.inv.build_hostnames(changed_hosts)
-        self.assertEqual(expected, result)
-
-    def test_build_hostnames_different_ips_add_two(self):
         changed_hosts = ['10.90.0.2,192.168.0.2', '10.90.0.3,192.168.0.3']
         expected = OrderedDict([
             ('node1', {'ansible_host': '192.168.0.2',
@@ -341,6 +363,210 @@ class TestInventory(unittest.TestCase):
             ('node2', {'ansible_host': '192.168.0.3',
                        'ip': '10.90.0.3',
                        'access_ip': '192.168.0.3'})])
-        self.inv.yaml_config['all']['hosts'] = OrderedDict()
         result = self.inv.build_hostnames(changed_hosts)
         self.assertEqual(expected, result)
+
+    def test_build_hostnames_create_with_three_different_ips(self):
+        changed_hosts = ['10.90.0.2,192.168.0.2',
+                         '10.90.0.3,192.168.0.3',
+                         '10.90.0.4,192.168.0.4']
+        expected = OrderedDict([
+            ('node1', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node2', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node3', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'})])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_overwrite_one_with_different_ips(self):
+        changed_hosts = ['10.90.0.2,192.168.0.2']
+        expected = OrderedDict([('node1',
+                                 {'ansible_host': '192.168.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '192.168.0.2'})])
+        existing = OrderedDict([('node5',
+                                 {'ansible_host': '192.168.0.5',
+                                  'ip': '10.90.0.5',
+                                  'access_ip': '192.168.0.5'})])
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_overwrite_three_with_different_ips(self):
+        changed_hosts = ['10.90.0.2,192.168.0.2']
+        expected = OrderedDict([('node1',
+                                 {'ansible_host': '192.168.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '192.168.0.2'})])
+        existing = OrderedDict([
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'}),
+            ('node5', {'ansible_host': '192.168.0.5',
+                       'ip': '10.90.0.5',
+                       'access_ip': '192.168.0.5'})])
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_different_ips_add_duplicate(self):
+        changed_hosts = ['10.90.0.2,192.168.0.2']
+        expected = OrderedDict([('node3',
+                                 {'ansible_host': '192.168.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '192.168.0.2'})])
+        existing = expected
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_two_different_ips_into_one_existing(self):
+        changed_hosts = ['10.90.0.3,192.168.0.3', '10.90.0.4,192.168.0.4']
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'})])
+
+        existing = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'})])
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_two_different_ips_into_two_existing(self):
+        changed_hosts = ['10.90.0.4,192.168.0.4', '10.90.0.5,192.168.0.5']
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'}),
+            ('node5', {'ansible_host': '192.168.0.5',
+                       'ip': '10.90.0.5',
+                       'access_ip': '192.168.0.5'})])
+
+        existing = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'})])
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_add_two_different_ips_into_three_existing(self):
+        changed_hosts = ['10.90.0.5,192.168.0.5', '10.90.0.6,192.168.0.6']
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'}),
+            ('node5', {'ansible_host': '192.168.0.5',
+                       'ip': '10.90.0.5',
+                       'access_ip': '192.168.0.5'}),
+            ('node6', {'ansible_host': '192.168.0.6',
+                       'ip': '10.90.0.6',
+                       'access_ip': '192.168.0.6'})])
+
+        existing = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'})])
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)
+
+    # Add two IP addresses into a config that has
+    # three already defined IP addresses. One of the IP addresses
+    # is a duplicate.
+    def test_build_hostnames_add_two_duplicate_one_overlap(self):
+        changed_hosts = ['10.90.0.4,192.168.0.4', '10.90.0.5,192.168.0.5']
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'}),
+            ('node5', {'ansible_host': '192.168.0.5',
+                       'ip': '10.90.0.5',
+                       'access_ip': '192.168.0.5'})])
+
+        existing = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'})])
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)
+
+    # Add two duplicate IP addresses into a config that has
+    # three already defined IP addresses
+    def test_build_hostnames_add_two_duplicate_two_overlap(self):
+        changed_hosts = ['10.90.0.3,192.168.0.3', '10.90.0.4,192.168.0.4']
+        expected = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'})])
+
+        existing = OrderedDict([
+            ('node2', {'ansible_host': '192.168.0.2',
+                       'ip': '10.90.0.2',
+                       'access_ip': '192.168.0.2'}),
+            ('node3', {'ansible_host': '192.168.0.3',
+                       'ip': '10.90.0.3',
+                       'access_ip': '192.168.0.3'}),
+            ('node4', {'ansible_host': '192.168.0.4',
+                       'ip': '10.90.0.4',
+                       'access_ip': '192.168.0.4'})])
+        self.inv.yaml_config['all']['hosts'] = existing
+        result = self.inv.build_hostnames(changed_hosts, True)
+        self.assertEqual(expected, result)

contrib/kvm-setup/roles/kvm-setup/tasks/main.yml

@@ -1,7 +1,7 @@
 ---
 
 - name: Install required packages
-  yum:
+  package:
     name: "{{ item }}"
     state: present
   with_items:

contrib/kvm-setup/roles/kvm-setup/tasks/user.yml

@@ -11,6 +11,7 @@
     state: directory
     owner: "{{ k8s_deployment_user }}"
     group: "{{ k8s_deployment_user }}"
+    mode: 0700
 
 - name: Configure sudo for deployment user
   copy:

contrib/network-storage/glusterfs/inventory.example

@@ -11,8 +11,8 @@
 # ## Set disk_volume_device_1 to desired device for gluster brick, if different to /dev/vdb (default).
 # ## As in the previous case, you can set ip to give direct communication on internal IPs
 # gfs_node1 ansible_ssh_host=95.54.0.18 # disk_volume_device_1=/dev/vdc  ip=10.3.0.7
-# gfs_node2 ansible_ssh_host=95.54.0.19 # disk_volume_device_1=/dev/vdc  ip=10.3.0.8 
+# gfs_node2 ansible_ssh_host=95.54.0.19 # disk_volume_device_1=/dev/vdc  ip=10.3.0.8
-# gfs_node3 ansible_ssh_host=95.54.0.20 # disk_volume_device_1=/dev/vdc  ip=10.3.0.9 
+# gfs_node3 ansible_ssh_host=95.54.0.20 # disk_volume_device_1=/dev/vdc  ip=10.3.0.9
 
 # [kube_control_plane]
 # node1

contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/setup-RedHat.yml

@@ -1,10 +1,10 @@
 ---
 - name: Install Prerequisites
-  yum: name={{ item }}  state=present
+  package: name={{ item }}  state=present
   with_items:
     - "centos-release-gluster{{ glusterfs_default_release }}"
 
 - name: Install Packages
-  yum: name={{ item }}  state=present
+  package: name={{ item }}  state=present
   with_items:
     - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/main.yml

@@ -9,7 +9,7 @@
   when: ansible_os_family == "Debian"
 
 - name: install xfs RedHat
-  yum: name=xfsprogs state=present
+  package: name=xfsprogs state=present
   when: ansible_os_family == "RedHat"
 
 # Format external volumes in xfs
@@ -82,6 +82,7 @@
   template:
     dest: "{{ gluster_mount_dir }}/.test-file.txt"
     src: test-file.txt
+    mode: 0644
   when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
 
 - name: Unmount glusterfs

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/setup-RedHat.yml

@@ -1,11 +1,11 @@
 ---
 - name: Install Prerequisites
-  yum: name={{ item }}  state=present
+  package: name={{ item }}  state=present
   with_items:
     - "centos-release-gluster{{ glusterfs_default_release }}"
 
 - name: Install Packages
-  yum: name={{ item }}  state=present
+  package: name={{ item }}  state=present
   with_items:
     - glusterfs-server
     - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/server/tests/test.yml

@@ -1,5 +0,0 @@
----
-- hosts: all
-
-  roles:
-    - role_under_test

contrib/network-storage/heketi/roles/prepare/tasks/main.yml

@@ -11,7 +11,7 @@
 
 - name: "Install glusterfs mount utils (RedHat)"
   become: true
-  yum:
+  package:
     name: "glusterfs-fuse"
     state: "present"
   when: "ansible_os_family == 'RedHat'"

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml

@@ -1,7 +1,10 @@
 ---
 - name: "Kubernetes Apps | Lay Down Heketi Bootstrap"
   become: true
-  template: { src: "heketi-bootstrap.json.j2", dest: "{{ kube_config_dir }}/heketi-bootstrap.json" }
+  template:
+    src: "heketi-bootstrap.json.j2"
+    dest: "{{ kube_config_dir }}/heketi-bootstrap.json"
+    mode: 0640
   register: "rendering"
 - name: "Kubernetes Apps | Install and configure Heketi Bootstrap"
   kube:

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml

@@ -10,6 +10,7 @@
   template:
     src: "topology.json.j2"
     dest: "{{ kube_config_dir }}/topology.json"
+    mode: 0644
 - name: "Copy topology configuration into container."
   changed_when: false
   command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ initial_heketi_pod_name }}:/tmp/topology.json"

contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml

@@ -1,6 +1,9 @@
 ---
 - name: "Kubernetes Apps | Lay Down GlusterFS Daemonset"
-  template: { src: "glusterfs-daemonset.json.j2", dest: "{{ kube_config_dir }}/glusterfs-daemonset.json" }
+  template:
+    src: "glusterfs-daemonset.json.j2"
+    dest: "{{ kube_config_dir }}/glusterfs-daemonset.json"
+    mode: 0644
   become: true
   register: "rendering"
 - name: "Kubernetes Apps | Install and configure GlusterFS daemonset"
@@ -27,7 +30,10 @@
   delay: 5
 
 - name: "Kubernetes Apps | Lay Down Heketi Service Account"
-  template: { src: "heketi-service-account.json.j2", dest: "{{ kube_config_dir }}/heketi-service-account.json" }
+  template:
+    src: "heketi-service-account.json.j2"
+    dest: "{{ kube_config_dir }}/heketi-service-account.json"
+    mode: 0644
   become: true
   register: "rendering"
 - name: "Kubernetes Apps | Install and configure Heketi Service Account"

contrib/network-storage/heketi/roles/provision/tasks/heketi.yml

@@ -4,6 +4,7 @@
   template:
     src: "heketi-deployment.json.j2"
     dest: "{{ kube_config_dir }}/heketi-deployment.json"
+    mode: 0644
   register: "rendering"
 
 - name: "Kubernetes Apps | Install and configure Heketi"

contrib/network-storage/heketi/roles/provision/tasks/secret.yml

@@ -5,7 +5,7 @@
   changed_when: false
 
 - name: "Kubernetes Apps | Deploy cluster role binding."
-  when: "clusterrolebinding_state.stdout == \"\""
+  when: "clusterrolebinding_state.stdout | length > 0"
   command: "{{ bin_dir }}/kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account"
 
 - name: Get clusterrolebindings again
@@ -15,7 +15,7 @@
 
 - name: Make sure that clusterrolebindings are present now
   assert:
-    that: "clusterrolebinding_state.stdout != \"\""
+    that: "clusterrolebinding_state.stdout | length > 0"
     msg: "Cluster role binding is not present."
 
 - name: Get the heketi-config-secret secret
@@ -28,9 +28,10 @@
   template:
     src: "heketi.json.j2"
     dest: "{{ kube_config_dir }}/heketi.json"
+    mode: 0644
 
 - name: "Deploy Heketi config secret"
-  when: "secret_state.stdout == \"\""
+  when: "secret_state.stdout | length > 0"
   command: "{{ bin_dir }}/kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json"
 
 - name: Get the heketi-config-secret secret again

contrib/network-storage/heketi/roles/provision/tasks/storage.yml

@@ -2,7 +2,10 @@
 - name: "Kubernetes Apps | Lay Down Heketi Storage"
   become: true
   vars: { nodes: "{{ groups['heketi-node'] }}" }
-  template: { src: "heketi-storage.json.j2", dest: "{{ kube_config_dir }}/heketi-storage.json" }
+  template:
+    src: "heketi-storage.json.j2"
+    dest: "{{ kube_config_dir }}/heketi-storage.json"
+    mode: 0644
   register: "rendering"
 - name: "Kubernetes Apps | Install and configure Heketi Storage"
   kube:

contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml

@@ -16,6 +16,7 @@
   template:
     src: "storageclass.yml.j2"
     dest: "{{ kube_config_dir }}/storageclass.yml"
+    mode: 0644
   register: "rendering"
 - name: "Kubernetes Apps | Install and configure Storace Class"
   kube:

contrib/network-storage/heketi/roles/provision/tasks/topology.yml

@@ -10,6 +10,7 @@
   template:
     src: "topology.json.j2"
     dest: "{{ kube_config_dir }}/topology.json"
+    mode: 0644
 - name: "Copy topology configuration into container."  # noqa 503
   when: "rendering.changed"
   command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ heketi_pod_name }}:/tmp/topology.json"

contrib/network-storage/heketi/roles/tear-down-disks/tasks/main.yml

@@ -1,7 +1,7 @@
 ---
 - name: "Install lvm utils (RedHat)"
   become: true
-  yum:
+  package:
     name: "lvm2"
     state: "present"
   when: "ansible_os_family == 'RedHat'"
@@ -19,7 +19,7 @@
   become: true
   shell: "pvs {{ disk_volume_device_1 }} --option vg_name | tail -n+2"
   register: "volume_groups"
-  ignore_errors: true
+  ignore_errors: true   # noqa ignore-errors
   changed_when: false
 
 - name: "Remove volume groups."  # noqa 301
@@ -35,11 +35,11 @@
     PATH: "{{ ansible_env.PATH }}:/sbin"  # Make sure we can workaround RH / CentOS conservative path management
   become: true
   command: "pvremove {{ disk_volume_device_1 }} --yes"
-  ignore_errors: true
+  ignore_errors: true   # noqa ignore-errors
 
 - name: "Remove lvm utils (RedHat)"
   become: true
-  yum:
+  package:
     name: "lvm2"
     state: "absent"
   when: "ansible_os_family == 'RedHat' and heketi_remove_lvm"

contrib/network-storage/heketi/roles/tear-down/tasks/main.yml

@@ -1,51 +1,51 @@
 ---
-- name: "Remove storage class."  # noqa 301
+- name: Remove storage class.  # noqa 301
   command: "{{ bin_dir }}/kubectl delete storageclass gluster"
-  ignore_errors: true
+  ignore_errors: true  # noqa ignore-errors
-- name: "Tear down heketi."  # noqa 301
+- name: Tear down heketi.  # noqa 301
   command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\""
-  ignore_errors: true
+  ignore_errors: true  # noqa ignore-errors
-- name: "Tear down heketi."  # noqa 301
+- name: Tear down heketi.  # noqa 301
   command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\""
-  ignore_errors: true
+  ignore_errors: true  # noqa ignore-errors
-- name: "Tear down bootstrap."
+- name: Tear down bootstrap.
   include_tasks: "../../provision/tasks/bootstrap/tear-down.yml"
-- name: "Ensure there is nothing left over."  # noqa 301
+- name: Ensure there is nothing left over.  # noqa 301
   command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\" -o=json"
   register: "heketi_result"
   until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0"
   retries: 60
   delay: 5
-- name: "Ensure there is nothing left over."  # noqa 301
+- name: Ensure there is nothing left over.  # noqa 301
   command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\" -o=json"
   register: "heketi_result"
   until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0"
   retries: 60
   delay: 5
-- name: "Tear down glusterfs."  # noqa 301
+- name: Tear down glusterfs.  # noqa 301
   command: "{{ bin_dir }}/kubectl delete daemonset.extensions/glusterfs"
-  ignore_errors: true
+  ignore_errors: true  # noqa ignore-errors
-- name: "Remove heketi storage service."  # noqa 301
+- name: Remove heketi storage service.  # noqa 301
   command: "{{ bin_dir }}/kubectl delete service heketi-storage-endpoints"
-  ignore_errors: true
+  ignore_errors: true  # noqa ignore-errors
-- name: "Remove heketi gluster role binding"  # noqa 301
+- name: Remove heketi gluster role binding  # noqa 301
   command: "{{ bin_dir }}/kubectl delete clusterrolebinding heketi-gluster-admin"
-  ignore_errors: true
+  ignore_errors: true  # noqa ignore-errors
-- name: "Remove heketi config secret"  # noqa 301
+- name: Remove heketi config secret  # noqa 301
   command: "{{ bin_dir }}/kubectl delete secret heketi-config-secret"
-  ignore_errors: true
+  ignore_errors: true  # noqa ignore-errors
-- name: "Remove heketi db backup"  # noqa 301
+- name: Remove heketi db backup  # noqa 301
   command: "{{ bin_dir }}/kubectl delete secret heketi-db-backup"
-  ignore_errors: true
+  ignore_errors: true  # noqa ignore-errors
-- name: "Remove heketi service account"  # noqa 301
+- name: Remove heketi service account  # noqa 301
   command: "{{ bin_dir }}/kubectl delete serviceaccount heketi-service-account"
-  ignore_errors: true
+  ignore_errors: true  # noqa ignore-errors
-- name: "Get secrets"
+- name: Get secrets
   command: "{{ bin_dir }}/kubectl get secrets --output=\"json\""
   register: "secrets"
   changed_when: false
-- name: "Remove heketi storage secret"
+- name: Remove heketi storage secret
   vars: { storage_query: "items[?metadata.annotations.\"kubernetes.io/service-account.name\"=='heketi-service-account'].metadata.name|[0]" }
   command: "{{ bin_dir }}/kubectl delete secret {{ secrets.stdout|from_json|json_query(storage_query) }}"
   when: "storage_query is defined"
-  ignore_errors: true
+  ignore_errors: true  # noqa ignore-errors

contrib/offline/manage-offline-container-images.sh

@@ -100,15 +100,35 @@ function register_container_images() {
 
 	tar -zxvf ${IMAGE_TAR_FILE}
 	sudo docker load -i ${IMAGE_DIR}/registry-latest.tar
-	sudo docker run --restart=always -d -p 5000:5000 --name registry registry:latest
 	set +e
-
+	sudo docker container inspect registry >/dev/null 2>&1
+	if [ $? -ne 0 ]; then
+		sudo docker run --restart=always -d -p 5000:5000 --name registry registry:latest
+	fi
 	set -e
+
 	while read -r line; do
 		file_name=$(echo ${line} | awk '{print $1}')
-		org_image=$(echo ${line} | awk '{print $2}')
+		raw_image=$(echo ${line} | awk '{print $2}')
-		new_image="${LOCALHOST_NAME}:5000/${org_image}"
+		new_image="${LOCALHOST_NAME}:5000/${raw_image}"
-		image_id=$(tar -tf ${IMAGE_DIR}/${file_name} | grep "\.json" | grep -v manifest.json | sed s/"\.json"//)
+		org_image=$(sudo docker load -i ${IMAGE_DIR}/${file_name} | head -n1 | awk '{print $3}')
+		image_id=$(sudo docker image inspect ${org_image} | grep "\"Id\":" | awk -F: '{print $3}'| sed s/'\",'//)
+		if [ -z "${file_name}" ]; then
+			echo "Failed to get file_name for line ${line}"
+			exit 1
+		fi
+		if [ -z "${raw_image}" ]; then
+			echo "Failed to get raw_image for line ${line}"
+			exit 1
+		fi
+		if [ -z "${org_image}" ]; then
+			echo "Failed to get org_image for line ${line}"
+			exit 1
+		fi
+		if [ -z "${image_id}" ]; then
+			echo "Failed to get image_id for file ${file_name}"
+			exit 1
+		fi
 		sudo docker load -i ${IMAGE_DIR}/${file_name}
 		sudo docker tag  ${image_id} ${new_image}
 		sudo docker push ${new_image}

contrib/packaging/rpm/kubespray.spec

@@ -9,8 +9,8 @@ Summary:        Ansible modules for installing Kubernetes
 
 Group:          System Environment/Libraries
 License:        ASL 2.0
-Url:            https://github.com/kubernetes-incubator/kubespray
+Url:            https://github.com/kubernetes-sigs/kubespray
-Source0:        https://github.com/kubernetes-incubator/kubespray/archive/%{upstream_version}.tar.gz#/%{name}-%{release}.tar.gz
+Source0:        https://github.com/kubernetes-sigs/kubespray/archive/%{upstream_version}.tar.gz#/%{name}-%{release}.tar.gz
 
 BuildArch:      noarch
 BuildRequires:  git

contrib/terraform/aws/.gitignore

@@ -1,2 +1,3 @@
 *.tfstate*
+.terraform.lock.hcl
 .terraform

contrib/terraform/aws/create-infrastructure.tf

@@ -20,7 +20,7 @@ module "aws-vpc" {
 
   aws_cluster_name         = var.aws_cluster_name
   aws_vpc_cidr_block       = var.aws_vpc_cidr_block
-  aws_avail_zones          = slice(data.aws_availability_zones.available.names, 0, 2)
+  aws_avail_zones          = slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names))
   aws_cidr_subnets_private = var.aws_cidr_subnets_private
   aws_cidr_subnets_public  = var.aws_cidr_subnets_public
   default_tags             = var.default_tags
@@ -31,7 +31,7 @@ module "aws-elb" {
 
   aws_cluster_name      = var.aws_cluster_name
   aws_vpc_id            = module.aws-vpc.aws_vpc_id
-  aws_avail_zones       = slice(data.aws_availability_zones.available.names, 0, 2)
+  aws_avail_zones       = slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names))
   aws_subnet_ids_public = module.aws-vpc.aws_subnet_ids_public
   aws_elb_api_port      = var.aws_elb_api_port
   k8s_secure_api_port   = var.k8s_secure_api_port
@@ -52,9 +52,9 @@ module "aws-iam" {
 resource "aws_instance" "bastion-server" {
   ami                         = data.aws_ami.distro.id
   instance_type               = var.aws_bastion_size
-  count                       = length(var.aws_cidr_subnets_public)
+  count                       = var.aws_bastion_num
   associate_public_ip_address = true
-  availability_zone           = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
+  availability_zone           = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
   subnet_id                   = element(module.aws-vpc.aws_subnet_ids_public, count.index)
 
   vpc_security_group_ids = module.aws-vpc.aws_security_group
@@ -79,11 +79,15 @@ resource "aws_instance" "k8s-master" {
 
   count = var.aws_kube_master_num
 
-  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
+  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
   subnet_id         = element(module.aws-vpc.aws_subnet_ids_private, count.index)
 
   vpc_security_group_ids = module.aws-vpc.aws_security_group
 
+  root_block_device {
+    volume_size = var.aws_kube_master_disk_size
+  }
+
   iam_instance_profile = module.aws-iam.kube_control_plane-profile
   key_name             = var.AWS_SSH_KEY_NAME
 
@@ -106,11 +110,15 @@ resource "aws_instance" "k8s-etcd" {
 
   count = var.aws_etcd_num
 
-  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
+  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
   subnet_id         = element(module.aws-vpc.aws_subnet_ids_private, count.index)
 
   vpc_security_group_ids = module.aws-vpc.aws_security_group
 
+  root_block_device {
+    volume_size = var.aws_etcd_disk_size
+  }
+
   key_name = var.AWS_SSH_KEY_NAME
 
   tags = merge(var.default_tags, tomap({
@@ -126,11 +134,15 @@ resource "aws_instance" "k8s-worker" {
 
   count = var.aws_kube_worker_num
 
-  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
+  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
   subnet_id         = element(module.aws-vpc.aws_subnet_ids_private, count.index)
 
   vpc_security_group_ids = module.aws-vpc.aws_security_group
 
+  root_block_device {
+    volume_size = var.aws_kube_worker_disk_size
+  }
+
   iam_instance_profile = module.aws-iam.kube-worker-profile
   key_name             = var.AWS_SSH_KEY_NAME
 
@@ -152,10 +164,10 @@ data "template_file" "inventory" {
     public_ip_address_bastion = join("\n", formatlist("bastion ansible_host=%s", aws_instance.bastion-server.*.public_ip))
     connection_strings_master = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-master.*.private_dns, aws_instance.k8s-master.*.private_ip))
     connection_strings_node   = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-worker.*.private_dns, aws_instance.k8s-worker.*.private_ip))
-    connection_strings_etcd   = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-etcd.*.private_dns, aws_instance.k8s-etcd.*.private_ip))
     list_master               = join("\n", aws_instance.k8s-master.*.private_dns)
     list_node                 = join("\n", aws_instance.k8s-worker.*.private_dns)
-    list_etcd                 = join("\n", aws_instance.k8s-etcd.*.private_dns)
+    connection_strings_etcd   = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-etcd.*.private_dns, aws_instance.k8s-etcd.*.private_ip))
+    list_etcd                 = join("\n", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_dns) : (aws_instance.k8s-master.*.private_dns)))
     elb_api_fqdn              = "apiserver_loadbalancer_domain_name=\"${module.aws-elb.aws_elb_api_fqdn}\""
   }
 }

contrib/terraform/aws/output.tf

@@ -11,7 +11,7 @@ output "workers" {
 }
 
 output "etcd" {
-  value = join("\n", aws_instance.k8s-etcd.*.private_ip)
+  value = join("\n", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_ip) : (aws_instance.k8s-master.*.private_ip)))
 }
 
 output "aws_elb_api_fqdn" {

contrib/terraform/aws/sample-inventory/cluster.tfvars

@@ -9,6 +9,8 @@ aws_cidr_subnets_private = ["10.250.192.0/20", "10.250.208.0/20"]
 aws_cidr_subnets_public = ["10.250.224.0/20", "10.250.240.0/20"]
 
 #Bastion Host
+aws_bastion_num = 1
+
 aws_bastion_size = "t2.medium"
 
 #Kubernetes Cluster
@@ -17,22 +19,26 @@ aws_kube_master_num = 3
 
 aws_kube_master_size = "t2.medium"
 
+aws_kube_master_disk_size = 50
+
 aws_etcd_num = 3
 
 aws_etcd_size = "t2.medium"
 
+aws_etcd_disk_size = 50
+
 aws_kube_worker_num = 4
 
 aws_kube_worker_size = "t2.medium"
 
+aws_kube_worker_disk_size = 50
+
 #Settings AWS ELB
 
 aws_elb_api_port = 6443
 
 k8s_secure_api_port = 6443
 
-kube_insecure_apiserver_address = "0.0.0.0"
-
 default_tags = {
   #  Env = "devtest"  #  Product = "kubernetes"
 }

contrib/terraform/aws/templates/inventory.tpl

@@ -10,19 +10,18 @@ ${public_ip_address_bastion}
 [kube_control_plane]
 ${list_master}
 
-
 [kube_node]
 ${list_node}
 
-
 [etcd]
 ${list_etcd}
 
+[calico_rr]
 
 [k8s_cluster:children]
 kube_node
 kube_control_plane
-
+calico_rr
 
 [k8s_cluster:vars]
 ${elb_api_fqdn}

contrib/terraform/aws/terraform.tfvars

@@ -6,26 +6,34 @@ aws_vpc_cidr_block       = "10.250.192.0/18"
 aws_cidr_subnets_private = ["10.250.192.0/20", "10.250.208.0/20"]
 aws_cidr_subnets_public  = ["10.250.224.0/20", "10.250.240.0/20"]
 
-#Bastion Host
+# single AZ deployment
-aws_bastion_size = "t2.medium"
+#aws_cidr_subnets_private = ["10.250.192.0/20"]
+#aws_cidr_subnets_public  = ["10.250.224.0/20"]
 
+# 3+ AZ deployment
+#aws_cidr_subnets_private = ["10.250.192.0/24","10.250.193.0/24","10.250.194.0/24","10.250.195.0/24"]
+#aws_cidr_subnets_public  = ["10.250.224.0/24","10.250.225.0/24","10.250.226.0/24","10.250.227.0/24"]
+
+#Bastion Host
+aws_bastion_num  = 1
+aws_bastion_size = "t3.small"
 
 #Kubernetes Cluster
+aws_kube_master_num       = 3
+aws_kube_master_size      = "t3.medium"
+aws_kube_master_disk_size = 50
 
-aws_kube_master_num  = 3
+aws_etcd_num       = 0
-aws_kube_master_size = "t2.medium"
+aws_etcd_size      = "t3.medium"
+aws_etcd_disk_size = 50
 
-aws_etcd_num  = 3
+aws_kube_worker_num       = 4
-aws_etcd_size = "t2.medium"
+aws_kube_worker_size      = "t3.medium"
-
+aws_kube_worker_disk_size = 50
-aws_kube_worker_num  = 4
-aws_kube_worker_size = "t2.medium"
 
 #Settings AWS ELB
-
+aws_elb_api_port    = 6443
-aws_elb_api_port                = 6443
+k8s_secure_api_port = 6443
-k8s_secure_api_port             = 6443
-kube_insecure_apiserver_address = "0.0.0.0"
 
 default_tags = {
   #  Env = "devtest"

contrib/terraform/aws/terraform.tfvars.example

@@ -8,25 +8,26 @@ aws_cidr_subnets_public = ["10.250.224.0/20","10.250.240.0/20"]
 aws_avail_zones = ["eu-central-1a","eu-central-1b"]
 
 #Bastion Host
-aws_bastion_ami = "ami-5900cc36"
+aws_bastion_num = 1
-aws_bastion_size = "t2.small"
+aws_bastion_size = "t3.small"
-
 
 #Kubernetes Cluster
-
 aws_kube_master_num = 3
-aws_kube_master_size = "t2.medium"
+aws_kube_master_size = "t3.medium"
+aws_kube_master_disk_size = 50
 
 aws_etcd_num = 3
-aws_etcd_size = "t2.medium"
+aws_etcd_size = "t3.medium"
+aws_etcd_disk_size = 50
 
 aws_kube_worker_num = 4
-aws_kube_worker_size = "t2.medium"
+aws_kube_worker_size = "t3.medium"
-
+aws_kube_worker_disk_size = 50
-aws_cluster_ami = "ami-903df7ff"
 
 #Settings AWS ELB
-
 aws_elb_api_port = 6443
 k8s_secure_api_port = 6443
-kube_insecure_apiserver_address = 0.0.0.0
+
+default_tags = { }
+
+inventory_file = "../../../inventory/hosts"

contrib/terraform/aws/variables.tf

@@ -25,7 +25,7 @@ data "aws_ami" "distro" {
 
   filter {
     name   = "name"
-    values = ["ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*"]
+    values = ["debian-10-amd64-*"]
   }
 
   filter {
@@ -33,7 +33,7 @@ data "aws_ami" "distro" {
     values = ["hvm"]
   }
 
-  owners = ["099720109477"] # Canonical
+  owners = ["136693071363"] # Debian-10
 }
 
 //AWS VPC Variables
@@ -63,10 +63,18 @@ variable "aws_bastion_size" {
 * The number should be divisable by the number of used
 * AWS Availability Zones without an remainder.
 */
+variable "aws_bastion_num" {
+  description = "Number of Bastion Nodes"
+}
+
 variable "aws_kube_master_num" {
   description = "Number of Kubernetes Master Nodes"
 }
 
+variable "aws_kube_master_disk_size" {
+  description = "Disk size for Kubernetes Master Nodes (in GiB)"
+}
+
 variable "aws_kube_master_size" {
   description = "Instance size of Kube Master Nodes"
 }
@@ -75,6 +83,10 @@ variable "aws_etcd_num" {
   description = "Number of etcd Nodes"
 }
 
+variable "aws_etcd_disk_size" {
+  description = "Disk size for etcd Nodes (in GiB)"
+}
+
 variable "aws_etcd_size" {
   description = "Instance size of etcd Nodes"
 }
@@ -83,6 +95,10 @@ variable "aws_kube_worker_num" {
   description = "Number of Kubernetes Worker Nodes"
 }
 
+variable "aws_kube_worker_disk_size" {
+  description = "Disk size for Kubernetes Worker Nodes (in GiB)"
+}
+
 variable "aws_kube_worker_size" {
   description = "Instance size of Kubernetes Worker Nodes"
 }

contrib/terraform/openstack/README.md

@@ -273,6 +273,7 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
 |`wait_for_floatingip` | Let Terraform poll the instance until the floating IP has been associated, `false` by default. |
 |`node_root_volume_size_in_gb` | Size of the root volume for nodes, 0 to use ephemeral storage |
 |`master_root_volume_size_in_gb` | Size of the root volume for masters, 0 to use ephemeral storage |
+|`master_volume_type` | Volume type of the root volume for control_plane, 'Default' by default |
 |`gfs_root_volume_size_in_gb` | Size of the root volume for gluster, 0 to use ephemeral storage |
 |`etcd_root_volume_size_in_gb` | Size of the root volume for etcd nodes, 0 to use ephemeral storage |
 |`bastion_root_volume_size_in_gb` | Size of the root volume for bastions, 0 to use ephemeral storage |

contrib/terraform/openstack/kubespray.tf

@@ -83,6 +83,7 @@ module "compute" {
   use_server_groups                            = var.use_server_groups
   extra_sec_groups                             = var.extra_sec_groups
   extra_sec_groups_name                        = var.extra_sec_groups_name
+  group_vars_path                              = var.group_vars_path
 
   network_id = module.network.router_id
 }

contrib/terraform/openstack/modules/compute/main.tf

@@ -204,7 +204,7 @@ resource "openstack_compute_instance_v2" "bastion" {
   }
 
   provisioner "local-exec" {
-    command = "sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${var.bastion_fips[0]}/ > group_vars/no_floating.yml"
+    command = "sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${var.bastion_fips[0]}/ > ${var.group_vars_path}/no_floating.yml"
   }
 }
 
@@ -251,7 +251,7 @@ resource "openstack_compute_instance_v2" "k8s_master" {
   }
 
   provisioner "local-exec" {
-    command = "sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_master_fips), 0)}/ > group_vars/no_floating.yml"
+    command = "sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_master_fips), 0)}/ > ${var.group_vars_path}/no_floating.yml"
   }
 }
 
@@ -298,7 +298,7 @@ resource "openstack_compute_instance_v2" "k8s_master_no_etcd" {
   }
 
   provisioner "local-exec" {
-    command = "sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_master_fips), 0)}/ > group_vars/no_floating.yml"
+    command = "sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_master_fips), 0)}/ > ${var.group_vars_path}/no_floating.yml"
   }
 }
 
@@ -468,7 +468,7 @@ resource "openstack_compute_instance_v2" "k8s_node" {
   }
 
   provisioner "local-exec" {
-    command = "sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_node_fips), 0)}/ > group_vars/no_floating.yml"
+    command = "sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_node_fips), 0)}/ > ${var.group_vars_path}/no_floating.yml"
   }
 }
 
@@ -554,7 +554,7 @@ resource "openstack_compute_instance_v2" "k8s_nodes" {
   }
 
   provisioner "local-exec" {
-    command = "%{if each.value.floating_ip}sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, [for key, value in var.k8s_nodes_fips : value.address]), 0)}/ > group_vars/no_floating.yml%{else}true%{endif}"
+    command = "%{if each.value.floating_ip}sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, [for key, value in var.k8s_nodes_fips : value.address]), 0)}/ > ${var.group_vars_path}/no_floating.yml%{else}true%{endif}"
   }
 }
 

contrib/terraform/openstack/modules/compute/variables.tf

@@ -151,3 +151,7 @@ variable "image_master" {
 variable "image_master_uuid" {
   type = string
 }
+
+variable "group_vars_path" {
+  type = string
+}

contrib/terraform/openstack/variables.tf

@@ -278,3 +278,9 @@ variable "image_master_uuid" {
   description = "uuid of image to be used on master nodes. If empty defaults to image_uuid"
   default     = ""
 }
+
+variable "group_vars_path" {
+  description = "path to the inventory group vars directory"
+  type        = string
+  default     = "./group_vars"
+}

contrib/terraform/packet/README.md

@@ -1,16 +1,16 @@
-# Kubernetes on Packet with Terraform
+# Kubernetes on Equinix Metal with Terraform
 
 Provision a Kubernetes cluster with [Terraform](https://www.terraform.io) on
-[Packet](https://www.packet.com).
+[Equinix Metal](https://metal.equinix.com) ([formerly Packet](https://blog.equinix.com/blog/2020/10/06/equinix-metal-metal-and-more/)).
 
 ## Status
 
-This will install a Kubernetes cluster on Packet bare metal. It should work in all locations and on most server types.
+This will install a Kubernetes cluster on Equinix Metal. It should work in all locations and on most server types.
 
 ## Approach
 
 The terraform configuration inspects variables found in
-[variables.tf](variables.tf) to create resources in your Packet project.
+[variables.tf](variables.tf) to create resources in your Equinix Metal project.
 There is a [python script](../terraform.py) that reads the generated`.tfstate`
 file to generate a dynamic inventory that is consumed by [cluster.yml](../../..//cluster.yml)
 to actually install Kubernetes with Kubespray.
@@ -36,12 +36,12 @@ now six total etcd replicas.
 
 - [Install Terraform](https://www.terraform.io/intro/getting-started/install.html)
 - Install dependencies: `sudo pip install -r requirements.txt`
-- Account with Packet Host
+- Account with Equinix Metal
 - An SSH key pair
 
 ## SSH Key Setup
 
-An SSH keypair is required so Ansible can access the newly provisioned nodes (bare metal Packet hosts). By default, the public SSH key defined in cluster.tfvars will be installed in authorized_key on the newly provisioned nodes (~/.ssh/id_rsa.pub). Terraform will upload this public key and then it will be distributed out to all the nodes. If you have already set this public key in Packet (i.e. via the portal), then set the public keyfile name in cluster.tfvars to blank to prevent the duplicate key from being uploaded which will cause an error.
+An SSH keypair is required so Ansible can access the newly provisioned nodes (Equinix Metal hosts). By default, the public SSH key defined in cluster.tfvars will be installed in authorized_key on the newly provisioned nodes (~/.ssh/id_rsa.pub). Terraform will upload this public key and then it will be distributed out to all the nodes. If you have already set this public key in Equinix Metal (i.e. via the portal), then set the public keyfile name in cluster.tfvars to blank to prevent the duplicate key from being uploaded which will cause an error.
 
 If you don't already have a keypair generated (~/.ssh/id_rsa and ~/.ssh/id_rsa.pub), then a new keypair can be generated with the command:
 
@@ -51,7 +51,7 @@ ssh-keygen -f ~/.ssh/id_rsa
 
 ## Terraform
 
-Terraform will be used to provision all of the Packet resources with base software as appropriate.
+Terraform will be used to provision all of the Equinix Metal resources with base software as appropriate.
 
 ### Configuration
 
@@ -67,18 +67,18 @@ ln -s ../../contrib/terraform/packet/hosts
 
 This will be the base for subsequent Terraform commands.
 
-#### Packet API access
+#### Equinix Metal API access
 
-Your Packet API key must be available in the `PACKET_AUTH_TOKEN` environment variable.
+Your Equinix Metal API key must be available in the `PACKET_AUTH_TOKEN` environment variable.
 This key is typically stored outside of the code repo since it is considered secret.
 If someone gets this key, they can startup/shutdown hosts in your project!
 
 For more information on how to generate an API key or find your project ID, please see
-[API Integrations](https://support.packet.com/kb/articles/api-integrations)
+[Accounts Index](https://metal.equinix.com/developers/docs/accounts/).
 
-The Packet Project ID associated with the key will be set later in cluster.tfvars.
+The Equinix Metal Project ID associated with the key will be set later in `cluster.tfvars`.
 
-For more information about the API, please see [Packet API](https://www.packet.com/developers/api/)
+For more information about the API, please see [Equinix Metal API](https://metal.equinix.com/developers/api/).
 
 Example:
 
@@ -101,7 +101,7 @@ This helps when identifying which hosts are associated with each cluster.
 While the defaults in variables.tf will successfully deploy a cluster, it is recommended to set the following values:
 
 - cluster_name = the name of the inventory directory created above as $CLUSTER
-- packet_project_id = the Packet Project ID associated with the Packet API token above
+- packet_project_id = the Equinix Metal Project ID associated with the Equinix Metal API token above
 
 #### Enable localhost access
 

contrib/terraform/packet/kubespray.tf

@@ -1,4 +1,4 @@
-# Configure the Packet Provider
+# Configure the Equinix Metal Provider
 provider "packet" {
   version = "~> 2.0"
 }

contrib/terraform/packet/sample-inventory/cluster.tfvars

@@ -1,12 +1,12 @@
 # your Kubernetes cluster name here
 cluster_name = "mycluster"
 
-# Your Packet project ID. See https://support.packet.com/kb/articles/api-integrations
+# Your Equinix Metal project ID. See hhttps://metal.equinix.com/developers/docs/accounts/
 packet_project_id = "Example-API-Token"
 
-# The public SSH key to be uploaded into authorized_keys in bare metal Packet nodes provisioned
+# The public SSH key to be uploaded into authorized_keys in bare metal Equinix Metal nodes provisioned
-# leave this value blank if the public key is already setup in the Packet project
+# leave this value blank if the public key is already setup in the Equinix Metal project
-# Terraform will complain if the public key is setup in Packet
+# Terraform will complain if the public key is setup in Equinix Metal
 public_key_path = "~/.ssh/id_rsa.pub"
 
 # cluster location

contrib/terraform/packet/variables.tf

@@ -3,7 +3,7 @@ variable "cluster_name" {
 }
 
 variable "packet_project_id" {
-  description = "Your Packet project ID. See https://support.packet.com/kb/articles/api-integrations"
+  description = "Your Equinix Metal project ID. See https://metal.equinix.com/developers/docs/accounts/"
 }
 
 variable "operating_system" {

contrib/terraform/terraform.py

@@ -323,11 +323,11 @@ def openstack_host(resource, module_name):
     })
 
     # add groups based on attrs
-    groups.append('os_image=' + attrs['image']['id'])
+    groups.append('os_image=' + str(attrs['image']['id']))
-    groups.append('os_flavor=' + attrs['flavor']['name'])
+    groups.append('os_flavor=' + str(attrs['flavor']['name']))
     groups.extend('os_metadata_%s=%s' % item
                   for item in list(attrs['metadata'].items()))
-    groups.append('os_region=' + attrs['region'])
+    groups.append('os_region=' + str(attrs['region']))
 
     # groups specific to kubespray
     for group in attrs['metadata'].get('kubespray_groups', "").split(","):

contrib/terraform/upcloud/README.md

@@ -8,27 +8,29 @@ The setup looks like following
 
 ```text
    Kubernetes cluster
-+-----------------------+
++--------------------------+
-|   +--------------+    |
+|      +--------------+    |
-|   | +--------------+  |
+|      | +--------------+  |
-|   | |              |  |
+| -->  | |              |  |
-|   | | Master/etcd  |  |
+|      | | Master/etcd  |  |
-|   | | node(s)      |  |
+|      | | node(s)      |  |
-|   +-+              |  |
+|      +-+              |  |
-|     +--------------+  |
+|        +--------------+  |
-|           ^           |
+|              ^           |
-|           |           |
+|              |           |
-|           v           |
+|              v           |
-|   +--------------+    |
+|      +--------------+    |
-|   | +--------------+  |
+|      | +--------------+  |
-|   | |              |  |
+| -->  | |              |  |
-|   | |    Worker    |  |
+|      | |    Worker    |  |
-|   | |    node(s)   |  |
+|      | |    node(s)   |  |
-|   +-+              |  |
+|      +-+              |  |
-|     +--------------+  |
+|        +--------------+  |
-+-----------------------+
++--------------------------+
 ```
 
+The nodes uses a private network for node to node communication and a public interface for all external communication.
+
 ## Requirements
 
 * Terraform 0.13.0 or newer
@@ -94,9 +96,10 @@ terraform destroy --var-file cluster-settings.tfvars \
 
 ## Variables
 
-* `hostname`: A valid domain name, e.g. example.com. The maximum length is 128 characters.
+* `prefix`: Prefix to add to all resources, if set to "" don't set any prefix
 * `template_name`: The name or UUID  of a base image
-* `username`: a user to access the nodes
+* `username`: a user to access the nodes, defaults to "ubuntu"
+* `private_network_cidr`: CIDR to use for the private network, defaults to "172.16.0.0/24"
 * `ssh_public_keys`: List of public SSH keys to install on all machines
 * `zone`: The zone where to run the cluster
 * `machines`: Machines to provision. Key of this object will be used as the name of the machine
@@ -104,3 +107,6 @@ terraform destroy --var-file cluster-settings.tfvars \
   * `cpu`: number of cpu cores
   * `mem`: memory size in MB
   * `disk_size`: The size of the storage in GB
+  * `additional_disks`: Additional disks to attach to the node.
+    * `size`: The size of the additional disk in GB
+    * `tier`: The tier of disk to use (`maxiops` is the only one you can choose atm)

contrib/terraform/upcloud/cluster-settings.tfvars

@@ -1,12 +1,11 @@
-
 # See: https://developers.upcloud.com/1.3/5-zones/
 zone     = "fi-hel1"
 username = "ubuntu"
 
-inventory_file = "inventory.ini"
+# Prefix to use for all resources to separate them from other resources
+prefix = "kubespray"
 
-#  A valid domain name, e.g. host.example.com. The maximum length is 128 characters.
+inventory_file = "inventory.ini"
-hostname = "example.com"
 
 #  Set the operating system using UUID or exact name
 template_name = "Ubuntu Server 20.04 LTS (Focal Fossa)"
@@ -17,7 +16,7 @@ ssh_public_keys = [
   "ssh-rsa public key 2",
 ]
 
-#check list of available plan https://developers.upcloud.com/1.3/7-plans/
+# check list of available plan https://developers.upcloud.com/1.3/7-plans/
 machines = {
   "master-0" : {
     "node_type" : "master",
@@ -27,6 +26,7 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks" : {}
   },
   "worker-0" : {
     "node_type" : "worker",
@@ -36,6 +36,16 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks" : {
+      # "some-disk-name-1": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # },
+      # "some-disk-name-2": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # }
+    }
   },
   "worker-1" : {
     "node_type" : "worker",
@@ -45,6 +55,16 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks" : {
+      # "some-disk-name-1": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # },
+      # "some-disk-name-2": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # }
+    }
   },
   "worker-2" : {
     "node_type" : "worker",
@@ -54,5 +74,15 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks" : {
+      # "some-disk-name-1": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # },
+      # "some-disk-name-2": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # }
+    }
   }
 }

contrib/terraform/upcloud/main.tf

@@ -11,12 +11,14 @@ provider "upcloud" {
 module "kubernetes" {
   source = "./modules/kubernetes-cluster"
 
-  zone     = var.zone
+  prefix = var.prefix
-  hostname = var.hostname
+  zone   = var.zone
 
   template_name = var.template_name
   username      = var.username
 
+  private_network_cidr = var.private_network_cidr
+
   machines = var.machines
 
   ssh_public_keys = var.ssh_public_keys
@@ -30,13 +32,15 @@ data "template_file" "inventory" {
   template = file("${path.module}/templates/inventory.tpl")
 
   vars = {
-    connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s etcd_member_name=etcd%d",
+    connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d",
       keys(module.kubernetes.master_ip),
-      values(module.kubernetes.master_ip),
+      values(module.kubernetes.master_ip).*.public_ip,
+      values(module.kubernetes.master_ip).*.private_ip,
     range(1, length(module.kubernetes.master_ip) + 1)))
-    connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s",
+    connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s",
       keys(module.kubernetes.worker_ip),
-    values(module.kubernetes.worker_ip)))
+      values(module.kubernetes.worker_ip).*.public_ip,
+    values(module.kubernetes.worker_ip).*.private_ip))
     list_master = join("\n", formatlist("%s",
     keys(module.kubernetes.master_ip)))
     list_worker = join("\n", formatlist("%s",

contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf

@@ -1,3 +1,41 @@
+locals {
+  # Create a list of all disks to create
+  disks = flatten([
+    for node_name, machine in var.machines : [
+      for disk_name, disk in machine.additional_disks : {
+        disk = disk
+        disk_name = disk_name
+        node_name = node_name
+      }
+    ]
+  ])
+
+  # If prefix is set, all resources will be prefixed with "${var.prefix}-"
+  # Else don't prefix with anything
+  resource-prefix = "%{ if var.prefix != ""}${var.prefix}-%{ endif }"
+}
+
+resource "upcloud_network" "private" {
+  name = "${local.resource-prefix}k8s-network"
+  zone = var.zone
+
+  ip_network {
+    address = var.private_network_cidr
+    dhcp    = true
+    family  = "IPv4"
+  }
+}
+
+resource "upcloud_storage" "additional_disks" {
+  for_each = {
+    for disk in local.disks: "${disk.node_name}_${disk.disk_name}" => disk.disk
+  }
+
+  size  = each.value.size
+  tier  = each.value.tier
+  title = "${local.resource-prefix}${each.key}"
+  zone  = var.zone
+}
 
 resource "upcloud_server" "master" {
   for_each = {
@@ -6,35 +44,48 @@ resource "upcloud_server" "master" {
     if machine.node_type == "master"
   }
 
-  hostname    = "${each.key}.${var.hostname}"
+  hostname = "${local.resource-prefix}${each.key}"
-  cpu            = each.value.cpu
+  cpu      = each.value.cpu
-  mem       = each.value.mem
+  mem      = each.value.mem
-  zone            = var.zone
+  zone     = var.zone
 
   template {
   storage = var.template_name
-  size = each.value.disk_size
+  size    = each.value.disk_size
   }
 
-  # Network interfaces
+  # Public network interface
- network_interface {
+  network_interface {
-   type = "public"
+    type = "public"
- }
+  }
 
- network_interface {
+  # Private network interface
-   type = "utility"
+  network_interface {
- }
+    type    = "private"
- # Include at least one public SSH key
+    network = upcloud_network.private.id
- login {
+  }
-   user = var.username
-   keys = var.ssh_public_keys
-   create_password = false
 
- }
+  dynamic "storage_devices" {
+    for_each = {
+      for disk_key_name, disk in upcloud_storage.additional_disks :
+        disk_key_name => disk
+        # Only add the disk if it matches the node name in the start of its name
+        if length(regexall("^${each.key}_.+", disk_key_name)) > 0
+    }
 
+    content {
+      storage = storage_devices.value.id
+    }
+  }
+
+  # Include at least one public SSH key
+  login {
+    user            = var.username
+    keys            = var.ssh_public_keys
+    create_password = false
+  }
 }
 
-
 resource "upcloud_server" "worker" {
   for_each = {
     for name, machine in var.machines :
@@ -42,25 +93,44 @@ resource "upcloud_server" "worker" {
     if machine.node_type == "worker"
   }
 
-  hostname    = "${each.key}.${var.hostname}"
+  hostname = "${local.resource-prefix}${each.key}"
-  cpu            = each.value.cpu
+  cpu      = each.value.cpu
-  mem       = each.value.mem
+  mem      = each.value.mem
-  zone            = var.zone
+  zone     = var.zone
 
   template {
-  storage = var.template_name
+    storage = var.template_name
-  size = each.value.disk_size
+    size    = each.value.disk_size
   }
 
-  # Network interfaces
+  # Public network interface
- network_interface {
+  network_interface {
-   type = "public"
+    type = "public"
- }
+  }
 
- # Include at least one public SSH key
+  # Private network interface
- login {
+  network_interface {
-   user = var.username
+    type    = "private"
-   keys = var.ssh_public_keys
+    network = upcloud_network.private.id
-   create_password = false
+  }
- }
+
+  dynamic "storage_devices" {
+    for_each = {
+      for disk_key_name, disk in upcloud_storage.additional_disks :
+        disk_key_name => disk
+        # Only add the disk if it matches the node name in the start of its name
+        if length(regexall("^${each.key}_.+", disk_key_name)) > 0
+    }
+
+    content {
+      storage = storage_devices.value.id
+    }
+  }
+
+  # Include at least one public SSH key
+  login {
+    user            = var.username
+    keys            = var.ssh_public_keys
+    create_password = false
+  }
 }

contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf

@@ -2,13 +2,19 @@
 output "master_ip" {
   value = {
     for instance in upcloud_server.master :
-    instance.hostname => instance.network_interface[0].ip_address
+    instance.hostname => {
+      "public_ip": instance.network_interface[0].ip_address
+      "private_ip": instance.network_interface[1].ip_address
+    }
   }
 }
 
 output "worker_ip" {
   value = {
     for instance in upcloud_server.worker :
-    instance.hostname => instance.network_interface[0].ip_address
+    instance.hostname => {
+      "public_ip": instance.network_interface[0].ip_address
+      "private_ip": instance.network_interface[1].ip_address
+    }
   }
 }

contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf

@@ -1,22 +1,28 @@
+variable "prefix" {
+  type = string
+}
+
 variable "zone" {
   type = string
 }
 
-variable "hostname"{
+variable "template_name" {}
- default ="example.com"
-}
 
-variable "template_name"{}
+variable "username" {}
 
-variable "username"{}
+variable "private_network_cidr" {}
 
 variable "machines" {
   description = "Cluster machines"
   type = map(object({
-    node_type = string
+    node_type       = string
-    cpu      = string
+    cpu             = string
-    mem      = string
+    mem             = string
-    disk_size =  number
+    disk_size       =  number
+    additional_disks = map(object({
+      size = number
+      tier = string
+    }))
   }))
 }
 

contrib/terraform/upcloud/sample-inventory/cluster.tfvars

@@ -2,20 +2,21 @@
 zone     = "fi-hel1"
 username = "ubuntu"
 
-inventory_file = "inventory.ini"
+# Prefix to use for all resources to separate them from other resources
+prefix = "kubespray"
 
-#  A valid domain name, e.g. host.example.com. The maximum length is 128 characters.
+inventory_file = "inventory.ini"
-hostname = "example.com"
 
 #  Set the operating system using UUID or exact name
 template_name = "Ubuntu Server 20.04 LTS (Focal Fossa)"
+
 ssh_public_keys = [
   # Put your public SSH key here
   "ssh-rsa I-did-not-read-the-docs",
   "ssh-rsa I-did-not-read-the-docs 2",
 ]
 
-check list of available plan https://developers.upcloud.com/1.3/7-plans/
+# check list of available plan https://developers.upcloud.com/1.3/7-plans/
 machines = {
   "master-0" : {
     "node_type" : "master",
@@ -25,6 +26,7 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks": {}
   },
   "worker-0" : {
     "node_type" : "worker",
@@ -34,6 +36,16 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks": {
+      # "some-disk-name-1": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # },
+      # "some-disk-name-2": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # }
+    }
   },
   "worker-1" : {
     "node_type" : "worker",
@@ -43,6 +55,16 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks": {
+      # "some-disk-name-1": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # },
+      # "some-disk-name-2": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # }
+    }
   },
   "worker-2" : {
     "node_type" : "worker",
@@ -52,5 +74,15 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks": {
+      # "some-disk-name-1": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # },
+      # "some-disk-name-2": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # }
+    }
   }
 }

contrib/terraform/upcloud/variables.tf

@@ -1,23 +1,40 @@
+variable "prefix" {
+  type    = string
+  default = "kubespray"
+
+  description = "Prefix that is used to distinguish these resources from others"
+}
 
 variable "zone" {
   description = "The zone where to run the cluster"
 }
 
-variable "hostname" {
+variable "template_name" {
-  default = "example.com"
+  description = "Block describing the preconfigured operating system"
 }
 
-variable "template_name" {}
+variable "username" {
+  description = "The username to use for the nodes"
+  default     = "ubuntu"
+}
 
-variable "username" {}
+variable "private_network_cidr" {
+  description = "CIDR to use for the private network"
+  default     = "172.16.0.0/24"
+}
 
 variable "machines" {
   description = "Cluster machines"
+
   type = map(object({
     node_type = string
     cpu       = string
     mem       = string
     disk_size = number
+    additional_disks = map(object({
+      size = number
+      tier = string
+    }))
   }))
 }
 
@@ -30,6 +47,10 @@ variable "inventory_file" {
   description = "Where to store the generated inventory file"
 }
 
-variable "UPCLOUD_USERNAME" {}
+variable "UPCLOUD_USERNAME" {
+  description = "UpCloud username with API access"
+}
 
-variable "UPCLOUD_PASSWORD" {}
+variable "UPCLOUD_PASSWORD" {
+  description = "Password for UpCloud API user"
+}

docs/_sidebar.md

@@ -14,30 +14,46 @@
   * [Calico](docs/calico.md)
   * [Flannel](docs/flannel.md)
   * [Kube Router](docs/kube-router.md)
+  * [Kube OVN](docs/kube-ovn.md)
   * [Weave](docs/weave.md)
   * [Multus](docs/multus.md)
+  * [OVN4NFV](docs/ovn4nfv.md)
 * Ingress
-  * [Ambassador](docs/ambassador.md)
+  * [ALB Ingress](docs/ingress_controller/alb_ingress_controller.md)
+  * [Ambassador](docs/ingress_controller/ambassador.md)
+  * [MetalLB](docs/metallb.md)
+  * [Nginx Ingress](docs/ingress_controller/ingress_nginx.md)
 * [Cloud providers](docs/cloud.md)
   * [AWS](docs/aws.md)
   * [Azure](docs/azure.md)
   * [OpenStack](/docs/openstack.md)
-  * [Packet](/docs/packet.md)
+  * [Equinix Metal](/docs/equinix-metal.md)
   * [vSphere](/docs/vsphere.md)
-* Operating Systems
+* [Operating Systems](docs/bootstrap-os.md)
   * [Debian](docs/debian.md)
   * [Flatcar Container Linux](docs/flatcar.md)
   * [Fedora CoreOS](docs/fcos.md)
   * [OpenSUSE](docs/opensuse.md)
+  * [RedHat Enterprise Linux](docs/rhel.md)
+  * [CentOS/OracleLinux/AlmaLinux](docs/centos8.md)
+  * [Amazon Linux 2](docs/amazonlinux.md)
 * CRI
   * [Containerd](docs/containerd.md)
   * [CRI-O](docs/cri-o.md)
+  * [Kata Containers](docs/kata-containers.md)
+  * [gVisor](docs/gvisor.md)
 * Advanced
   * [Proxy](/docs/proxy.md)
   * [Downloads](docs/downloads.md)
   * [Netcheck](docs/netcheck.md)
+  * [Cert Manager](docs/cert_manager.md)
   * [DNS Stack](docs/dns-stack.md)
   * [Kubernetes reliability](docs/kubernetes-reliability.md)
+  * [Local Registry](docs/kubernetes-apps/registry.md)
+* External Storage Provisioners
+  * [RBD Provisioner](docs/kubernetes-apps/rbd_provisioner.md)
+  * [CEPHFS Provisioner](docs/kubernetes-apps/cephfs_provisioner.md)
+  * [Local Volume Provisioner](docs/kubernetes-apps/local_volume_provisioner.md)
 * Developers
   * [Test cases](docs/test_cases.md)
   * [Vagrant](docs/vagrant.md)

docs/ansible.md

@@ -20,7 +20,7 @@ When _kube_node_ contains _etcd_, you define your etcd cluster to be as well sch
 If you want it a standalone, make sure those groups do not intersect.
 If you want the server to act both as control-plane and node, the server must be defined
 on both groups _kube_control_plane_ and _kube_node_. If you want a standalone and
-unschedulable master, the server must be defined only in the _kube_control_plane_ and
+unschedulable control plane, the server must be defined only in the _kube_control_plane_ and
 not _kube_node_.
 
 There are also two special groups:
@@ -67,7 +67,7 @@ The group variables to control main deployment options are located in the direct
 Optional variables are located in the `inventory/sample/group_vars/all.yml`.
 Mandatory variables that are common for at least one role (or a node group) can be found in the
 `inventory/sample/group_vars/k8s_cluster.yml`.
-There are also role vars for docker, kubernetes preinstall and master roles.
+There are also role vars for docker, kubernetes preinstall and control plane roles.
 According to the [ansible docs](https://docs.ansible.com/ansible/latest/playbooks_variables.html#variable-precedence-where-should-i-put-a-variable),
 those cannot be overridden from the group vars. In order to override, one should use
 the `-e` runtime flags (most simple way) or other layers described in the docs.
@@ -98,46 +98,112 @@ task vars (only for the task) | Unused for roles, but only for helper scripts
 
 The following tags are defined in playbooks:
 
-|                 Tag name | Used for
+|                       Tag name | Used for
-|--------------------------|---------
+|--------------------------------|---------
-|                     apps | K8s apps definitions
+|                     ambassador | Ambassador Ingress Controller
-|                    azure | Cloud-provider Azure
+|                       annotate | Create kube-router annotation
-|                  bastion | Setup ssh config for bastion
+|                           apps | K8s apps definitions
-|             bootstrap-os | Anything related to host OS configuration
+|                        asserts | Check tasks for download role
-|                   calico | Network plugin Calico
+|             aws-ebs-csi-driver | Configuring csi driver: aws-ebs
-|                    canal | Network plugin Canal
+|               azure-csi-driver | Configuring csi driver: azure
-|           cloud-provider | Cloud-provider related tasks
+|                        bastion | Setup ssh config for bastion
-|                   docker | Configuring docker for hosts
+|                   bootstrap-os | Anything related to host OS configuration
-|                 download | Fetching container images to a delegate host
+|                         calico | Network plugin Calico
-|                     etcd | Configuring etcd cluster
+|                      calico_rr | Configuring Calico route reflector
-|         etcd-pre-upgrade | Upgrading etcd cluster
+|                          canal | Network plugin Canal
-|             etcd-secrets | Configuring etcd certs/keys
+|             cephfs-provisioner | Configuring CephFS
-|                 etchosts | Configuring /etc/hosts entries for hosts
+|                   cert-manager | Configuring certificate manager for K8s
-|                    facts | Gathering facts and misc check results
+|                         cilium | Network plugin Cilium
-|                  flannel | Network plugin flannel
+|              cinder-csi-driver | Configuring csi driver: cinder
-|                      gce | Cloud-provider GCP
+|                         client | Kubernetes clients role
-|          k8s-pre-upgrade | Upgrading K8s cluster
+|                 cloud-provider | Cloud-provider related tasks
-|              k8s-secrets | Configuring K8s certs/keys
+|                  cluster-roles | Configuring cluster wide application (psp ...)
-|           kube-apiserver | Configuring static pod kube-apiserver
+|                            cni | CNI plugins for Network Plugins
-|  kube-controller-manager | Configuring static pod kube-controller-manager
+|                     containerd | Configuring containerd engine runtime for hosts
-|                  kubectl | Installing kubectl and bash completion
+|   container_engine_accelerator | Enable nvidia accelerator for runtimes
-|                  kubelet | Configuring kubelet service
+|               container-engine | Configuring container engines
-|               kube-proxy | Configuring static pod kube-proxy
+|             container-runtimes | Configuring container runtimes
-|           kube-scheduler | Configuring static pod kube-scheduler
+|                        coredns | Configuring coredns deployment
-|                localhost | Special steps for the localhost (ansible runner)
+|                           crio | Configuring crio container engine for hosts
-|                   master | Configuring K8s master node role
+|                           crun | Configuring crun runtime
-|               netchecker | Installing netchecker K8s app
+|                     csi-driver | Configuring csi driver
-|                  network | Configuring networking plugins for K8s
+|                      dashboard | Installing and configuring the Kubernetes Dashboard
-|                    nginx | Configuring LB for kube-apiserver instances
+|                            dns | Remove dns entries when resetting
-|                     node | Configuring K8s minion (compute) node role
+|                         docker | Configuring docker engine runtime for hosts
-|                openstack | Cloud-provider OpenStack
+|                       download | Fetching container images to a delegate host
-|               preinstall | Preliminary configuration steps
+|                           etcd | Configuring etcd cluster
-|               resolvconf | Configuring /etc/resolv.conf for hosts/apps
+|                   etcd-secrets | Configuring etcd certs/keys
-|                  upgrade | Upgrading, f.e. container images/binaries
+|                       etchosts | Configuring /etc/hosts entries for hosts
-|                   upload | Distributing images/binaries across hosts
+|      external-cloud-controller | Configure cloud controllers
-|                    weave | Network plugin Weave
+|             external-openstack | Cloud controller : openstack
-|              ingress_alb | AWS ALB Ingress Controller
+|           external-provisioner | Configure external provisioners
-|              ambassador  | Ambassador Ingress Controller
+|               external-vsphere | Cloud controller : vsphere
+|                          facts | Gathering facts and misc check results
+|                          files | Remove files when resetting
+|                        flannel | Network plugin flannel
+|                            gce | Cloud-provider GCP
+|              gcp-pd-csi-driver | Configuring csi driver: gcp-pd
+|                         gvisor | Configuring gvisor runtime
+|                           helm | Installing and configuring Helm
+|             ingress-controller | Configure ingress controllers
+|                    ingress_alb | AWS ALB Ingress Controller
+|                           init | Windows kubernetes init nodes
+|                       iptables | Flush and clear iptable when resetting
+|                k8s-pre-upgrade | Upgrading K8s cluster
+|                    k8s-secrets | Configuring K8s certs/keys
+|                 k8s-gen-tokens | Configuring K8s tokens
+|                kata-containers | Configuring kata-containers runtime
+|                           krew | Install and manage krew
+|                        kubeadm | Roles linked to kubeadm tasks
+|                 kube-apiserver | Configuring static pod kube-apiserver
+|        kube-controller-manager | Configuring static pod kube-controller-manager
+|                        kubectl | Installing kubectl and bash completion
+|                        kubelet | Configuring kubelet service
+|                       kube-ovn | Network plugin kube-ovn
+|                    kube-router | Network plugin kube-router
+|                     kube-proxy | Configuring static pod kube-proxy
+|                      localhost | Special steps for the localhost (ansible runner)
+|         local-path-provisioner | Configure External provisioner: local-path
+|       local-volume-provisioner | Configure External provisioner: local-volume
+|                        macvlan | Network plugin macvlan
+|                         master | Configuring K8s master node role
+|                        metallb | Installing and configuring metallb
+|                 metrics_server | Configuring metrics_server
+|                     netchecker | Installing netchecker K8s app
+|                        network | Configuring networking plugins for K8s
+|                         mounts | Umount kubelet dirs when reseting
+|                         multus | Network plugin multus
+|                          nginx | Configuring LB for kube-apiserver instances
+|                           node | Configuring K8s minion (compute) node role
+|                   nodelocaldns | Configuring nodelocaldns daemonset
+|                     node-label | Tasks linked to labeling of nodes
+|                   node-webhook | Tasks linked to webhook (grating access to resources)
+|                     nvidia_gpu | Enable nvidia accelerator for runtimes
+|                            oci | Cloud provider: oci
+|                        ovn4nfv | Network plugin ovn4nfv
+|             persistent_volumes | Configure csi volumes
+| persistent_volumes_aws_ebs_csi | Configuring csi driver: aws-ebs
+| persistent_volumes_cinder_csi  | Configuring csi driver: cinder
+| persistent_volumes_gcp_pd_csi  | Configuring csi driver: gcp-pd
+| persistent_volumes_openstack   | Configuring csi driver: openstack
+|              policy-controller | Configuring Calico policy controller
+|                    post-remove | Tasks running post-remove operation
+|                   post-upgrade | Tasks running post-upgrade operation
+|                     pre-remove | Tasks running pre-remove operation
+|                    pre-upgrade | Tasks running pre-upgrade operation
+|                     preinstall | Preliminary configuration steps
+|                       registry | Configuring local docker registry
+|                          reset | Tasks running doing the node reset
+|                     resolvconf | Configuring /etc/resolv.conf for hosts/apps
+|                rbd-provisioner | Configure External provisioner: rdb
+|                       services | Remove services (etcd, kubelet etc...) when resetting
+|                       snapshot | Enabling csi snapshot
+|            snapshot-controller | Configuring csi snapshot controller
+|                        upgrade | Upgrading, f.e. container images/binaries
+|                         upload | Distributing images/binaries across hosts
+|             vsphere-csi-driver | Configuring csi driver: vsphere
+|                          weave | Network plugin Weave
+|                      win_nodes | Running windows specific tasks
 
 Note: Use the ``bash scripts/gen_tags.sh`` command to generate a list of all
 tags found in the codebase. New tags will be listed with the empty "Used for"
@@ -187,3 +253,28 @@ For more information about Ansible and bastion hosts, read
 ## Mitogen
 
 You can use [mitogen](mitogen.md) to speed up kubespray.
+
+## Beyond ansible 2.9
+
+Ansible project has decided, in order to ease their maintenance burden, to split between
+two projects which are now joined under the Ansible umbrella.
+
+Ansible-base (2.10.x branch) will contain just the ansible language implementation while
+ansible modules that were previously bundled into a single repository will be part of the
+ansible 3.x package. Pleasee see [this blog post](https://blog.while-true-do.io/ansible-release-3-0-0/)
+that explains in detail the need and the evolution plan.
+
+**Note:** this change means that ansible virtual envs cannot be upgraded with `pip install -U`.
+You first need to uninstall your old ansible (pre 2.10) version and install the new one.
+
+```ShellSession
+pip uninstall ansible
+cd kubespray/
+pip install -U .
+```
+
+**Note:** some changes needed to support ansible 2.10+ are not backwards compatible with 2.9
+Kubespray needs to evolve and keep pace with upstream ansible and will be forced to eventually
+drop 2.9 support. Kubespray CIs use only the ansible version specified in the `requirements.txt`
+and while the `ansible_version.yml` may allow older versions to be used, these are not
+exercised in the CI and compatibility is not guaranteed.

docs/azure.md

@@ -10,7 +10,7 @@ Not all features are supported yet though, for a list of the current status have
 
 Before creating the instances you must first set the `azure_` variables in the `group_vars/all/all.yml` file.
 
-All of the values can be retrieved using the azure cli tool which can be downloaded here: <https://docs.microsoft.com/en-gb/azure/xplat-cli-install>
+All of the values can be retrieved using the Azure CLI tool which can be downloaded here: <https://docs.microsoft.com/en-gb/cli/azure/install-azure-cli>
 After installation you have to run `az login` to get access to your account.
 
 ### azure_cloud

docs/calico.md

@@ -189,7 +189,7 @@ To re-define default action please set the following variable in your inventory:
 calico_endpoint_to_host_action: "ACCEPT"
 ```
 
-## Optional : Define address on which Felix will respond to health requests
+### Optional : Define address on which Felix will respond to health requests
 
 Since Calico 3.2.0, HealthCheck default behavior changed from listening on all interfaces to just listening on localhost.
 
@@ -199,6 +199,15 @@ To re-define health host please set the following variable in your inventory:
 calico_healthhost: "0.0.0.0"
 ```
 
+### Optional : Configure Calico Node probe timeouts
+
+Under certain conditions a deployer may need to tune the Calico liveness and readiness probes timeout settings. These can be configured like this:
+
+```yml
+calico_node_livenessprobe_timeout: 10
+calico_node_readinessprobe_timeout: 10
+```
+
 ## Config encapsulation for cross server traffic
 
 Calico supports two types of encapsulation: [VXLAN and IP in IP](https://docs.projectcalico.org/v3.11/networking/vxlan-ipip). VXLAN is supported in some environments where IP in IP is not (for example, Azure).
@@ -219,6 +228,19 @@ calico_vxlan_mode: 'Never'
 
 If you use VXLAN mode, BGP networking is not required. You can disable BGP to reduce the moving parts in your cluster by `calico_network_backend: vxlan`
 
+## Configuring interface MTU
+
+This is an advanced topic and should usually not be modified unless you know exactly what you are doing. Calico is smart enough to deal with the defaults and calculate the proper MTU. If you do need to set up a custom MTU you can change `calico_veth_mtu` as follows:
+
+* If Wireguard is enabled, subtract 60 from your network MTU (i.e. 1500-60=1440)
+* If using VXLAN or BPF mode is enabled, subtract 50 from your network MTU (i.e. 1500-50=1450)
+* If using IPIP, subtract 20 from your network MTU (i.e. 1500-20=1480)
+* if not using any encapsulation, set to your network MTU (i.e. 1500 or 9000)
+
+```yaml
+calico_veth_mtu: 1440
+```
+
 ## Cloud providers configuration
 
 Please refer to the official documentation, for example [GCE configuration](http://docs.projectcalico.org/v1.5/getting-started/docker/installation/gce) requires a security rule for calico ip-ip tunnels. Note, calico is always configured with ``calico_ipip_mode: Always`` if the cloud provider was defined.
@@ -260,3 +282,93 @@ calico_ipam_host_local: true
 ```
 
 Refer to Project Calico section [Using host-local IPAM](https://docs.projectcalico.org/reference/cni-plugin/configuration#using-host-local-ipam) for further information.
+
+## eBPF Support
+
+Calico supports eBPF for its data plane see [an introduction to the Calico eBPF Dataplane](https://www.projectcalico.org/introducing-the-calico-ebpf-dataplane/) for further information.
+
+Note that it is advisable to always use the latest version of Calico when using the eBPF dataplane.
+
+### Enabling eBPF support
+
+To enable the eBPF dataplane support ensure you add the following to your inventory. Note that the `kube-proxy` is incompatible with running Calico in eBPF mode and the kube-proxy should be removed from the system.
+
+```yaml
+calico_bpf_enabled: true
+kube_proxy_remove: true
+```
+
+### Cleaning up after kube-proxy
+
+Calico node cannot clean up after kube-proxy has run in ipvs mode. If you are converting an existing cluster to eBPF you will need to ensure the `kube-proxy` DaemonSet is deleted and that ipvs rules are cleaned.
+
+To check that kube-proxy was running in ipvs mode:
+
+```ShellSession
+# ipvsadm -l
+```
+
+To clean up any ipvs leftovers:
+
+```ShellSession
+# ipvsadm -C
+```
+
+### Calico access to the kube-api
+
+Calico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.projectcalico.org/maintenance/ebpf/enabling-bpf) for guidelines on how to do this.
+
+Kubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `loadbalancer_apiserver` inventory variable documented in [HA Mode](./ha-mode.md).
+
+If no external loadbalancer is used, Calico eBPF can also use the localhost loadbalancer option. In this case Calico Automatic Host Endpoints need to be enabled to allow services like `coredns` and `metrics-server` to communicate with the kubernetes host endpoint. See [this blog post](https://www.projectcalico.org/securing-kubernetes-nodes-with-calico-automatic-host-endpoints/) on enabling automatic host endpoints.
+
+```yaml
+loadbalancer_apiserver_localhost: true
+use_localhost_as_kubeapi_loadbalancer: true
+```
+
+### Tunneled versus Direct Server Return
+
+By default Calico usese Tunneled service mode but it can use direct server return (DSR) in order to optimize the return path for a service.
+
+To configure DSR:
+
+```yaml
+calico_bpf_service_mode: "DSR"
+```
+
+### eBPF Logging and Troubleshooting
+
+In order to enable Calico eBPF mode logging:
+
+```yaml
+calico_bpf_log_level: "Debug"
+```
+
+To view the logs you need to use the `tc` command to read the kernel trace buffer:
+
+```ShellSession
+tc exec bpf debug
+```
+
+Please see [Calico eBPF troubleshooting guide](https://docs.projectcalico.org/maintenance/troubleshoot/troubleshoot-ebpf#ebpf-program-debug-logs).
+
+## Wireguard Encryption
+
+Calico supports using Wireguard for encryption. Please see the docs on [encryptiong cluster pod traffic](https://docs.projectcalico.org/security/encrypt-cluster-pod-traffic).
+
+To enable wireguard support:
+
+```yaml
+calico_wireguard_enabled: true
+```
+
+The following OSes will require enabling the EPEL repo in order to bring in wireguard tools:
+
+* CentOS 7 & 8
+* AlmaLinux 8
+* Amazon Linux 2
+
+```yaml
+epel_enabled: true
+```

docs/cert_manager.md

@@ -56,7 +56,7 @@ ingress_nginx_enabled: true
 For example, if you're using the Nginx ingress controller, you can secure the Prometheus ingress by adding the annotation `cert-manager.io/cluster-issuer: ca-issuer` and the `spec.tls` section to the `Ingress` resource definition.
 
 ```yaml
-apiVersion: networking.k8s.io/v1beta1
+apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: prometheus-k8s
@@ -76,9 +76,12 @@ spec:
     http:
       paths:
       - path: /
+        pathType: ImplementationSpecific
         backend:
-          serviceName: prometheus-k8s
+          service:
-          servicePort: web
+            name: prometheus-k8s
+            port:
+              name: web
 ```
 
 Once deployed to your K8s cluster, every 3 months cert-manager will automatically rotate the Prometheus `prometheus.example.com` TLS client certificate and key, and store these as the Kubernetes `prometheus-dashboard-certs` secret.
@@ -89,7 +92,7 @@ For further information, read the official [Cert-Manager Ingress](https://cert-m
 
 #### Install Cloudflare PKI/TLS `cfssl` Toolkit
 
-e.g. For Ubuntu/Debian distibutions, the toolkit is part of the `golang-cfssl` package.
+e.g. For Ubuntu/Debian distributions, the toolkit is part of the `golang-cfssl` package.
 
 ```shell
 sudo apt-get install -y golang-cfssl

docs/ci.md

@@ -10,9 +10,10 @@ amazon |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 centos7 |  :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :white_check_mark: |
 centos8 |  :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
 debian10 |  :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian11 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
-fedora32 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
 fedora33 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora34 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
 opensuse |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 oracle7 |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu16 |  :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :white_check_mark: |
@@ -27,9 +28,10 @@ amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 centos7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 centos8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian10 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian11 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora32 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora33 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora34 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 opensuse |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 oracle7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu16 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
@@ -44,9 +46,10 @@ amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 centos7 |  :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
 centos8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian10 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian11 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora32 |  :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
 fedora33 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora34 |  :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
 opensuse |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 oracle7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu16 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |

docs/cilium.md

@@ -15,7 +15,7 @@ balancer deployed by Kubespray and **only contacts the first master**.
 ## Choose Cilium version
 
 ```yml
-cilium_version: v1.8.9 ## or v1.9.6
+cilium_version: v1.9.9
 ```
 
 ## Add variable to config

docs/comparisons.md

@@ -21,7 +21,6 @@ does generic configuration management tasks from the "OS operators" ansible
 world, plus some initial K8s clustering (with networking plugins included) and
 control plane bootstrapping.
 
-Kubespray supports `kubeadm` for cluster creation since v2.3
+Kubespray has started using `kubeadm` internally for cluster creation since v2.3
-(and deprecated non-kubeadm deployment starting from v2.8)
 in order to consume life cycle management domain knowledge from it
 and offload generic OS configuration things from it, which hopefully benefits both sides.

docs/coreos.md

@@ -1,14 +0,0 @@
-CoreOS bootstrap
-===============
-
-Example with Ansible:
-
-Before running the cluster playbook you must satisfy the following requirements:
-
-General CoreOS Pre-Installation Notes:
-
-- Ensure that the bin_dir is set to `/opt/bin`
-- ansible_python_interpreter should be `/opt/bin/python`. This will be laid down by the bootstrap task.
-- The default resolvconf_mode setting of `docker_dns` **does not** work for CoreOS. This is because we do not edit the systemd service file for docker on CoreOS nodes. Instead, just use the `host_resolvconf` mode. It should work out of the box.
-
-Then you can proceed to [cluster deployment](#run-deployment)

docs/equinix-metal.md

@@ -1,15 +1,15 @@
-# Packet
+# Equinix Metal
 
-Kubespray provides support for bare metal deployments using the [Packet bare metal cloud](http://www.packet.com).
+Kubespray provides support for bare metal deployments using the [Equinix Metal](http://metal.equinix.com).
 Deploying upon bare metal allows Kubernetes to run at locations where an existing public or private cloud might not exist such
-as cell tower, edge collocated installations. The deployment mechanism used by Kubespray for Packet is similar to that used for
+as cell tower, edge collocated installations. The deployment mechanism used by Kubespray for Equinix Metal is similar to that used for
-AWS and OpenStack clouds (notably using Terraform to deploy the infrastructure). Terraform uses the Packet provider plugin
+AWS and OpenStack clouds (notably using Terraform to deploy the infrastructure). Terraform uses the Equinix Metal provider plugin
 to provision and configure hosts which are then used by the Kubespray Ansible playbooks. The Ansible inventory is generated
 dynamically from the Terraform state file.
 
 ## Local Host Configuration
 
-To perform this installation, you will need a localhost to run Terraform/Ansible (laptop, VM, etc) and an account with Packet.
+To perform this installation, you will need a localhost to run Terraform/Ansible (laptop, VM, etc) and an account with Equinix Metal.
 In this example, we're using an m1.large CentOS 7 OpenStack VM as the localhost to kickoff the Kubernetes installation.
 You'll need Ansible, Git, and PIP.
 
@@ -64,7 +64,7 @@ ln -s ../../contrib/terraform/packet/hosts
 ```
 
 Details about the cluster, such as the name, as well as the authentication tokens and project ID
-for Packet need to be defined. To find these values see [Packet API Integration](https://support.packet.com/kb/articles/api-integrations)
+for Equinix Metal need to be defined. To find these values see [Equinix Metal API Accounts](https://metal.equinix.com/developers/docs/accounts/).
 
 ```bash
 vi cluster.tfvars

docs/fcos.md

@@ -1,6 +1,6 @@
 # Fedora CoreOS
 
-Tested with stable version 31.20200223.3.0.
+Tested with stable version 34.20210611.3.0
 
 Because package installation with `rpm-ostree` requires a reboot, playbook may fail while bootstrap.
 Restart playbook again.
@@ -9,33 +9,9 @@ Restart playbook again.
 
 Tested with
 
-- docker
+- containerd
 - crio
 
-### docker
-
-OS base packages contains docker.
-
-### cri-o
-
-To use `cri-o` disable docker service with ignition:
-
-```yaml
-#workaround, see https://github.com/coreos/fedora-coreos-tracker/issues/229
-systemd:
-  units:
-    - name: docker.service
-      enabled: false
-      contents: |
-        [Unit]
-        Description=disable docker
-
-        [Service]
-
-        [Install]
-        WantedBy=multi-user.target
-```
-
 ## Network
 
 ### calico
@@ -79,11 +55,14 @@ Prepare ignition and serve via http (a.e. python -m http.server )
 ### create guest
 
 ```shell script
-fcos_version=31.20200223.3.0
+machine_name=myfcos1
+ignition_url=http://mywebserver/fcos.ign
+
+fcos_version=34.20210611.3.0
 kernel=https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/${fcos_version}/x86_64/fedora-coreos-${fcos_version}-live-kernel-x86_64
 initrd=https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/${fcos_version}/x86_64/fedora-coreos-${fcos_version}-live-initramfs.x86_64.img
-ignition_url=http://mywebserver/fcos.ign
+rootfs=https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/${fcos_version}/x86_64/fedora-coreos-${fcos_version}-live-rootfs.x86_64.img
-kernel_args="ip=dhcp rd.neednet=1 console=tty0 coreos.liveiso=/ console=ttyS0 coreos.inst.install_dev=/dev/sda coreos.inst.stream=stable coreos.inst.ignition_url=${ignition_url}"
+kernel_args="console=ttyS0 coreos.live.rootfs_url=${rootfs} coreos.inst.install_dev=/dev/sda coreos.inst.stream=stable coreos.inst.ignition_url=${ignition_url}"
 sudo virt-install --name ${machine_name} --ram 4048 --graphics=none --vcpus 2 --disk size=20 \
                 --network bridge=virbr0 \
                 --install kernel=${kernel},initrd=${initrd},kernel_args_overwrite=yes,kernel_args="${kernel_args}"

docs/getting-started.md

@@ -11,7 +11,7 @@ You can use an
 to create or modify an Ansible inventory. Currently, it is limited in
 functionality and is only used for configuring a basic Kubespray cluster inventory, but it does
 support creating inventory file for large clusters as well. It now supports
-separated ETCD and Kubernetes master roles from node role if the size exceeds a
+separated ETCD and Kubernetes control plane roles from node role if the size exceeds a
 certain threshold. Run `python3 contrib/inventory_builder/inventory.py help` for more information.
 
 Example inventory generator usage:
@@ -40,7 +40,7 @@ See more details in the [ansible guide](/docs/ansible.md).
 
 ### Adding nodes
 
-You may want to add worker, master or etcd nodes to your existing cluster. This can be done by re-running the `cluster.yml` playbook, or you can target the bare minimum needed to get kubelet installed on the worker and talking to your masters. This is especially helpful when doing something like autoscaling your clusters.
+You may want to add worker, control plane or etcd nodes to your existing cluster. This can be done by re-running the `cluster.yml` playbook, or you can target the bare minimum needed to get kubelet installed on the worker and talking to your control planes. This is especially helpful when doing something like autoscaling your clusters.
 
 - Add the new worker node to your inventory in the appropriate group (or utilize a [dynamic inventory](https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html)).
 - Run the ansible-playbook command, substituting `cluster.yml` for `scale.yml`:
@@ -52,7 +52,7 @@ ansible-playbook -i inventory/mycluster/hosts.yml scale.yml -b -v \
 
 ### Remove nodes
 
-You may want to remove **master**, **worker**, or **etcd** nodes from your
+You may want to remove **control plane**, **worker**, or **etcd** nodes from your
 existing cluster. This can be done by re-running the `remove-node.yml`
 playbook. First, all specified nodes will be drained, then stop some
 kubernetes services and delete some certificates,
@@ -108,11 +108,11 @@ Accessing through Ingress is highly recommended. For proxy access, please note t
 
 For token authentication, guide to create Service Account is provided in [dashboard sample user](https://github.com/kubernetes/dashboard/blob/master/docs/user/access-control/creating-sample-user.md) doc. Still take care of default namespace.
 
-Access can also by achieved via ssh tunnel on a master :
+Access can also by achieved via ssh tunnel on a control plane :
 
 ```bash
-# localhost:8081 will be sent to master-1's own localhost:8081
+# localhost:8081 will be sent to control-plane-1's own localhost:8081
-ssh -L8001:localhost:8001 user@master-1
+ssh -L8001:localhost:8001 user@control-plane-1
 sudo -i
 kubectl proxy
 ```

docs/gvisor.md

@@ -0,0 +1,16 @@
+# gVisor
+
+[gVisor](https://gvisor.dev/docs/) is an application kernel, written in Go, that implements a substantial portion of the Linux system call interface. It provides an additional layer of isolation between running applications and the host operating system.
+
+gVisor includes an Open Container Initiative (OCI) runtime called runsc that makes it easy to work with existing container tooling. The runsc runtime integrates with Docker and Kubernetes, making it simple to run sandboxed containers.
+
+## Usage
+
+To enable gVisor you should be using a container manager that is compatible with selecting the [RuntimeClass](https://kubernetes.io/docs/concepts/containers/runtime-class/) such as `containerd`.
+
+Containerd support:
+
+```yaml
+container_manager: containerd
+gvisor_enabled: true
+```

docs/ingress_controller/ambassador.md

@@ -16,6 +16,16 @@ URL rewriting, CORS, rate limiting, and automatic metrics collection.
   installation/updates.
 * `ingress_ambassador_secure_port` (default: 443): HTTPS port to listen at.
 * `ingress_ambassador_insecure_port` (default: 80): HTTP port to listen at.
+* `ingress_ambassador_multi_namespaces` (default `false`): By default, Ambassador will only
+  watch the `ingress_ambassador_namespace` namespace for `AmbassadorInstallation` CRD resources.
+  When set to `true`, this value will tell the Ambassador Operator to watch **all** namespaces
+  for CRDs. If you want to run multiple Ambassador ingress instances, set this to `true`.
+
+### Ingress annotations
+
+The Ambassador API Gateway will automatically load balance `Ingress` resources
+that include the annotation `kubernetes.io/ingress.class=ambassador`. All the other
+resources will be just ignored.
 
 ### Ambassador Operator
 

docs/integration.md

@@ -95,7 +95,7 @@ If you made useful changes or fixed a bug in existent kubespray repo, use this f
 
 3. Setup desired user.name and user.email for submodule.
 If kubespray is only one submodule in your repo you could use something like:
-```git submodule foreach --recursive 'git config user.name "First Last" && git config user.email "your-email-addres@used.for.cncf"'```
+```git submodule foreach --recursive 'git config user.name "First Last" && git config user.email "your-email-address@used.for.cncf"'```
 
 4. Sync with upstream master:
 

docs/kubernetes-apps/cephfs_provisioner.md

@@ -1,12 +1,10 @@
-CephFS Volume Provisioner for Kubernetes 1.5+
+# CephFS Volume Provisioner for Kubernetes 1.5+
-=============================================
 
 [![Docker Repository on Quay](https://quay.io/repository/external_storage/cephfs-provisioner/status "Docker Repository on Quay")](https://quay.io/repository/external_storage/cephfs-provisioner)
 
 Using Ceph volume client
 
-Development
+## Development
------------
 
 Compile the provisioner
 
@@ -20,8 +18,7 @@ Make the container image and push to the registry
 make push
 ```
 
-Test instruction
+## Test instruction
-----------------
 
 - Start Kubernetes local cluster
 
@@ -65,14 +62,12 @@ kubectl create -f example/claim.yaml
 kubectl create -f example/test-pod.yaml
 ```
 
-Known limitations
+## Known limitations
------------------
 
 - Kernel CephFS doesn't work with SELinux, setting SELinux label in Pod's securityContext will not work.
 - Kernel CephFS doesn't support quota or capacity, capacity requested by PVC is not enforced or validated.
 - Currently each Ceph user created by the provisioner has `allow r` MDS cap to permit CephFS mount.
 
-Acknowledgement
+## Acknowledgement
----------------
 
 Inspired by CephFS Manila provisioner and conversation with John Spray

docs/kubernetes-apps/local_volume_provisioner.md

@@ -1,5 +1,4 @@
-Local Storage Provisioner
+# Local Storage Provisioner
-=========================
 
 The [local storage provisioner](https://github.com/kubernetes-incubator/external-storage/tree/master/local-volume)
 is NOT a dynamic storage provisioner as you would
@@ -47,8 +46,7 @@ data:
 The default StorageClass is local-storage on /mnt/disks,
 the rest of this doc will use that path as an example.
 
-Examples to create local storage volumes
+## Examples to create local storage volumes
-----------------------------------------
 
 1. tmpfs method:
 
@@ -80,7 +78,7 @@ for disk in /dev/sdc /dev/sdd /dev/sde; do
 done
 ```
 
-This saves time of precreatnig filesystems. Note that your storageclass must have
+This saves time of precreating filesystems. Note that your storageclass must have
 volume_mode set to "Filesystem" and fs_type defined. If either is not set, the
 disk will be added as a raw block device.
 
@@ -106,8 +104,7 @@ management.
 Create a symbolic link under discovery directory to the block device on the node. To use
 raw block devices in pods, volume_type should be set to "Block".
 
-Usage notes
+## Usage notes
------------
 
 Beta PV.NodeAffinity field is used by default. If running against an older K8s
 version, the useAlphaAPI flag must be set in the configMap.
@@ -120,7 +117,6 @@ Make sure to make any mounts persist via /etc/fstab or with systemd mounts (for
 Flatcar Container Linux). Pods with persistent volume claims will not be
 able to start if the mounts become unavailable.
 
-Further reading
+## Further reading
----------------
 
 Refer to the upstream docs here: <https://github.com/kubernetes-incubator/external-storage/tree/master/local-volume>

docs/kubernetes-apps/registry.md

@@ -1,12 +1,10 @@
-Private Docker Registry in Kubernetes
+# Private Docker Registry in Kubernetes
-=====================================
 
 Kubernetes offers an optional private Docker registry addon, which you can turn
 on when you bring up a cluster or install later. This gives you a place to
 store truly private Docker images for your cluster.
 
-How it works
+## How it works
-------------
 
 The private registry runs as a `Pod` in your cluster. It does not currently
 support SSL or authentication, which triggers Docker's "insecure registry"
@@ -14,8 +12,7 @@ logic. To work around this, we run a proxy on each node in the cluster,
 exposing a port onto the node (via a hostPort), which Docker accepts as
 "secure", since it is accessed by `localhost`.
 
-Turning it on
+## Turning it on
--------------
 
 Some cluster installs (e.g. GCE) support this as a cluster-birth flag. The
 `ENABLE_CLUSTER_REGISTRY` variable in `cluster/gce/config-default.sh` governs
@@ -24,7 +21,7 @@ whether the registry is run or not. To set this flag, you can specify
 does not include this flag, the following steps should work. Note that some of
 this is cloud-provider specific, so you may have to customize it a bit.
 
-- Make some storage
+### Make some storage
 
 The primary job of the registry is to store data. To do that we have to decide
 where to store it. For cloud environments that have networked storage, we can
@@ -58,15 +55,14 @@ If, for example, you wanted to use NFS you would just need to change the
 Note that in any case, the storage (in the case the GCE PersistentDisk) must be
 created independently - this is not something Kubernetes manages for you (yet).
 
-- I don't want or don't have persistent storage
+### I don't want or don't have persistent storage
 
 If you are running in a place that doesn't have networked storage, or if you
 just want to kick the tires on this without committing to it, you can easily
 adapt the `ReplicationController` specification below to use a simple
 `emptyDir` volume instead of a `persistentVolumeClaim`.
 
-Claim the storage
+## Claim the storage
------------------
 
 Now that the Kubernetes cluster knows that some storage exists, you can put a
 claim on that storage. As with the `PersistentVolume` above, you can start
@@ -93,8 +89,7 @@ you created before will be bound to this claim (unless you have other
 `PersistentVolumes` in which case those might get bound instead). This claim
 gives you the right to use this storage until you release the claim.
 
-Run the registry
+## Run the registry
-----------------
 
 Now we can run a Docker registry:
 
@@ -145,8 +140,7 @@ spec:
 ```
 <!-- END MUNGE: EXAMPLE registry-rc.yaml -->
 
-Expose the registry in the cluster
+## Expose the registry in the cluster
-----------------------------------
 
 Now that we have a registry `Pod` running, we can expose it as a Service:
 
@@ -170,8 +164,7 @@ spec:
 ```
 <!-- END MUNGE: EXAMPLE registry-svc.yaml -->
 
-Expose the registry on each node
+## Expose the registry on each node
---------------------------------
 
 Now that we have a running `Service`, we need to expose it onto each Kubernetes
 `Node` so that Docker will see it as `localhost`. We can load a `Pod` on every
@@ -229,8 +222,7 @@ $ curl localhost:5000
 404 page not found
 ```
 
-Using the registry
+## Using the registry
-------------------
 
 To use an image hosted by this registry, simply say this in your `Pod`'s
 `spec.containers[].image` field:
@@ -258,15 +250,3 @@ $ kubectl port-forward --namespace kube-system $POD 5000:5000 &
 Now you can build and push images on your local computer as
 `localhost:5000/yourname/container` and those images will be available inside
 your kubernetes cluster with the same name.
-
-More Extensions
----------------
-
-- [Use GCS as storage backend](gcs/README.md)
-- [Enable TLS/SSL](tls/README.md)
-- [Enable Authentication](auth/README.md)
-
-Future improvements
--------------------
-
-- Allow port-forwarding to a Service rather than a pod (\#15180)

docs/kubernetes-reliability.md

@@ -21,7 +21,7 @@ By default the normal behavior looks like:
 
 > Kubernetes controller manager and Kubelet work asynchronously. It means that
 > the delay may include any network latency, API Server latency, etcd latency,
-> latency caused by load on one's master nodes and so on. So if
+> latency caused by load on one's control plane nodes and so on. So if
 > `--node-status-update-frequency` is set to 5s in reality it may appear in
 > etcd in 6-7 seconds or even longer when etcd cannot commit data to quorum
 > nodes.
@@ -56,7 +56,7 @@ services so pods from failed node won't be accessible anymore.
 
 ## Fast Update and Fast Reaction
 
-If `-–node-status-update-frequency` is set to **4s** (10s is default).
+If `--node-status-update-frequency` is set to **4s** (10s is default).
 `--node-monitor-period` to **2s** (5s is default).
 `--node-monitor-grace-period` to **20s** (40s is default).
 `--default-not-ready-toleration-seconds` and ``--default-unreachable-toleration-seconds`` are set to **30**
@@ -78,7 +78,7 @@ minute which may require large etcd containers or even dedicated nodes for etcd.
 
 ## Medium Update and Average Reaction
 
-Let's set `-–node-status-update-frequency` to **20s**
+Let's set `--node-status-update-frequency` to **20s**
 `--node-monitor-grace-period` to **2m** and `--default-not-ready-toleration-seconds` and
 ``--default-unreachable-toleration-seconds`` to **60**.
 In that case, Kubelet will try to update status every 20s. So, it will be 6 * 5
@@ -94,7 +94,7 @@ etcd updates per minute.
 
 ## Low Update and Slow reaction
 
-Let's set `-–node-status-update-frequency` to **1m**.
+Let's set `--node-status-update-frequency` to **1m**.
 `--node-monitor-grace-period` will set to **5m** and `--default-not-ready-toleration-seconds` and
 ``--default-unreachable-toleration-seconds`` to **60**. In this scenario, every kubelet will try to update the status
 every minute. There will be 5 * 5 = 25 attempts before unhealthy status. After 5m,

docs/large-deployments.md

@@ -11,8 +11,8 @@ For a large scaled deployments, consider the following configuration changes:
 * Override the ``download_run_once: true`` and/or ``download_localhost: true``.
   See download modes for details.
 
-* Adjust the `retry_stagger` global var as appropriate. It should provide sane
+* Adjust the `retry_stagger` global var as appropriate. It should provide same
-  load on a delegate (the first K8s master node) then retrying failed
+  load on a delegate (the first K8s control plane node) then retrying failed
   push or download operations.
 
 * Tune parameters for DNS related applications

docs/metallb.md

@@ -4,6 +4,14 @@ MetalLB hooks into your Kubernetes cluster, and provides a network load-balancer
 It allows you to create Kubernetes services of type "LoadBalancer" in clusters that don't run on a cloud provider, and thus cannot simply hook into 3rd party products to provide load-balancers.
 The default operationg mode of MetalLB is in ["Layer2"](https://metallb.universe.tf/concepts/layer2/) but it can also operate in ["BGP"](https://metallb.universe.tf/concepts/bgp/) mode.
 
+## Prerequisites
+
+You have to configure arp_ignore and arp_announce to avoid answering ARP queries from kube-ipvs0 interface for MetalLB to work.
+
+```yaml
+kube_proxy_strict_arp: true
+```
+
 ## Install
 
 You have to explicitly enable the MetalLB extension and set an IP address range from which to allocate LoadBalancer IPs.

docs/nodes.md

@@ -6,9 +6,9 @@ Modified from [comments in #3471](https://github.com/kubernetes-sigs/kubespray/i
 
 Currently you can't remove the first node in your kube_control_plane and etcd-master list. If you still want to remove this node you have to:
 
-### 1) Change order of current masters
+### 1) Change order of current control planes
 
-Modify the order of your master list by pushing your first entry to any other position. E.g. if you want to remove `node-1` of the following example:
+Modify the order of your control plane list by pushing your first entry to any other position. E.g. if you want to remove `node-1` of the following example:
 
 ```yaml
   children:
@@ -69,21 +69,21 @@ Before using `--limit` run playbook `facts.yml` without the limit to refresh fac
 ### 3) Remove an old node with remove-node.yml
 
 With the old node still in the inventory, run `remove-node.yml`. You need to pass `-e node=NODE_NAME` to the playbook to limit the execution to the node being removed.
-  
-If the node you want to remove is not online, you should add `reset_nodes=false` to your extra-vars: `-e node=NODE_NAME -e reset_nodes=false`.
-Use this flag even when you remove other types of nodes like a master or etcd nodes.
 
-### 5) Remove the node from the inventory
+If the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars: `-e node=NODE_NAME -e reset_nodes=false -e allow_ungraceful_removal=true`.
+Use this flag even when you remove other types of nodes like a control plane or etcd nodes.
+
+### 4) Remove the node from the inventory
 
 That's it.
 
-## Adding/replacing a master node
+## Adding/replacing a control plane node
 
 ### 1) Run `cluster.yml`
 
 Append the new host to the inventory and run `cluster.yml`. You can NOT use `scale.yml` for that.
 
-### 3) Restart kube-system/nginx-proxy
+### 2) Restart kube-system/nginx-proxy
 
 In all hosts, restart nginx-proxy pod. This pod is a local proxy for the apiserver. Kubespray will update its static config, but it needs to be restarted in order to reload.
 
@@ -92,10 +92,49 @@ In all hosts, restart nginx-proxy pod. This pod is a local proxy for the apiserv
 docker ps | grep k8s_nginx-proxy_nginx-proxy | awk '{print $1}' | xargs docker restart
 ```
 
-### 4) Remove old master nodes
+### 3) Remove old control plane nodes
 
 With the old node still in the inventory, run `remove-node.yml`. You need to pass `-e node=NODE_NAME` to the playbook to limit the execution to the node being removed.
-If the node you want to remove is not online, you should add `reset_nodes=false` to your extra-vars.
+If the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars.
+
+## Replacing a first control plane node
+
+### 1) Change control plane nodes order in inventory
+
+from
+
+```ini
+[kube_control_plane]
+ node-1
+ node-2
+ node-3
+```
+
+to
+
+```ini
+[kube_control_plane]
+ node-2
+ node-3
+ node-1
+```
+
+### 2) Remove old first control plane node from cluster
+
+With the old node still in the inventory, run `remove-node.yml`. You need to pass `-e node=node-1` to the playbook to limit the execution to the node being removed.
+If the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars.
+
+### 3) Edit cluster-info configmap in kube-system namespace
+
+`kubectl  edit cm -n kube-public cluster-info`
+
+Change ip of old kube_control_plane node with ip of live kube_control_plane node (`server` field). Also, update `certificate-authority-data` field if you changed certs.
+
+### 4) Add new control plane node
+
+Update inventory (if needed)
+
+Run `cluster.yml` with `--limit=kube_control_plane`
 
 ## Adding an etcd node
 
@@ -104,23 +143,27 @@ You need to make sure there are always an odd number of etcd nodes in the cluste
 ### 1) Add the new node running cluster.yml
 
 Update the inventory and run `cluster.yml` passing `--limit=etcd,kube_control_plane -e ignore_assert_errors=yes`.
-If the node you want to add as an etcd node is already a worker or master node in your cluster, you have to remove him first using `remove-node.yml`.
+If the node you want to add as an etcd node is already a worker or control plane node in your cluster, you have to remove him first using `remove-node.yml`.
 
-Run `upgrade-cluster.yml` also passing `--limit=etcd,kube_control_plane -e ignore_assert_errors=yes`. This is necessary to update all etcd configuration in the cluster.  
+Run `upgrade-cluster.yml` also passing `--limit=etcd,kube_control_plane -e ignore_assert_errors=yes`. This is necessary to update all etcd configuration in the cluster.
 
 At this point, you will have an even number of nodes.
 Everything should still be working, and you should only have problems if the cluster decides to elect a new etcd leader before you remove a node.
 Even so, running applications should continue to be available.
 
-If you add multiple ectd nodes with one run, you might want to append `-e etcd_retries=10` to increase the amount of retries between each ectd node join.
+If you add multiple etcd nodes with one run, you might want to append `-e etcd_retries=10` to increase the amount of retries between each etcd node join.
 Otherwise the etcd cluster might still be processing the first join and fail on subsequent nodes. `etcd_retries=10` might work to join 3 new nodes.
 
+### 2) Add the new node to apiserver config
+
+In every control plane node, edit `/etc/kubernetes/manifests/kube-apiserver.yaml`. Make sure the new etcd nodes are present in the apiserver command line parameter `--etcd-servers=...`.
+
 ## Removing an etcd node
 
 ### 1) Remove an old etcd node
 
 With the node still in the inventory, run `remove-node.yml` passing `-e node=NODE_NAME` as the name of the node that should be removed.
-If the node you want to remove is not online, you should add `reset_nodes=false` to your extra-vars.
+If the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars.
 
 ### 2) Make sure only remaining nodes are in your inventory
 
@@ -130,6 +173,10 @@ Remove `NODE_NAME` from your inventory file.
 
 Run `cluster.yml` to regenerate the configuration files on all remaining nodes.
 
-### 4) Shutdown the old instance
+### 4) Remove the old etcd node from apiserver config
+
+In every control plane node, edit `/etc/kubernetes/manifests/kube-apiserver.yaml`. Make sure only active etcd nodes are still present in the apiserver command line parameter `--etcd-servers=...`.
+
+### 5) Shutdown the old instance
 
 That's it.

docs/openstack.md

@@ -12,6 +12,7 @@ Kubespray has been tested on a number of OpenStack Public Clouds including (in a
 - [ELASTX](https://elastx.se/)
 - [EnterCloudSuite](https://www.entercloudsuite.com/)
 - [FugaCloud](https://fuga.cloud/)
+- [Infomaniak](https://infomaniak.com)
 - [Open Telekom Cloud](https://cloud.telekom.de/) : requires to set the variable `wait_for_floatingip = "true"` in your cluster.tfvars
 - [OVHcloud](https://www.ovhcloud.com/)
 - [Rackspace](https://www.rackspace.com/)
@@ -104,6 +105,12 @@ The new cloud provider is configured to have Octavia by default in Kubespray.
   cinder_topology: true
   ```
 
+- Enabling `cinder_csi_ignore_volume_az: true`, ignores volumeAZ and schedules on any of the available node AZ.
+
+  ```yaml
+  cinder_csi_ignore_volume_az: true
+  ```
+
 - If you are using OpenStack loadbalancer(s) replace the `openstack_lbaas_subnet_id` with the new `external_openstack_lbaas_subnet_id`. **Note** The new cloud provider is using Octavia instead of Neutron LBaaS by default!
 - Enable 3 feature gates to allow migration of all volumes and storage classes (if you have any feature gates already set just add the 3 listed below):
 

docs/proxy.md

@@ -18,6 +18,6 @@ If you set http and https proxy, all nodes and loadbalancer will be excluded fro
 ## Exclude workers from no_proxy
 
 Since workers are included in the no_proxy variable, by default, docker engine will be restarted on all nodes (all
-pods will restart) when adding or removing workers.  To override this behaviour by only including master nodes in the
+pods will restart) when adding or removing workers.  To override this behaviour by only including control plane nodes in the
 no_proxy variable, set:
 `no_proxy_exclude_workers: true`

docs/recover-control-plane.md

@@ -20,7 +20,7 @@ __Note that you need at least one functional node to be able to recover using th
 ## Runbook
 
 * Move any broken etcd nodes into the "broken\_etcd" group, make sure the "etcd\_member\_name" variable is set.
-* Move any broken master nodes into the "broken\_kube\_control\_plane" group.
+* Move any broken control plane nodes into the "broken\_kube\_control\_plane" group.
 
 Then run the playbook with ```--limit etcd,kube_control_plane``` and increase the number of ETCD retries by setting ```-e etcd_retries=10``` or something even larger. The amount of retries required is difficult to predict.
 

docs/roadmap.md

@@ -14,7 +14,7 @@
   - [ ] GCE
   - [x] AWS (contrib/terraform/aws)
   - [x] OpenStack (contrib/terraform/openstack)
-  - [x] Packet
+  - [x] Equinix Metal
   - [ ] Digital Ocean
   - [ ] Azure
 - [ ] On AWS autoscaling, multi AZ
@@ -28,7 +28,7 @@
 - [x] Run kubernetes e2e tests
 - [ ] Test idempotency on single OS but for all network plugins/container engines
 - [ ] single test on AWS per day
-- [ ] test scale up cluster:  +1 etcd, +1 master, +1 node
+- [ ] test scale up cluster:  +1 etcd, +1 control plane, +1 node
 - [x] Reorganize CI test vars into group var files
 
 ## Lifecycle

docs/test_cases.md

@@ -8,7 +8,7 @@ and the `etcd` group merged with the `kube_control_plane`.
 `separate` layout is when there is only node of each type, which includes
  a kube_control_plane, kube_node, and etcd cluster member.
 
-`ha` layout consists of two etcd nodes, two masters and a single worker node,
+`ha` layout consists of two etcd nodes, two control planes and a single worker node,
 with role intersection.
 
 `scale` layout can be combined with above layouts (`ha-scale`, `separate-scale`). It includes 200 fake hosts

docs/upgrades.md

@@ -62,6 +62,29 @@ If you want to manually control the upgrade procedure, you can use the variables
 `upgrade_node_confirm: true` - waiting to confirmation to upgrade next node
 `upgrade_node_pause_seconds: 60` - pause 60 seconds before upgrade next node
 
+## Node-based upgrade
+
+If you don't want to upgrade all nodes in one run, you can use `--limit` [patterns](https://docs.ansible.com/ansible/latest/user_guide/intro_patterns.html#patterns-and-ansible-playbook-flags).
+
+Before using `--limit` run playbook `facts.yml` without the limit to refresh facts cache for all nodes:
+
+```ShellSession
+ansible-playbook facts.yml -b -i inventory/sample/hosts.ini
+```
+
+After this upgrade control plane and etcd groups [#5147](https://github.com/kubernetes-sigs/kubespray/issues/5147):
+
+```ShellSession
+ansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=v1.20.7 --limit "kube_control_plane:etcd"
+```
+
+Now you can upgrade other nodes in any order and quantity:
+
+```ShellSession
+ansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=v1.20.7 --limit "node4:node6:node7:node12"
+ansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=v1.20.7 --limit "node5*"
+```
+
 ## Multiple upgrades
 
 :warning: [Do not skip releases when upgrading--upgrade by one tag at a time.](https://github.com/kubernetes-sigs/kubespray/issues/3849#issuecomment-451386515) :warning:
@@ -313,6 +336,12 @@ Upgrade etcd:
 ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=etcd
 ```
 
+Upgrade etcd without rotating etcd certs:
+
+```ShellSession
+ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=etcd --limit=etcd --skip-tags=etcd-secrets
+```
+
 Upgrade kubelet:
 
 ```ShellSession

docs/vars.md

@@ -26,7 +26,7 @@ Some variables of note include:
 * *kube_version* - Specify a given Kubernetes version
 * *searchdomains* - Array of DNS domains to search when looking up hostnames
 * *nameservers* - Array of nameservers to use for DNS lookup
-* *preinstall_selinux_state* - Set selinux state, permitted values are permissive and disabled.
+* *preinstall_selinux_state* - Set selinux state, permitted values are permissive, enforcing and disabled.
 
 ## Addressing variables
 
@@ -136,14 +136,12 @@ Stack](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/dns-stack.m
   [kubelet-rubber-stamp](https://github.com/kontena/kubelet-rubber-stamp).
 * *node_labels* - Labels applied to nodes via kubelet --node-labels parameter.
   For example, labels can be set in the inventory as variables or more widely in group_vars.
-  *node_labels* can be defined either as a dict or a comma-separated labels string:
+  *node_labels* can only be defined as a dict:
 
 ```yml
 node_labels:
   label1_name: label1_value
   label2_name: label2_value
-
-node_labels: "label1_name=label1_value,label2_name=label2_value"
 ```
 
 * *node_taints* - Taints applied to nodes via kubelet --register-with-taints parameter.
@@ -180,7 +178,7 @@ node_taints:
 For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments.
 
 Extra flags for the kubelet can be specified using these variables,
-in the form of dicts of key-value pairs of configuration parameters that will be inserted into the kubelet YAML config file. The `kubelet_node_config_extra_args` apply kubelet settings only to nodes and not masters. Example:
+in the form of dicts of key-value pairs of configuration parameters that will be inserted into the kubelet YAML config file. The `kubelet_node_config_extra_args` apply kubelet settings only to nodes and not control planes. Example:
 
 ```yml
 kubelet_config_extra_args:
@@ -202,7 +200,7 @@ Previously, the same parameters could be passed as flags to kubelet binary with
 * *kubelet_custom_flags*
 * *kubelet_node_custom_flags*
 
-The `kubelet_node_custom_flags` apply kubelet settings only to nodes and not masters. Example:
+The `kubelet_node_custom_flags` apply kubelet settings only to nodes and not control planes. Example:
 
 ```yml
 kubelet_custom_flags:

docs/vsphere-csi.md

@@ -2,32 +2,41 @@
 
 vSphere CSI driver allows you to provision volumes over a vSphere deployment. The Kubernetes historic in-tree cloud provider is deprecated and will be removed in future versions.
 
+## Prerequisites
+
+The vSphere user for CSI driver requires a set of privileges to perform Cloud Native Storage operations. Follow the [official guide](https://vsphere-csi-driver.sigs.k8s.io/driver-deployment/prerequisites.html#roles_and_privileges) to configure those.
+
+## Kubespray configuration
+
 To enable vSphere CSI driver, uncomment the `vsphere_csi_enabled` option in `group_vars/all/vsphere.yml` and set it to `true`.
 
 To set the number of replicas for the vSphere CSI controller, you can change `vsphere_csi_controller_replicas` option in `group_vars/all/vsphere.yml`.
 
 You need to source the vSphere credentials you use to deploy your machines that will host Kubernetes.
 
-| Variable                                    | Required | Type    | Choices                    | Default                   | Comment                                                        |
+| Variable                                    | Required | Type    | Choices                    | Default                   | Comment                                                                                                             |
-|---------------------------------------------|----------|---------|----------------------------|---------------------------|----------------------------------------------------------------|
+|---------------------------------------------|----------|---------|----------------------------|---------------------------|---------------------------------------------------------------------------------------------------------------------|
-| external_vsphere_vcenter_ip                 | TRUE     | string  |                            |                           | IP/URL of the vCenter                                          |
+| external_vsphere_vcenter_ip                 | TRUE     | string  |                            |                           | IP/URL of the vCenter                                                                                               |
-| external_vsphere_vcenter_port               | TRUE     | string  |                            | "443"                     | Port of the vCenter API                                        |
+| external_vsphere_vcenter_port               | TRUE     | string  |                            | "443"                     | Port of the vCenter API                                                                                             |
-| external_vsphere_insecure                   | TRUE     | string  | "true", "false"            | "true"                    | set to "true" if the host above uses a self-signed cert        |
+| external_vsphere_insecure                   | TRUE     | string  | "true", "false"            | "true"                    | set to "true" if the host above uses a self-signed cert                                                             |
-| external_vsphere_user                       | TRUE     | string  |                            |                           | User name for vCenter with required privileges                 |
+| external_vsphere_user                       | TRUE     | string  |                            |                           | User name for vCenter with required privileges (Can also be specified with the `VSPHERE_USER` environment variable) |
-| external_vsphere_password                   | TRUE     | string  |                            |                           | Password for vCenter                                           |
+| external_vsphere_password                   | TRUE     | string  |                            |                           | Password for vCenter (Can also be specified with the `VSPHERE_PASSWORD` environment variable)                       |
-| external_vsphere_datacenter                 | TRUE     | string  |                            |                           | Datacenter name to use                                         |
+| external_vsphere_datacenter                 | TRUE     | string  |                            |                           | Datacenter name to use                                                                                              |
-| external_vsphere_kubernetes_cluster_id      | TRUE     | string  |                            | "kubernetes-cluster-id"   | Kubernetes cluster ID to use                                   |
+| external_vsphere_kubernetes_cluster_id      | TRUE     | string  |                            | "kubernetes-cluster-id"   | Kubernetes cluster ID to use                                                                                        |
-| external_vsphere_version          | TRUE     | string  |                            | "6.7u3"                  | Vmware Vsphere version where located all VMs                                   |
+| external_vsphere_version                    | TRUE     | string  |                            | "6.7u3"                   | Vmware Vsphere version where located all VMs                                                                        |
-| vsphere_cloud_controller_image_tag          | TRUE     | string  |                            | "latest"                  | Kubernetes cluster ID to use                                   |
+| external_vsphere_cloud_controller_image_tag          | TRUE     | string  |                            | "latest"                  | Kubernetes cluster ID to use                                                                                        |
-| vsphere_syncer_image_tag                    | TRUE     | string  |                            | "v1.0.2"                  | Syncer image tag to use                                        |
+| vsphere_syncer_image_tag                    | TRUE     | string  |                            | "v2.2.1"                  | Syncer image tag to use                                                                                             |
-| vsphere_csi_attacher_image_tag              | TRUE     | string  |                            | "v1.1.1"                  | CSI attacher image tag to use                                  |
+| vsphere_csi_attacher_image_tag              | TRUE     | string  |                            | "v3.1.0"                  | CSI attacher image tag to use                                                                                       |
-| vsphere_csi_controller                      | TRUE     | string  |                            | "v1.0.2"                  | CSI controller image tag to use                                |
+| vsphere_csi_controller                      | TRUE     | string  |                            | "v2.2.1"                  | CSI controller image tag to use                                                                                     |
-| vsphere_csi_controller_replicas             | TRUE     | integer |                            | 1                         | Number of pods Kubernetes should deploy for the CSI controller |
+| vsphere_csi_controller_replicas             | TRUE     | integer |                            | 1                         | Number of pods Kubernetes should deploy for the CSI controller                                                      |
-| vsphere_csi_liveness_probe_image_tag        | TRUE     | string  |                            | "v1.1.0"                  | CSI liveness probe image tag to use                            |
+| vsphere_csi_liveness_probe_image_tag        | TRUE     | string  |                            | "v2.2.0"                  | CSI liveness probe image tag to use                                                                                 |
-| vsphere_csi_provisioner_image_tag           | TRUE     | string  |                            | "v1.2.2"                  | CSI provisioner image tag to use                               |
+| vsphere_csi_provisioner_image_tag           | TRUE     | string  |                            | "v2.1.0"                  | CSI provisioner image tag to use                                                                                    |
-| vsphere_csi_node_driver_registrar_image_tag | TRUE     | string  |                            | "v1.1.0"                  | CSI node driver registrat image tag to use                     |
+| vsphere_csi_node_driver_registrar_image_tag | TRUE     | string  |                            | "v1.1.0"                  | CSI node driver registrat image tag to use                                                                          |
-| vsphere_csi_driver_image_tag                | TRUE     | string  |                            | "v1.0.2"                  | CSI driver image tag to use                                    |
+| vsphere_csi_driver_image_tag                | TRUE     | string  |                            | "v1.0.2"                  | CSI driver image tag to use                                                                                         |
-vsphere_csi_resizer_tag                | TRUE     | string  |                            |  "v1.0.0"                  | CSI resizer image tag to use
+| vsphere_csi_resizer_tag                     | TRUE     | string  |                            | "v1.1.0"                  | CSI resizer image tag to use
+| vsphere_csi_aggressive_node_drain           | FALSE    | boolean |                            | false                     | Enable aggressive node drain strategy                                                                               |
+| vsphere_csi_aggressive_node_unreachable_timeout            | FALSE     | int  | 300   |                           | Timeout till node will be drained when it in an unreachable state                                                           |
+| vsphere_csi_aggressive_node_not_ready_timeout              | FALSE     | int  | 300   |                           | Timeout till node will be drained when it in not-ready state                                                                |
 
 ## Usage example
 
@@ -61,7 +70,7 @@ spec:
     - containerPort: 80
       protocol: TCP
     volumeMounts:
-      - mountPath: /var/lib/www/html
+      - mountPath: /usr/share/nginx/html
         name: csi-data-vsphere
   volumes:
   - name: csi-data-vsphere
@@ -83,8 +92,8 @@ csi-pvc-vsphere   Bound    pvc-dc7b1d21-ee41-45e1-98d9-e877cc1533ac   1Gi
 And the volume mounted to the Nginx Pod (wait until the Pod is Running):
 
 ```ShellSession
-kubectl exec -it nginx -- df -h | grep /var/lib/www/html
+kubectl exec -it nginx -- df -h | grep /usr/share/nginx/html
-/dev/sdb         976M  2.6M  907M   1% /var/lib/www/html
+/dev/sdb         976M  2.6M  907M   1% /usr/share/nginx/html
 ```
 
 ## More info

docs/vsphere.md

@@ -30,16 +30,16 @@ external_cloud_provider: "vsphere"
 
 Then, `inventory/sample/group_vars/vsphere.yml`, you need to declare your vCenter credentials and enable the vSphere CSI following the description below.
 
-| Variable                               | Required | Type    | Choices                    | Default | Comment                                                                   |
+| Variable                               | Required | Type    | Choices                    | Default                   | Comment                                                                                                             |
-|----------------------------------------|----------|---------|----------------------------|---------|---------------------------------------------------------------------------|
+|----------------------------------------|----------|---------|----------------------------|---------------------------|---------------------------------------------------------------------------------------------------------------------|
-| external_vsphere_vcenter_ip            | TRUE     | string  |                            |                           | IP/URL of the vCenter                                   |
+| external_vsphere_vcenter_ip            | TRUE     | string  |                            |                           | IP/URL of the vCenter                                                                                               |
-| external_vsphere_vcenter_port          | TRUE     | string  |                            | "443"                     | Port of the vCenter API                                 |
+| external_vsphere_vcenter_port          | TRUE     | string  |                            | "443"                     | Port of the vCenter API                                                                                             |
-| external_vsphere_insecure              | TRUE     | string  | "true", "false"            | "true"                    | set to "true" if the host above uses a self-signed cert |
+| external_vsphere_insecure              | TRUE     | string  | "true", "false"            | "true"                    | set to "true" if the host above uses a self-signed cert                                                             |
-| external_vsphere_user                  | TRUE     | string  |                            |                           | User name for vCenter with required privileges          |
+| external_vsphere_user                  | TRUE     | string  |                            |                           | User name for vCenter with required privileges (Can also be specified with the `VSPHERE_USER` environment variable) |
-| external_vsphere_password              | TRUE     | string  |                            |                           | Password for vCenter                                    |
+| external_vsphere_password              | TRUE     | string  |                            |                           | Password for vCenter (Can also be specified with the `VSPHERE_PASSWORD` environment variable)                       |
-| external_vsphere_datacenter            | TRUE     | string  |                            |                           | Datacenter name to use                                  |
+| external_vsphere_datacenter            | TRUE     | string  |                            |                           | Datacenter name to use                                                                                              |
-| external_vsphere_kubernetes_cluster_id | TRUE     | string  |                            | "kubernetes-cluster-id"   | Kubernetes cluster ID to use                            |
+| external_vsphere_kubernetes_cluster_id | TRUE     | string  |                            | "kubernetes-cluster-id"   | Kubernetes cluster ID to use                                                                                        |
-| vsphere_csi_enabled                    | TRUE     | boolean |                            | false                     | Enable vSphere CSI                                      |
+| vsphere_csi_enabled                    | TRUE     | boolean |                            | false                     | Enable vSphere CSI                                                                                                  |
 
 Example configuration:
 

facts.yml

@@ -2,6 +2,7 @@
 - name: Gather facts
   hosts: k8s_cluster:etcd:calico_rr
   gather_facts: False
+  tags: always
   tasks:
     - name: Gather minimal facts
       setup: