debian: Fix test failed after bullseye release (#7888)

(cherry picked from commit 79166496f3)

.ansible-lint

@@ -18,13 +18,3 @@ skip_list:
   # While it can be useful to have these metadata available, they are also available in the existing documentation.
   # (Disabled in May 2019)
   - '701'
-
-  # [role-name] "meta/main.yml" Role name role-name does not match ``^+$`` pattern
-  # Meta roles in Kubespray don't need proper names
-  # (Disabled in June 2021)
-  - 'role-name'
-
-  # [var-naming] "defaults/main.yml" File defines variable 'apiVersion' that violates variable naming standards
-  # In Kubespray we use variables that use camelCase to match their k8s counterparts
-  # (Disabled in June 2021)
-  - 'var-naming'

.gitlab-ci.yml

@@ -8,7 +8,7 @@ stages:
   - deploy-special
 
 variables:
-  KUBESPRAY_VERSION: v2.16.0
+  KUBESPRAY_VERSION: v2.15.1
   FAILFASTCI_NAMESPACE: 'kargo-ci'
   GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray'
   ANSIBLE_FORCE_COLOR: "true"
@@ -31,13 +31,12 @@ variables:
   ANSIBLE_LOG_LEVEL: "-vv"
   RECOVER_CONTROL_PLANE_TEST: "false"
   RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:],kube_control_plane[1:]"
-  TERRAFORM_14_VERSION: 0.14.11
+  TERRAFORM_14_VERSION: 0.14.10
-  TERRAFORM_15_VERSION: 0.15.5
+  TERRAFORM_13_VERSION: 0.13.6
 
 before_script:
   - ./tests/scripts/rebase.sh
   - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
-  - python -m pip uninstall -y ansible
   - python -m pip install -r tests/requirements.txt
   - mkdir -p /.ssh
 
@@ -52,7 +51,6 @@ before_script:
 
 .testcases: &testcases
   <<: *job
-  retry: 1
   before_script:
     - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
     - ./tests/scripts/rebase.sh

.gitlab-ci/lint.yml

@@ -14,7 +14,7 @@ vagrant-validate:
   stage: unit-tests
   tags: [light]
   variables:
-    VAGRANT_VERSION: 2.2.19
+    VAGRANT_VERSION: 2.2.15
   script:
     - ./tests/scripts/vagrant-validate.sh
   except: ['triggers', 'master']
@@ -53,7 +53,6 @@ tox-inventory-builder:
     - ./tests/scripts/rebase.sh
     - apt-get update && apt-get install -y python3-pip
     - update-alternatives --install /usr/bin/python python /usr/bin/python3 10
-    - python -m pip uninstall -y ansible
     - python -m pip install -r tests/requirements.txt
   script:
     - pip3 install tox

.gitlab-ci/packet.yml

@@ -91,11 +91,6 @@ packet_debian10-containerd:
   variables:
     MITOGEN_ENABLE: "true"
 
-packet_debian11-calico:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: on_success
-
 packet_centos7-calico-ha-once-localhost:
   stage: deploy-part2
   extends: .packet_pr
@@ -116,7 +111,7 @@ packet_centos8-calico:
   extends: .packet_pr
   when: on_success
 
-packet_fedora34-weave:
+packet_fedora32-weave:
   stage: deploy-part2
   extends: .packet_pr
   when: on_success
@@ -182,18 +177,15 @@ packet_fedora33-calico:
   stage: deploy-part2
   extends: .packet_periodic
   when: on_success
-
+  variables:
-packet_fedora34-calico-selinux:
+    MITOGEN_ENABLE: "true"
-  stage: deploy-part2
-  extends: .packet_periodic
-  when: on_success
 
 packet_amazon-linux-2-aio:
   stage: deploy-part2
   extends: .packet_pr
   when: manual
 
-packet_fedora34-kube-ovn-containerd:
+packet_fedora32-kube-ovn-containerd:
   stage: deploy-part2
   extends: .packet_periodic
   when: on_success
@@ -209,14 +201,6 @@ packet_centos7-weave-upgrade-ha:
     UPGRADE_TEST: basic
     MITOGEN_ENABLE: "false"
 
-# Calico HA Wireguard
-packet_ubuntu20-calico-ha-wireguard:
-  stage: deploy-part2
-  extends: .packet_pr
-  when: manual
-  variables:
-    MITOGEN_ENABLE: "true"
-
 packet_debian9-calico-upgrade:
   stage: deploy-part3
   extends: .packet_pr

.gitlab-ci/terraform.yml

@@ -12,13 +12,13 @@
     # Prepare inventory
     - cp contrib/terraform/$PROVIDER/sample-inventory/cluster.tfvars .
     - ln -s contrib/terraform/$PROVIDER/hosts
-    - terraform -chdir="contrib/terraform/$PROVIDER" init
+    - terraform init contrib/terraform/$PROVIDER
     # Copy SSH keypair
     - mkdir -p ~/.ssh
     - echo "$PACKET_PRIVATE_KEY" | base64 -d > ~/.ssh/id_rsa
     - chmod 400 ~/.ssh/id_rsa
     - echo "$PACKET_PUBLIC_KEY" | base64 -d > ~/.ssh/id_rsa.pub
-    - mkdir -p contrib/terraform/$PROVIDER/group_vars
+    - mkdir -p group_vars
     # Random subnet to avoid routing conflicts
     - export TF_VAR_subnet_cidr="10.$(( $RANDOM % 256 )).$(( $RANDOM % 256 )).0/24"
 
@@ -28,8 +28,8 @@
   tags: [light]
   only: ['master', /^pr-.*$/]
   script:
-    - terraform -chdir="contrib/terraform/$PROVIDER" validate
+    - terraform validate -var-file=cluster.tfvars contrib/terraform/$PROVIDER
-    - terraform -chdir="contrib/terraform/$PROVIDER" fmt -check -diff
+    - terraform fmt -check -diff contrib/terraform/$PROVIDER
 
 .terraform_apply:
   extends: .terraform_install
@@ -53,44 +53,44 @@
     # Cleanup regardless of exit code
     - chronic ./tests/scripts/testcases_cleanup.sh
 
-tf-0.15.x-validate-openstack:
+tf-0.13.x-validate-openstack:
   extends: .terraform_validate
   variables:
-    TF_VERSION: $TERRAFORM_15_VERSION
+    TF_VERSION: $TERRAFORM_13_VERSION
     PROVIDER: openstack
     CLUSTER: $CI_COMMIT_REF_NAME
 
-tf-0.15.x-validate-packet:
+tf-0.13.x-validate-packet:
   extends: .terraform_validate
   variables:
-    TF_VERSION: $TERRAFORM_15_VERSION
+    TF_VERSION: $TERRAFORM_13_VERSION
     PROVIDER: packet
     CLUSTER: $CI_COMMIT_REF_NAME
 
-tf-0.15.x-validate-aws:
+tf-0.13.x-validate-aws:
   extends: .terraform_validate
   variables:
-    TF_VERSION: $TERRAFORM_15_VERSION
+    TF_VERSION: $TERRAFORM_13_VERSION
     PROVIDER: aws
     CLUSTER: $CI_COMMIT_REF_NAME
 
-tf-0.15.x-validate-exoscale:
+tf-0.13.x-validate-exoscale:
   extends: .terraform_validate
   variables:
-    TF_VERSION: $TERRAFORM_15_VERSION
+    TF_VERSION: $TERRAFORM_13_VERSION
     PROVIDER: exoscale
 
-tf-0.15.x-validate-vsphere:
+tf-0.13.x-validate-vsphere:
   extends: .terraform_validate
   variables:
-    TF_VERSION: $TERRAFORM_15_VERSION
+    TF_VERSION: $TERRAFORM_13_VERSION
     PROVIDER: vsphere
     CLUSTER: $CI_COMMIT_REF_NAME
 
-tf-0.15.x-validate-upcloud:
+tf-0.13.x-validate-upcloud:
   extends: .terraform_validate
   variables:
-    TF_VERSION: $TERRAFORM_15_VERSION
+    TF_VERSION: $TERRAFORM_13_VERSION
     PROVIDER: upcloud
     CLUSTER: $CI_COMMIT_REF_NAME
 
@@ -210,7 +210,7 @@ tf-elastx_ubuntu18-calico:
   allow_failure: true
   variables:
     <<: *elastx_variables
-    TF_VERSION: $TERRAFORM_15_VERSION
+    TF_VERSION: $TERRAFORM_14_VERSION
     PROVIDER: openstack
     CLUSTER: $CI_COMMIT_REF_NAME
     ANSIBLE_TIMEOUT: "60"

.gitlab-ci/vagrant.yml

@@ -11,7 +11,6 @@ molecule_tests:
     - tests/scripts/rebase.sh
     - apt-get update && apt-get install -y python3-pip
     - update-alternatives --install /usr/bin/python python /usr/bin/python3 10
-    - python -m pip uninstall -y ansible
     - python -m pip install -r tests/requirements.txt
     - ./tests/scripts/vagrant_clean.sh
   script:
@@ -32,7 +31,6 @@ molecule_tests:
   before_script:
     - apt-get update && apt-get install -y python3-pip
     - update-alternatives --install /usr/bin/python python /usr/bin/python3 10
-    - python -m pip uninstall -y ansible
     - python -m pip install -r tests/requirements.txt
     - ./tests/scripts/vagrant_clean.sh
   script:

Dockerfile

@@ -4,7 +4,7 @@ FROM ubuntu:bionic-20200807
 RUN apt update -y \
     && apt install -y \
     libssl-dev python3-dev sshpass apt-transport-https jq moreutils \
-    ca-certificates curl gnupg2 software-properties-common python3-pip rsync git \
+    ca-certificates curl gnupg2 software-properties-common python3-pip rsync \
     && rm -rf /var/lib/apt/lists/*
 RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
     && add-apt-repository \
@@ -14,20 +14,17 @@ RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
     && apt update -y && apt-get install --no-install-recommends -y docker-ce \
     && rm -rf /var/lib/apt/lists/*
 
-# Some tools like yamllint need this
-# Pip needs this as well at the moment to install ansible
-# (and potentially other packages)
-# See: https://github.com/pypa/pip/issues/10219
-ENV LANG=C.UTF-8
-
 WORKDIR /kubespray
 COPY . .
-RUN /usr/bin/python3 -m pip install --no-cache-dir pip -U \
+RUN /usr/bin/python3 -m pip install pip -U \
-    && /usr/bin/python3 -m pip install --no-cache-dir -r tests/requirements.txt \
+    && /usr/bin/python3 -m pip install -r tests/requirements.txt \
-    && python3 -m pip install --no-cache-dir -r requirements.txt \
+    && python3 -m pip install -r requirements.txt \
     && update-alternatives --install /usr/bin/python python /usr/bin/python3 1
 
 RUN KUBE_VERSION=$(sed -n 's/^kube_version: //p' roles/kubespray-defaults/defaults/main.yaml) \
     && curl -LO https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/amd64/kubectl \
     && chmod a+x kubectl \
     && mv kubectl /usr/local/bin/kubectl
+
+# Some tools like yamllint need this
+ENV LANG=C.UTF-8

README.md

@@ -5,7 +5,7 @@
 If you have questions, check the documentation at [kubespray.io](https://kubespray.io) and join us on the [kubernetes slack](https://kubernetes.slack.com), channel **\#kubespray**.
 You can get your invite [here](http://slack.k8s.io/)
 
-- Can be deployed on **[AWS](docs/aws.md), GCE, [Azure](docs/azure.md), [OpenStack](docs/openstack.md), [vSphere](docs/vsphere.md), [Equinix Metal](docs/equinix-metal.md) (bare metal), Oracle Cloud Infrastructure (Experimental), or Baremetal**
+- Can be deployed on **[AWS](docs/aws.md), GCE, [Azure](docs/azure.md), [OpenStack](docs/openstack.md), [vSphere](docs/vsphere.md), [Packet](docs/packet.md) (bare metal), Oracle Cloud Infrastructure (Experimental), or Baremetal**
 - **Highly available** cluster
 - **Composable** (Choice of the network plugin for instance)
 - Supports most popular **Linux distributions**
@@ -32,7 +32,7 @@ CONFIG_FILE=inventory/mycluster/hosts.yaml python3 contrib/inventory_builder/inv
 
 # Review and change parameters under ``inventory/mycluster/group_vars``
 cat inventory/mycluster/group_vars/all/all.yml
-cat inventory/mycluster/group_vars/k8s_cluster/k8s-cluster.yml
+cat inventory/mycluster/group_vars/k8s_cluster/k8s_cluster.yml
 
 # Deploy Kubespray with Ansible Playbook - run the playbook as root
 # The option `--become` is required, as for example writing SSL keys in /etc/,
@@ -57,10 +57,10 @@ A simple way to ensure you get all the correct version of Ansible is to use the
 You will then need to use [bind mounts](https://docs.docker.com/storage/bind-mounts/) to get the inventory and ssh key into the container, like this:
 
 ```ShellSession
-docker pull quay.io/kubespray/kubespray:v2.16.0
+docker pull quay.io/kubespray/kubespray:v2.15.1
 docker run --rm -it --mount type=bind,source="$(pwd)"/inventory/sample,dst=/inventory \
   --mount type=bind,source="${HOME}"/.ssh/id_rsa,dst=/root/.ssh/id_rsa \
-  quay.io/kubespray/kubespray:v2.16.0 bash
+  quay.io/kubespray/kubespray:v2.15.1 bash
 # Inside the container you may now run the kubespray playbooks:
 ansible-playbook -i /inventory/inventory.ini --private-key /root/.ssh/id_rsa cluster.yml
 ```
@@ -105,7 +105,7 @@ vagrant up
 - [AWS](docs/aws.md)
 - [Azure](docs/azure.md)
 - [vSphere](docs/vsphere.md)
-- [Equinix Metal](docs/equinix-metal.md)
+- [Packet Host](docs/packet.md)
 - [Large deployments](docs/large-deployments.md)
 - [Adding/replacing a node](docs/nodes.md)
 - [Upgrades basics](docs/upgrades.md)
@@ -115,48 +115,48 @@ vagrant up
 ## Supported Linux Distributions
 
 - **Flatcar Container Linux by Kinvolk**
-- **Debian** Bullseye, Buster, Jessie, Stretch
+- **Debian** Buster, Jessie, Stretch, Wheezy
 - **Ubuntu** 16.04, 18.04, 20.04
 - **CentOS/RHEL** 7, [8](docs/centos8.md)
-- **Fedora** 33, 34
+- **Fedora** 32, 33
-- **Fedora CoreOS** (see [fcos Note](docs/fcos.md))
+- **Fedora CoreOS** (experimental: see [fcos Note](docs/fcos.md))
 - **openSUSE** Leap 15.x/Tumbleweed
 - **Oracle Linux** 7, [8](docs/centos8.md)
 - **Alma Linux** [8](docs/centos8.md)
-- **Amazon Linux 2** (experimental: see [amazon linux notes](docs/amazonlinux.md))
+- **Amazon Linux 2** (experimental: see [amazon linux notes](docs/amazonlinux.md)
 
 Note: Upstart/SysV init based OS types are not supported.
 
 ## Supported Components
 
 - Core
-  - [kubernetes](https://github.com/kubernetes/kubernetes) v1.21.6
+  - [kubernetes](https://github.com/kubernetes/kubernetes) v1.20.7
   - [etcd](https://github.com/coreos/etcd) v3.4.13
-  - [docker](https://www.docker.com/) v20.10 (see note)
+  - [docker](https://www.docker.com/) v19.03 (see note)
-  - [containerd](https://containerd.io/) v1.4.9
+  - [containerd](https://containerd.io/) v1.4.4
-  - [cri-o](http://cri-o.io/) v1.21 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS)
+  - [cri-o](http://cri-o.io/) v1.20 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
   - [cni-plugins](https://github.com/containernetworking/plugins) v0.9.1
-  - [calico](https://github.com/projectcalico/calico) v3.19.2
+  - [calico](https://github.com/projectcalico/calico) v3.17.4
   - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
-  - [cilium](https://github.com/cilium/cilium) v1.9.10
+  - [cilium](https://github.com/cilium/cilium) v1.8.9
-  - [flanneld](https://github.com/flannel-io/flannel) v0.14.0
+  - [flanneld](https://github.com/coreos/flannel) v0.13.0
-  - [kube-ovn](https://github.com/alauda/kube-ovn) v1.7.2
+  - [kube-ovn](https://github.com/alauda/kube-ovn) v1.6.2
-  - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.3.0
+  - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.2.2
-  - [multus](https://github.com/intel/multus-cni) v3.7.2
+  - [multus](https://github.com/intel/multus-cni) v3.7.0
   - [ovn4nfv](https://github.com/opnfv/ovn4nfv-k8s-plugin) v1.1.0
   - [weave](https://github.com/weaveworks/weave) v2.8.1
 - Application
   - [ambassador](https://github.com/datawire/ambassador): v1.5
   - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.0-k8s1.11
   - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.1-k8s1.11
-  - [cert-manager](https://github.com/jetstack/cert-manager) v1.0.4
+  - [cert-manager](https://github.com/jetstack/cert-manager) v0.16.1
-  - [coredns](https://github.com/coredns/coredns) v1.8.0
+  - [coredns](https://github.com/coredns/coredns) v1.7.0
-  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v1.0.0
+  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v0.43.0
 
 ## Container Runtime Notes
 
-- The list of available docker version is 18.09, 19.03 and 20.10. The recommended docker version is 20.10. The kubelet might break on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. yum versionlock plugin or apt pin).
+- The list of available docker version is 18.09, 19.03 and 20.10. The recommended docker version is 19.03. The kubelet might break on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. yum versionlock plugin or apt pin).
 - The cri-o version should be aligned with the respective kubernetes version (i.e. kube_version=1.20.x, crio_version=1.20)
 
 ## Requirements
@@ -239,6 +239,6 @@ See also [Network checker](docs/netcheck.md).
 
 [![Build graphs](https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/badges/master/pipeline.svg)](https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/pipelines)
 
-CI/end-to-end tests sponsored by: [CNCF](https://cncf.io), [Equinix Metal](https://metal.equinix.com/), [OVHcloud](https://www.ovhcloud.com/), [ELASTX](https://elastx.se/).
+CI/end-to-end tests sponsored by: [CNCF](https://cncf.io), [Packet](https://www.packet.com/), [OVHcloud](https://www.ovhcloud.com/), [ELASTX](https://elastx.se/).
 
 See the [test matrix](docs/test_cases.md) for details.

Vagrantfile

@@ -26,8 +26,8 @@ SUPPORTED_OS = {
   "centos-bento"        => {box: "bento/centos-7.6",           user: "vagrant"},
   "centos8"             => {box: "centos/8",                   user: "vagrant"},
   "centos8-bento"       => {box: "bento/centos-8",             user: "vagrant"},
+  "fedora32"            => {box: "fedora/32-cloud-base",       user: "vagrant"},
   "fedora33"            => {box: "fedora/33-cloud-base",       user: "vagrant"},
-  "fedora34"            => {box: "fedora/34-cloud-base",       user: "vagrant"},
   "opensuse"            => {box: "bento/opensuse-leap-15.2",   user: "vagrant"},
   "opensuse-tumbleweed" => {box: "opensuse/Tumbleweed.x86_64", user: "vagrant"},
   "oraclelinux"         => {box: "generic/oracle7",            user: "vagrant"},

ansible_version.yml

@@ -4,10 +4,8 @@
   become: no
   vars:
     minimal_ansible_version: 2.9.0
-    minimal_ansible_version_2_10: 2.10.11
     maximal_ansible_version: 2.11.0
     ansible_connection: local
-  tags: always
   tasks:
     - name: "Check {{ minimal_ansible_version }} <= Ansible version < {{ maximal_ansible_version }}"
       assert:
@@ -18,17 +16,6 @@
       tags:
         - check
 
-    - name: "Check Ansible version > {{ minimal_ansible_version_2_10 }} when using ansible 2.10"
-      assert:
-        msg: "When using Ansible 2.10, the minimum supported version is {{ minimal_ansible_version_2_10 }}"
-        that:
-          - ansible_version.string is version(minimal_ansible_version_2_10, ">=")
-          - ansible_version.string is version(maximal_ansible_version, "<")
-      when:
-        - ansible_version.string is version('2.10.0', ">=")
-      tags:
-        - check
-
     - name: "Check that python netaddr is installed"
       assert:
         msg: "Python netaddr is not present"

cluster.yml

@@ -86,8 +86,8 @@
   roles:
     - { role: kubespray-defaults }
     - { role: kubernetes/kubeadm, tags: kubeadm}
-    - { role: kubernetes/node-label, tags: node-label }
     - { role: network_plugin, tags: network }
+    - { role: kubernetes/node-label, tags: node-label }
 
 - hosts: calico_rr
   gather_facts: False
@@ -116,6 +116,13 @@
     - { role: kubernetes-apps/policy_controller, tags: policy-controller }
     - { role: kubernetes-apps/ingress_controller, tags: ingress-controller }
     - { role: kubernetes-apps/external_provisioner, tags: external-provisioner }
+
+- hosts: kube_control_plane
+  gather_facts: False
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  environment: "{{ proxy_disable_env }}"
+  roles:
+    - { role: kubespray-defaults }
     - { role: kubernetes-apps, tags: apps }
 
 - hosts: k8s_cluster

contrib/azurerm/roles/generate-inventory/tasks/main.yml

@@ -12,4 +12,3 @@
   template:
     src: inventory.j2
     dest: "{{ playbook_dir }}/inventory"
-    mode: 0644

contrib/azurerm/roles/generate-inventory_2/tasks/main.yml

@@ -22,10 +22,8 @@
   template:
     src: inventory.j2
     dest: "{{ playbook_dir }}/inventory"
-    mode: 0644
 
 - name: Generate Load Balancer variables
   template:
     src: loadbalancer_vars.j2
     dest: "{{ playbook_dir }}/loadbalancer_vars.yml"
-    mode: 0644

contrib/azurerm/roles/generate-templates/tasks/main.yml

@@ -8,13 +8,11 @@
     path: "{{ base_dir }}"
     state: directory
     recurse: true
-    mode: 0755
 
 - name: Store json files in base_dir
   template:
     src: "{{ item }}"
     dest: "{{ base_dir }}/{{ item }}"
-    mode: 0644
   with_items:
     - network.json
     - storage.json

contrib/dind/roles/dind-cluster/tasks/main.yaml

@@ -35,7 +35,6 @@
       path-exclude=/usr/share/doc/*
       path-include=/usr/share/doc/*/copyright
     dest: /etc/dpkg/dpkg.cfg.d/01_nodoc
-    mode: 0644
   when:
     - ansible_os_family == 'Debian'
 
@@ -64,7 +63,6 @@
   copy:
     content: "{{ distro_user }} ALL=(ALL) NOPASSWD:ALL"
     dest: "/etc/sudoers.d/{{ distro_user }}"
-    mode: 0640
 
 - name: Add my pubkey to "{{ distro_user }}" user authorized keys
   authorized_key:

contrib/inventory_builder/inventory.py

@@ -48,7 +48,7 @@ ROLES = ['all', 'kube_control_plane', 'kube_node', 'etcd', 'k8s_cluster',
          'calico_rr']
 PROTECTED_NAMES = ROLES
 AVAILABLE_COMMANDS = ['help', 'print_cfg', 'print_ips', 'print_hostnames',
-                      'load', 'add']
+                      'load']
 _boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
                    '0': False, 'no': False, 'false': False, 'off': False}
 yaml = YAML()
@@ -82,35 +82,22 @@ class KubesprayInventory(object):
     def __init__(self, changed_hosts=None, config_file=None):
         self.config_file = config_file
         self.yaml_config = {}
-        loadPreviousConfig = False
+        if self.config_file:
-        # See whether there are any commands to process
-        if changed_hosts and changed_hosts[0] in AVAILABLE_COMMANDS:
-            if changed_hosts[0] == "add":
-                loadPreviousConfig = True
-                changed_hosts = changed_hosts[1:]
-            else:
-                self.parse_command(changed_hosts[0], changed_hosts[1:])
-                sys.exit(0)
-
-        # If the user wants to remove a node, we need to load the config anyway
-        if changed_hosts and changed_hosts[0][0] == "-":
-            loadPreviousConfig = True
-
-        if self.config_file and loadPreviousConfig:  # Load previous YAML file
             try:
                 self.hosts_file = open(config_file, 'r')
-                self.yaml_config = yaml.load(self.hosts_file)
+                self.yaml_config = yaml.load_all(self.hosts_file)
-            except OSError as e:
+            except OSError:
-                # I am assuming we are catching "cannot open file" exceptions
+                pass
-                print(e)
+
-                sys.exit(1)
+        if changed_hosts and changed_hosts[0] in AVAILABLE_COMMANDS:
+            self.parse_command(changed_hosts[0], changed_hosts[1:])
+            sys.exit(0)
 
         self.ensure_required_groups(ROLES)
 
         if changed_hosts:
             changed_hosts = self.range2ips(changed_hosts)
-            self.hosts = self.build_hostnames(changed_hosts,
+            self.hosts = self.build_hostnames(changed_hosts)
-                                              loadPreviousConfig)
             self.purge_invalid_hosts(self.hosts.keys(), PROTECTED_NAMES)
             self.set_all(self.hosts)
             self.set_k8s_cluster()
@@ -171,29 +158,17 @@ class KubesprayInventory(object):
         except IndexError:
             raise ValueError("Host name must end in an integer")
 
-    # Keeps already specified hosts,
+    def build_hostnames(self, changed_hosts):
-    # and adds or removes the hosts provided as an argument
-    def build_hostnames(self, changed_hosts, loadPreviousConfig=False):
         existing_hosts = OrderedDict()
         highest_host_id = 0
-        # Load already existing hosts from the YAML
-        if loadPreviousConfig:
         try:
             for host in self.yaml_config['all']['hosts']:
-                    # Read configuration of an existing host
+                existing_hosts[host] = self.yaml_config['all']['hosts'][host]
-                    hostConfig = self.yaml_config['all']['hosts'][host]
-                    existing_hosts[host] = hostConfig
-                    # If the existing host seems
-                    # to have been created automatically, detect its ID
-                    if host.startswith(HOST_PREFIX):
                 host_id = self.get_host_id(host)
                 if host_id > highest_host_id:
                     highest_host_id = host_id
-            except Exception as e:
+        except Exception:
-                # I am assuming we are catching automatically
+            pass
-                # created hosts without IDs
-                print(e)
-                sys.exit(1)
 
         # FIXME(mattymo): Fix condition where delete then add reuses highest id
         next_host_id = highest_host_id + 1
@@ -201,7 +176,6 @@ class KubesprayInventory(object):
 
         all_hosts = existing_hosts.copy()
         for host in changed_hosts:
-            # Delete the host from config the hostname/IP has a "-" prefix
             if host[0] == "-":
                 realhost = host[1:]
                 if self.exists_hostname(all_hosts, realhost):
@@ -210,8 +184,6 @@ class KubesprayInventory(object):
                 elif self.exists_ip(all_hosts, realhost):
                     self.debug("Marked {0} for deletion.".format(realhost))
                     self.delete_host_by_ip(all_hosts, realhost)
-            # Host/Argument starts with a digit,
-            # then we assume its an IP address
             elif host[0].isdigit():
                 if ',' in host:
                     ip, access_ip = host.split(',')
@@ -231,15 +203,11 @@ class KubesprayInventory(object):
                     next_host = subprocess.check_output(cmd, shell=True)
                     next_host = next_host.strip().decode('ascii')
                 else:
-                    # Generates a hostname because we have only an IP address
                     next_host = "{0}{1}".format(HOST_PREFIX, next_host_id)
                     next_host_id += 1
-                # Uses automatically generated node name
-                # in case we dont provide it.
                 all_hosts[next_host] = {'ansible_host': access_ip,
                                         'ip': ip,
                                         'access_ip': access_ip}
-            # Host/Argument starts with a letter, then we assume its a hostname
             elif host[0].isalpha():
                 if ',' in host:
                     try:
@@ -258,7 +226,6 @@ class KubesprayInventory(object):
                                        'access_ip': access_ip}
         return all_hosts
 
-    # Expand IP ranges into individual addresses
     def range2ips(self, hosts):
         reworked_hosts = []
 
@@ -427,11 +394,9 @@ help - Display this message
 print_cfg - Write inventory file to stdout
 print_ips - Write a space-delimited list of IPs from "all" group
 print_hostnames - Write a space-delimited list of Hostnames from "all" group
-add - Adds specified hosts into an already existing inventory
 
 Advanced usage:
-Create new or overwrite old inventory file: inventory.py 10.10.1.5
+Add another host after initial creation: inventory.py 10.10.1.5
-Add another host after initial creation: inventory.py add 10.10.1.6
 Add range of hosts: inventory.py 10.10.1.3-10.10.1.5
 Add hosts with different ip and access ip: inventory.py 10.0.0.1,192.168.10.1 10.0.0.2,192.168.10.2 10.0.0.3,192.168.10.3
 Add hosts with a specific hostname, ip, and optional access ip: first,10.0.0.1,192.168.10.1 second,10.0.0.2 last,10.0.0.3
@@ -465,7 +430,6 @@ def main(argv=None):
     if not argv:
         argv = sys.argv[1:]
     KubesprayInventory(argv, CONFIG_FILE)
-    return 0
 
 
 if __name__ == "__main__":

contrib/inventory_builder/tests/test_inventory.py

@@ -67,14 +67,23 @@ class TestInventory(unittest.TestCase):
             self.assertRaisesRegex(ValueError, "Host name must end in an",
                                    self.inv.get_host_id, hostname)
 
+    def test_build_hostnames_add_one(self):
+        changed_hosts = ['10.90.0.2']
+        expected = OrderedDict([('node1',
+                                 {'ansible_host': '10.90.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '10.90.0.2'})])
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
     def test_build_hostnames_add_duplicate(self):
         changed_hosts = ['10.90.0.2']
-        expected = OrderedDict([('node3',
+        expected = OrderedDict([('node1',
                                  {'ansible_host': '10.90.0.2',
                                   'ip': '10.90.0.2',
                                   'access_ip': '10.90.0.2'})])
         self.inv.yaml_config['all']['hosts'] = expected
-        result = self.inv.build_hostnames(changed_hosts, True)
+        result = self.inv.build_hostnames(changed_hosts)
         self.assertEqual(expected, result)
 
     def test_build_hostnames_add_two(self):
@@ -90,30 +99,6 @@ class TestInventory(unittest.TestCase):
         result = self.inv.build_hostnames(changed_hosts)
         self.assertEqual(expected, result)
 
-    def test_build_hostnames_add_three(self):
-        changed_hosts = ['10.90.0.2', '10.90.0.3', '10.90.0.4']
-        expected = OrderedDict([
-            ('node1', {'ansible_host': '10.90.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '10.90.0.2'}),
-            ('node2', {'ansible_host': '10.90.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '10.90.0.3'}),
-            ('node3', {'ansible_host': '10.90.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '10.90.0.4'})])
-        result = self.inv.build_hostnames(changed_hosts)
-        self.assertEqual(expected, result)
-
-    def test_build_hostnames_add_one(self):
-        changed_hosts = ['10.90.0.2']
-        expected = OrderedDict([('node1',
-                                 {'ansible_host': '10.90.0.2',
-                                  'ip': '10.90.0.2',
-                                  'access_ip': '10.90.0.2'})])
-        result = self.inv.build_hostnames(changed_hosts)
-        self.assertEqual(expected, result)
-
     def test_build_hostnames_delete_first(self):
         changed_hosts = ['-10.90.0.2']
         existing_hosts = OrderedDict([
@@ -128,24 +113,7 @@ class TestInventory(unittest.TestCase):
             ('node2', {'ansible_host': '10.90.0.3',
                        'ip': '10.90.0.3',
                        'access_ip': '10.90.0.3'})])
-        result = self.inv.build_hostnames(changed_hosts, True)
+        result = self.inv.build_hostnames(changed_hosts)
-        self.assertEqual(expected, result)
-
-    def test_build_hostnames_delete_by_hostname(self):
-        changed_hosts = ['-node1']
-        existing_hosts = OrderedDict([
-            ('node1', {'ansible_host': '10.90.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '10.90.0.2'}),
-            ('node2', {'ansible_host': '10.90.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '10.90.0.3'})])
-        self.inv.yaml_config['all']['hosts'] = existing_hosts
-        expected = OrderedDict([
-            ('node2', {'ansible_host': '10.90.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '10.90.0.3'})])
-        result = self.inv.build_hostnames(changed_hosts, True)
         self.assertEqual(expected, result)
 
     def test_exists_hostname_positive(self):
@@ -345,7 +313,7 @@ class TestInventory(unittest.TestCase):
         self.assertRaisesRegex(Exception, "Range of ip_addresses isn't valid",
                                self.inv.range2ips, host_range)
 
-    def test_build_hostnames_create_with_one_different_ips(self):
+    def test_build_hostnames_different_ips_add_one(self):
         changed_hosts = ['10.90.0.2,192.168.0.2']
         expected = OrderedDict([('node1',
                                  {'ansible_host': '192.168.0.2',
@@ -354,7 +322,17 @@ class TestInventory(unittest.TestCase):
         result = self.inv.build_hostnames(changed_hosts)
         self.assertEqual(expected, result)
 
-    def test_build_hostnames_create_with_two_different_ips(self):
+    def test_build_hostnames_different_ips_add_duplicate(self):
+        changed_hosts = ['10.90.0.2,192.168.0.2']
+        expected = OrderedDict([('node1',
+                                 {'ansible_host': '192.168.0.2',
+                                  'ip': '10.90.0.2',
+                                  'access_ip': '192.168.0.2'})])
+        self.inv.yaml_config['all']['hosts'] = expected
+        result = self.inv.build_hostnames(changed_hosts)
+        self.assertEqual(expected, result)
+
+    def test_build_hostnames_different_ips_add_two(self):
         changed_hosts = ['10.90.0.2,192.168.0.2', '10.90.0.3,192.168.0.3']
         expected = OrderedDict([
             ('node1', {'ansible_host': '192.168.0.2',
@@ -363,210 +341,6 @@ class TestInventory(unittest.TestCase):
             ('node2', {'ansible_host': '192.168.0.3',
                        'ip': '10.90.0.3',
                        'access_ip': '192.168.0.3'})])
+        self.inv.yaml_config['all']['hosts'] = OrderedDict()
         result = self.inv.build_hostnames(changed_hosts)
         self.assertEqual(expected, result)
-
-    def test_build_hostnames_create_with_three_different_ips(self):
-        changed_hosts = ['10.90.0.2,192.168.0.2',
-                         '10.90.0.3,192.168.0.3',
-                         '10.90.0.4,192.168.0.4']
-        expected = OrderedDict([
-            ('node1', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'}),
-            ('node2', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'}),
-            ('node3', {'ansible_host': '192.168.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '192.168.0.4'})])
-        result = self.inv.build_hostnames(changed_hosts)
-        self.assertEqual(expected, result)
-
-    def test_build_hostnames_overwrite_one_with_different_ips(self):
-        changed_hosts = ['10.90.0.2,192.168.0.2']
-        expected = OrderedDict([('node1',
-                                 {'ansible_host': '192.168.0.2',
-                                  'ip': '10.90.0.2',
-                                  'access_ip': '192.168.0.2'})])
-        existing = OrderedDict([('node5',
-                                 {'ansible_host': '192.168.0.5',
-                                  'ip': '10.90.0.5',
-                                  'access_ip': '192.168.0.5'})])
-        self.inv.yaml_config['all']['hosts'] = existing
-        result = self.inv.build_hostnames(changed_hosts)
-        self.assertEqual(expected, result)
-
-    def test_build_hostnames_overwrite_three_with_different_ips(self):
-        changed_hosts = ['10.90.0.2,192.168.0.2']
-        expected = OrderedDict([('node1',
-                                 {'ansible_host': '192.168.0.2',
-                                  'ip': '10.90.0.2',
-                                  'access_ip': '192.168.0.2'})])
-        existing = OrderedDict([
-            ('node3', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'}),
-            ('node4', {'ansible_host': '192.168.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '192.168.0.4'}),
-            ('node5', {'ansible_host': '192.168.0.5',
-                       'ip': '10.90.0.5',
-                       'access_ip': '192.168.0.5'})])
-        self.inv.yaml_config['all']['hosts'] = existing
-        result = self.inv.build_hostnames(changed_hosts)
-        self.assertEqual(expected, result)
-
-    def test_build_hostnames_different_ips_add_duplicate(self):
-        changed_hosts = ['10.90.0.2,192.168.0.2']
-        expected = OrderedDict([('node3',
-                                 {'ansible_host': '192.168.0.2',
-                                  'ip': '10.90.0.2',
-                                  'access_ip': '192.168.0.2'})])
-        existing = expected
-        self.inv.yaml_config['all']['hosts'] = existing
-        result = self.inv.build_hostnames(changed_hosts, True)
-        self.assertEqual(expected, result)
-
-    def test_build_hostnames_add_two_different_ips_into_one_existing(self):
-        changed_hosts = ['10.90.0.3,192.168.0.3', '10.90.0.4,192.168.0.4']
-        expected = OrderedDict([
-            ('node2', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'}),
-            ('node3', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'}),
-            ('node4', {'ansible_host': '192.168.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '192.168.0.4'})])
-
-        existing = OrderedDict([
-            ('node2', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'})])
-        self.inv.yaml_config['all']['hosts'] = existing
-        result = self.inv.build_hostnames(changed_hosts, True)
-        self.assertEqual(expected, result)
-
-    def test_build_hostnames_add_two_different_ips_into_two_existing(self):
-        changed_hosts = ['10.90.0.4,192.168.0.4', '10.90.0.5,192.168.0.5']
-        expected = OrderedDict([
-            ('node2', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'}),
-            ('node3', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'}),
-            ('node4', {'ansible_host': '192.168.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '192.168.0.4'}),
-            ('node5', {'ansible_host': '192.168.0.5',
-                       'ip': '10.90.0.5',
-                       'access_ip': '192.168.0.5'})])
-
-        existing = OrderedDict([
-            ('node2', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'}),
-            ('node3', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'})])
-        self.inv.yaml_config['all']['hosts'] = existing
-        result = self.inv.build_hostnames(changed_hosts, True)
-        self.assertEqual(expected, result)
-
-    def test_build_hostnames_add_two_different_ips_into_three_existing(self):
-        changed_hosts = ['10.90.0.5,192.168.0.5', '10.90.0.6,192.168.0.6']
-        expected = OrderedDict([
-            ('node2', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'}),
-            ('node3', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'}),
-            ('node4', {'ansible_host': '192.168.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '192.168.0.4'}),
-            ('node5', {'ansible_host': '192.168.0.5',
-                       'ip': '10.90.0.5',
-                       'access_ip': '192.168.0.5'}),
-            ('node6', {'ansible_host': '192.168.0.6',
-                       'ip': '10.90.0.6',
-                       'access_ip': '192.168.0.6'})])
-
-        existing = OrderedDict([
-            ('node2', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'}),
-            ('node3', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'}),
-            ('node4', {'ansible_host': '192.168.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '192.168.0.4'})])
-        self.inv.yaml_config['all']['hosts'] = existing
-        result = self.inv.build_hostnames(changed_hosts, True)
-        self.assertEqual(expected, result)
-
-    # Add two IP addresses into a config that has
-    # three already defined IP addresses. One of the IP addresses
-    # is a duplicate.
-    def test_build_hostnames_add_two_duplicate_one_overlap(self):
-        changed_hosts = ['10.90.0.4,192.168.0.4', '10.90.0.5,192.168.0.5']
-        expected = OrderedDict([
-            ('node2', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'}),
-            ('node3', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'}),
-            ('node4', {'ansible_host': '192.168.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '192.168.0.4'}),
-            ('node5', {'ansible_host': '192.168.0.5',
-                       'ip': '10.90.0.5',
-                       'access_ip': '192.168.0.5'})])
-
-        existing = OrderedDict([
-            ('node2', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'}),
-            ('node3', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'}),
-            ('node4', {'ansible_host': '192.168.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '192.168.0.4'})])
-        self.inv.yaml_config['all']['hosts'] = existing
-        result = self.inv.build_hostnames(changed_hosts, True)
-        self.assertEqual(expected, result)
-
-    # Add two duplicate IP addresses into a config that has
-    # three already defined IP addresses
-    def test_build_hostnames_add_two_duplicate_two_overlap(self):
-        changed_hosts = ['10.90.0.3,192.168.0.3', '10.90.0.4,192.168.0.4']
-        expected = OrderedDict([
-            ('node2', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'}),
-            ('node3', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'}),
-            ('node4', {'ansible_host': '192.168.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '192.168.0.4'})])
-
-        existing = OrderedDict([
-            ('node2', {'ansible_host': '192.168.0.2',
-                       'ip': '10.90.0.2',
-                       'access_ip': '192.168.0.2'}),
-            ('node3', {'ansible_host': '192.168.0.3',
-                       'ip': '10.90.0.3',
-                       'access_ip': '192.168.0.3'}),
-            ('node4', {'ansible_host': '192.168.0.4',
-                       'ip': '10.90.0.4',
-                       'access_ip': '192.168.0.4'})])
-        self.inv.yaml_config['all']['hosts'] = existing
-        result = self.inv.build_hostnames(changed_hosts, True)
-        self.assertEqual(expected, result)

contrib/kvm-setup/roles/kvm-setup/tasks/main.yml

@@ -1,7 +1,7 @@
 ---
 
 - name: Install required packages
-  package:
+  yum:
     name: "{{ item }}"
     state: present
   with_items:

contrib/kvm-setup/roles/kvm-setup/tasks/user.yml

@@ -11,7 +11,6 @@
     state: directory
     owner: "{{ k8s_deployment_user }}"
     group: "{{ k8s_deployment_user }}"
-    mode: 0700
 
 - name: Configure sudo for deployment user
   copy:

contrib/network-storage/glusterfs/roles/glusterfs/client/tasks/setup-RedHat.yml

@@ -1,10 +1,10 @@
 ---
 - name: Install Prerequisites
-  package: name={{ item }}  state=present
+  yum: name={{ item }}  state=present
   with_items:
     - "centos-release-gluster{{ glusterfs_default_release }}"
 
 - name: Install Packages
-  package: name={{ item }}  state=present
+  yum: name={{ item }}  state=present
   with_items:
     - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/main.yml

@@ -9,7 +9,7 @@
   when: ansible_os_family == "Debian"
 
 - name: install xfs RedHat
-  package: name=xfsprogs state=present
+  yum: name=xfsprogs state=present
   when: ansible_os_family == "RedHat"
 
 # Format external volumes in xfs
@@ -82,7 +82,6 @@
   template:
     dest: "{{ gluster_mount_dir }}/.test-file.txt"
     src: test-file.txt
-    mode: 0644
   when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
 
 - name: Unmount glusterfs

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/setup-RedHat.yml

@@ -1,11 +1,11 @@
 ---
 - name: Install Prerequisites
-  package: name={{ item }}  state=present
+  yum: name={{ item }}  state=present
   with_items:
     - "centos-release-gluster{{ glusterfs_default_release }}"
 
 - name: Install Packages
-  package: name={{ item }}  state=present
+  yum: name={{ item }}  state=present
   with_items:
     - glusterfs-server
     - glusterfs-client

contrib/network-storage/glusterfs/roles/glusterfs/server/tests/test.yml

@@ -0,0 +1,5 @@
+---
+- hosts: all
+
+  roles:
+    - role_under_test

contrib/network-storage/heketi/roles/prepare/tasks/main.yml

@@ -11,7 +11,7 @@
 
 - name: "Install glusterfs mount utils (RedHat)"
   become: true
-  package:
+  yum:
     name: "glusterfs-fuse"
     state: "present"
   when: "ansible_os_family == 'RedHat'"

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml

@@ -1,10 +1,7 @@
 ---
 - name: "Kubernetes Apps | Lay Down Heketi Bootstrap"
   become: true
-  template:
+  template: { src: "heketi-bootstrap.json.j2", dest: "{{ kube_config_dir }}/heketi-bootstrap.json" }
-    src: "heketi-bootstrap.json.j2"
-    dest: "{{ kube_config_dir }}/heketi-bootstrap.json"
-    mode: 0640
   register: "rendering"
 - name: "Kubernetes Apps | Install and configure Heketi Bootstrap"
   kube:

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml

@@ -10,7 +10,6 @@
   template:
     src: "topology.json.j2"
     dest: "{{ kube_config_dir }}/topology.json"
-    mode: 0644
 - name: "Copy topology configuration into container."
   changed_when: false
   command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ initial_heketi_pod_name }}:/tmp/topology.json"

contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml

@@ -1,9 +1,6 @@
 ---
 - name: "Kubernetes Apps | Lay Down GlusterFS Daemonset"
-  template:
+  template: { src: "glusterfs-daemonset.json.j2", dest: "{{ kube_config_dir }}/glusterfs-daemonset.json" }
-    src: "glusterfs-daemonset.json.j2"
-    dest: "{{ kube_config_dir }}/glusterfs-daemonset.json"
-    mode: 0644
   become: true
   register: "rendering"
 - name: "Kubernetes Apps | Install and configure GlusterFS daemonset"
@@ -30,10 +27,7 @@
   delay: 5
 
 - name: "Kubernetes Apps | Lay Down Heketi Service Account"
-  template:
+  template: { src: "heketi-service-account.json.j2", dest: "{{ kube_config_dir }}/heketi-service-account.json" }
-    src: "heketi-service-account.json.j2"
-    dest: "{{ kube_config_dir }}/heketi-service-account.json"
-    mode: 0644
   become: true
   register: "rendering"
 - name: "Kubernetes Apps | Install and configure Heketi Service Account"

contrib/network-storage/heketi/roles/provision/tasks/heketi.yml

@@ -4,7 +4,6 @@
   template:
     src: "heketi-deployment.json.j2"
     dest: "{{ kube_config_dir }}/heketi-deployment.json"
-    mode: 0644
   register: "rendering"
 
 - name: "Kubernetes Apps | Install and configure Heketi"

contrib/network-storage/heketi/roles/provision/tasks/secret.yml

@@ -5,7 +5,7 @@
   changed_when: false
 
 - name: "Kubernetes Apps | Deploy cluster role binding."
-  when: "clusterrolebinding_state.stdout | length > 0"
+  when: "clusterrolebinding_state.stdout == \"\""
   command: "{{ bin_dir }}/kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account"
 
 - name: Get clusterrolebindings again
@@ -15,7 +15,7 @@
 
 - name: Make sure that clusterrolebindings are present now
   assert:
-    that: "clusterrolebinding_state.stdout | length > 0"
+    that: "clusterrolebinding_state.stdout != \"\""
     msg: "Cluster role binding is not present."
 
 - name: Get the heketi-config-secret secret
@@ -28,10 +28,9 @@
   template:
     src: "heketi.json.j2"
     dest: "{{ kube_config_dir }}/heketi.json"
-    mode: 0644
 
 - name: "Deploy Heketi config secret"
-  when: "secret_state.stdout | length > 0"
+  when: "secret_state.stdout == \"\""
   command: "{{ bin_dir }}/kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json"
 
 - name: Get the heketi-config-secret secret again

contrib/network-storage/heketi/roles/provision/tasks/storage.yml

@@ -2,10 +2,7 @@
 - name: "Kubernetes Apps | Lay Down Heketi Storage"
   become: true
   vars: { nodes: "{{ groups['heketi-node'] }}" }
-  template:
+  template: { src: "heketi-storage.json.j2", dest: "{{ kube_config_dir }}/heketi-storage.json" }
-    src: "heketi-storage.json.j2"
-    dest: "{{ kube_config_dir }}/heketi-storage.json"
-    mode: 0644
   register: "rendering"
 - name: "Kubernetes Apps | Install and configure Heketi Storage"
   kube:

contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml

@@ -16,7 +16,6 @@
   template:
     src: "storageclass.yml.j2"
     dest: "{{ kube_config_dir }}/storageclass.yml"
-    mode: 0644
   register: "rendering"
 - name: "Kubernetes Apps | Install and configure Storace Class"
   kube:

contrib/network-storage/heketi/roles/provision/tasks/topology.yml

@@ -10,7 +10,6 @@
   template:
     src: "topology.json.j2"
     dest: "{{ kube_config_dir }}/topology.json"
-    mode: 0644
 - name: "Copy topology configuration into container."  # noqa 503
   when: "rendering.changed"
   command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ heketi_pod_name }}:/tmp/topology.json"

contrib/network-storage/heketi/roles/tear-down-disks/tasks/main.yml

@@ -1,7 +1,7 @@
 ---
 - name: "Install lvm utils (RedHat)"
   become: true
-  package:
+  yum:
     name: "lvm2"
     state: "present"
   when: "ansible_os_family == 'RedHat'"
@@ -19,7 +19,7 @@
   become: true
   shell: "pvs {{ disk_volume_device_1 }} --option vg_name | tail -n+2"
   register: "volume_groups"
-  ignore_errors: true   # noqa ignore-errors
+  ignore_errors: true
   changed_when: false
 
 - name: "Remove volume groups."  # noqa 301
@@ -35,11 +35,11 @@
     PATH: "{{ ansible_env.PATH }}:/sbin"  # Make sure we can workaround RH / CentOS conservative path management
   become: true
   command: "pvremove {{ disk_volume_device_1 }} --yes"
-  ignore_errors: true   # noqa ignore-errors
+  ignore_errors: true
 
 - name: "Remove lvm utils (RedHat)"
   become: true
-  package:
+  yum:
     name: "lvm2"
     state: "absent"
   when: "ansible_os_family == 'RedHat' and heketi_remove_lvm"

contrib/network-storage/heketi/roles/tear-down/tasks/main.yml

@@ -1,51 +1,51 @@
 ---
-- name: Remove storage class.  # noqa 301
+- name: "Remove storage class."  # noqa 301
   command: "{{ bin_dir }}/kubectl delete storageclass gluster"
-  ignore_errors: true  # noqa ignore-errors
+  ignore_errors: true
-- name: Tear down heketi.  # noqa 301
+- name: "Tear down heketi."  # noqa 301
   command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\""
-  ignore_errors: true  # noqa ignore-errors
+  ignore_errors: true
-- name: Tear down heketi.  # noqa 301
+- name: "Tear down heketi."  # noqa 301
   command: "{{ bin_dir }}/kubectl delete all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\""
-  ignore_errors: true  # noqa ignore-errors
+  ignore_errors: true
-- name: Tear down bootstrap.
+- name: "Tear down bootstrap."
   include_tasks: "../../provision/tasks/bootstrap/tear-down.yml"
-- name: Ensure there is nothing left over.  # noqa 301
+- name: "Ensure there is nothing left over."  # noqa 301
   command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-pod\" -o=json"
   register: "heketi_result"
   until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0"
   retries: 60
   delay: 5
-- name: Ensure there is nothing left over.  # noqa 301
+- name: "Ensure there is nothing left over."  # noqa 301
   command: "{{ bin_dir }}/kubectl get all,service,jobs,deployment,secret --selector=\"glusterfs=heketi-deployment\" -o=json"
   register: "heketi_result"
   until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0"
   retries: 60
   delay: 5
-- name: Tear down glusterfs.  # noqa 301
+- name: "Tear down glusterfs."  # noqa 301
   command: "{{ bin_dir }}/kubectl delete daemonset.extensions/glusterfs"
-  ignore_errors: true  # noqa ignore-errors
+  ignore_errors: true
-- name: Remove heketi storage service.  # noqa 301
+- name: "Remove heketi storage service."  # noqa 301
   command: "{{ bin_dir }}/kubectl delete service heketi-storage-endpoints"
-  ignore_errors: true  # noqa ignore-errors
+  ignore_errors: true
-- name: Remove heketi gluster role binding  # noqa 301
+- name: "Remove heketi gluster role binding"  # noqa 301
   command: "{{ bin_dir }}/kubectl delete clusterrolebinding heketi-gluster-admin"
-  ignore_errors: true  # noqa ignore-errors
+  ignore_errors: true
-- name: Remove heketi config secret  # noqa 301
+- name: "Remove heketi config secret"  # noqa 301
   command: "{{ bin_dir }}/kubectl delete secret heketi-config-secret"
-  ignore_errors: true  # noqa ignore-errors
+  ignore_errors: true
-- name: Remove heketi db backup  # noqa 301
+- name: "Remove heketi db backup"  # noqa 301
   command: "{{ bin_dir }}/kubectl delete secret heketi-db-backup"
-  ignore_errors: true  # noqa ignore-errors
+  ignore_errors: true
-- name: Remove heketi service account  # noqa 301
+- name: "Remove heketi service account"  # noqa 301
   command: "{{ bin_dir }}/kubectl delete serviceaccount heketi-service-account"
-  ignore_errors: true  # noqa ignore-errors
+  ignore_errors: true
-- name: Get secrets
+- name: "Get secrets"
   command: "{{ bin_dir }}/kubectl get secrets --output=\"json\""
   register: "secrets"
   changed_when: false
-- name: Remove heketi storage secret
+- name: "Remove heketi storage secret"
   vars: { storage_query: "items[?metadata.annotations.\"kubernetes.io/service-account.name\"=='heketi-service-account'].metadata.name|[0]" }
   command: "{{ bin_dir }}/kubectl delete secret {{ secrets.stdout|from_json|json_query(storage_query) }}"
   when: "storage_query is defined"
-  ignore_errors: true  # noqa ignore-errors
+  ignore_errors: true

contrib/packaging/rpm/kubespray.spec

@@ -9,8 +9,8 @@ Summary:        Ansible modules for installing Kubernetes
 
 Group:          System Environment/Libraries
 License:        ASL 2.0
-Url:            https://github.com/kubernetes-sigs/kubespray
+Url:            https://github.com/kubernetes-incubator/kubespray
-Source0:        https://github.com/kubernetes-sigs/kubespray/archive/%{upstream_version}.tar.gz#/%{name}-%{release}.tar.gz
+Source0:        https://github.com/kubernetes-incubator/kubespray/archive/%{upstream_version}.tar.gz#/%{name}-%{release}.tar.gz
 
 BuildArch:      noarch
 BuildRequires:  git

contrib/terraform/aws/.gitignore

@@ -1,3 +1,2 @@
 *.tfstate*
-.terraform.lock.hcl
 .terraform

contrib/terraform/aws/create-infrastructure.tf

@@ -20,7 +20,7 @@ module "aws-vpc" {
 
   aws_cluster_name         = var.aws_cluster_name
   aws_vpc_cidr_block       = var.aws_vpc_cidr_block
-  aws_avail_zones          = slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names))
+  aws_avail_zones          = slice(data.aws_availability_zones.available.names, 0, 2)
   aws_cidr_subnets_private = var.aws_cidr_subnets_private
   aws_cidr_subnets_public  = var.aws_cidr_subnets_public
   default_tags             = var.default_tags
@@ -31,7 +31,7 @@ module "aws-elb" {
 
   aws_cluster_name      = var.aws_cluster_name
   aws_vpc_id            = module.aws-vpc.aws_vpc_id
-  aws_avail_zones       = slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names))
+  aws_avail_zones       = slice(data.aws_availability_zones.available.names, 0, 2)
   aws_subnet_ids_public = module.aws-vpc.aws_subnet_ids_public
   aws_elb_api_port      = var.aws_elb_api_port
   k8s_secure_api_port   = var.k8s_secure_api_port
@@ -52,9 +52,9 @@ module "aws-iam" {
 resource "aws_instance" "bastion-server" {
   ami                         = data.aws_ami.distro.id
   instance_type               = var.aws_bastion_size
-  count                       = var.aws_bastion_num
+  count                       = length(var.aws_cidr_subnets_public)
   associate_public_ip_address = true
-  availability_zone           = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
+  availability_zone           = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
   subnet_id                   = element(module.aws-vpc.aws_subnet_ids_public, count.index)
 
   vpc_security_group_ids = module.aws-vpc.aws_security_group
@@ -79,15 +79,11 @@ resource "aws_instance" "k8s-master" {
 
   count = var.aws_kube_master_num
 
-  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
+  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
   subnet_id         = element(module.aws-vpc.aws_subnet_ids_private, count.index)
 
   vpc_security_group_ids = module.aws-vpc.aws_security_group
 
-  root_block_device {
-    volume_size = var.aws_kube_master_disk_size
-  }
-
   iam_instance_profile = module.aws-iam.kube_control_plane-profile
   key_name             = var.AWS_SSH_KEY_NAME
 
@@ -110,15 +106,11 @@ resource "aws_instance" "k8s-etcd" {
 
   count = var.aws_etcd_num
 
-  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
+  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
   subnet_id         = element(module.aws-vpc.aws_subnet_ids_private, count.index)
 
   vpc_security_group_ids = module.aws-vpc.aws_security_group
 
-  root_block_device {
-    volume_size = var.aws_etcd_disk_size
-  }
-
   key_name = var.AWS_SSH_KEY_NAME
 
   tags = merge(var.default_tags, tomap({
@@ -134,15 +126,11 @@ resource "aws_instance" "k8s-worker" {
 
   count = var.aws_kube_worker_num
 
-  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
+  availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
   subnet_id         = element(module.aws-vpc.aws_subnet_ids_private, count.index)
 
   vpc_security_group_ids = module.aws-vpc.aws_security_group
 
-  root_block_device {
-    volume_size = var.aws_kube_worker_disk_size
-  }
-
   iam_instance_profile = module.aws-iam.kube-worker-profile
   key_name             = var.AWS_SSH_KEY_NAME
 
@@ -164,10 +152,10 @@ data "template_file" "inventory" {
     public_ip_address_bastion = join("\n", formatlist("bastion ansible_host=%s", aws_instance.bastion-server.*.public_ip))
     connection_strings_master = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-master.*.private_dns, aws_instance.k8s-master.*.private_ip))
     connection_strings_node   = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-worker.*.private_dns, aws_instance.k8s-worker.*.private_ip))
+    connection_strings_etcd   = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-etcd.*.private_dns, aws_instance.k8s-etcd.*.private_ip))
     list_master               = join("\n", aws_instance.k8s-master.*.private_dns)
     list_node                 = join("\n", aws_instance.k8s-worker.*.private_dns)
-    connection_strings_etcd   = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-etcd.*.private_dns, aws_instance.k8s-etcd.*.private_ip))
+    list_etcd                 = join("\n", aws_instance.k8s-etcd.*.private_dns)
-    list_etcd                 = join("\n", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_dns) : (aws_instance.k8s-master.*.private_dns)))
     elb_api_fqdn              = "apiserver_loadbalancer_domain_name=\"${module.aws-elb.aws_elb_api_fqdn}\""
   }
 }

contrib/terraform/aws/output.tf

@@ -11,7 +11,7 @@ output "workers" {
 }
 
 output "etcd" {
-  value = join("\n", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_ip) : (aws_instance.k8s-master.*.private_ip)))
+  value = join("\n", aws_instance.k8s-etcd.*.private_ip)
 }
 
 output "aws_elb_api_fqdn" {

contrib/terraform/aws/sample-inventory/cluster.tfvars

@@ -9,8 +9,6 @@ aws_cidr_subnets_private = ["10.250.192.0/20", "10.250.208.0/20"]
 aws_cidr_subnets_public = ["10.250.224.0/20", "10.250.240.0/20"]
 
 #Bastion Host
-aws_bastion_num = 1
-
 aws_bastion_size = "t2.medium"
 
 #Kubernetes Cluster
@@ -19,26 +17,22 @@ aws_kube_master_num = 3
 
 aws_kube_master_size = "t2.medium"
 
-aws_kube_master_disk_size = 50
-
 aws_etcd_num = 3
 
 aws_etcd_size = "t2.medium"
 
-aws_etcd_disk_size = 50
-
 aws_kube_worker_num = 4
 
 aws_kube_worker_size = "t2.medium"
 
-aws_kube_worker_disk_size = 50
-
 #Settings AWS ELB
 
 aws_elb_api_port = 6443
 
 k8s_secure_api_port = 6443
 
+kube_insecure_apiserver_address = "0.0.0.0"
+
 default_tags = {
   #  Env = "devtest"  #  Product = "kubernetes"
 }

contrib/terraform/aws/templates/inventory.tpl

@@ -10,18 +10,19 @@ ${public_ip_address_bastion}
 [kube_control_plane]
 ${list_master}
 
+
 [kube_node]
 ${list_node}
 
+
 [etcd]
 ${list_etcd}
 
-[calico_rr]
 
 [k8s_cluster:children]
 kube_node
 kube_control_plane
-calico_rr
+
 
 [k8s_cluster:vars]
 ${elb_api_fqdn}

contrib/terraform/aws/terraform.tfvars

@@ -6,34 +6,26 @@ aws_vpc_cidr_block       = "10.250.192.0/18"
 aws_cidr_subnets_private = ["10.250.192.0/20", "10.250.208.0/20"]
 aws_cidr_subnets_public  = ["10.250.224.0/20", "10.250.240.0/20"]
 
-# single AZ deployment
-#aws_cidr_subnets_private = ["10.250.192.0/20"]
-#aws_cidr_subnets_public  = ["10.250.224.0/20"]
-
-# 3+ AZ deployment
-#aws_cidr_subnets_private = ["10.250.192.0/24","10.250.193.0/24","10.250.194.0/24","10.250.195.0/24"]
-#aws_cidr_subnets_public  = ["10.250.224.0/24","10.250.225.0/24","10.250.226.0/24","10.250.227.0/24"]
-
 #Bastion Host
-aws_bastion_num  = 1
+aws_bastion_size = "t2.medium"
-aws_bastion_size = "t3.small"
+
 
 #Kubernetes Cluster
-aws_kube_master_num       = 3
-aws_kube_master_size      = "t3.medium"
-aws_kube_master_disk_size = 50
 
-aws_etcd_num       = 0
+aws_kube_master_num  = 3
-aws_etcd_size      = "t3.medium"
+aws_kube_master_size = "t2.medium"
-aws_etcd_disk_size = 50
+
+aws_etcd_num  = 3
+aws_etcd_size = "t2.medium"
 
 aws_kube_worker_num  = 4
-aws_kube_worker_size      = "t3.medium"
+aws_kube_worker_size = "t2.medium"
-aws_kube_worker_disk_size = 50
 
 #Settings AWS ELB
+
 aws_elb_api_port                = 6443
 k8s_secure_api_port             = 6443
+kube_insecure_apiserver_address = "0.0.0.0"
 
 default_tags = {
   #  Env = "devtest"

contrib/terraform/aws/terraform.tfvars.example

@@ -8,26 +8,25 @@ aws_cidr_subnets_public = ["10.250.224.0/20","10.250.240.0/20"]
 aws_avail_zones = ["eu-central-1a","eu-central-1b"]
 
 #Bastion Host
-aws_bastion_num = 1
+aws_bastion_ami = "ami-5900cc36"
-aws_bastion_size = "t3.small"
+aws_bastion_size = "t2.small"
+
 
 #Kubernetes Cluster
+
 aws_kube_master_num = 3
-aws_kube_master_size = "t3.medium"
+aws_kube_master_size = "t2.medium"
-aws_kube_master_disk_size = 50
 
 aws_etcd_num = 3
-aws_etcd_size = "t3.medium"
+aws_etcd_size = "t2.medium"
-aws_etcd_disk_size = 50
 
 aws_kube_worker_num = 4
-aws_kube_worker_size = "t3.medium"
+aws_kube_worker_size = "t2.medium"
-aws_kube_worker_disk_size = 50
+
+aws_cluster_ami = "ami-903df7ff"
 
 #Settings AWS ELB
+
 aws_elb_api_port = 6443
 k8s_secure_api_port = 6443
-
+kube_insecure_apiserver_address = 0.0.0.0
-default_tags = { }
-
-inventory_file = "../../../inventory/hosts"

contrib/terraform/aws/variables.tf

@@ -25,7 +25,7 @@ data "aws_ami" "distro" {
 
   filter {
     name   = "name"
-    values = ["debian-10-amd64-*"]
+    values = ["ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*"]
   }
 
   filter {
@@ -33,7 +33,7 @@ data "aws_ami" "distro" {
     values = ["hvm"]
   }
 
-  owners = ["136693071363"] # Debian-10
+  owners = ["099720109477"] # Canonical
 }
 
 //AWS VPC Variables
@@ -63,18 +63,10 @@ variable "aws_bastion_size" {
 * The number should be divisable by the number of used
 * AWS Availability Zones without an remainder.
 */
-variable "aws_bastion_num" {
-  description = "Number of Bastion Nodes"
-}
-
 variable "aws_kube_master_num" {
   description = "Number of Kubernetes Master Nodes"
 }
 
-variable "aws_kube_master_disk_size" {
-  description = "Disk size for Kubernetes Master Nodes (in GiB)"
-}
-
 variable "aws_kube_master_size" {
   description = "Instance size of Kube Master Nodes"
 }
@@ -83,10 +75,6 @@ variable "aws_etcd_num" {
   description = "Number of etcd Nodes"
 }
 
-variable "aws_etcd_disk_size" {
-  description = "Disk size for etcd Nodes (in GiB)"
-}
-
 variable "aws_etcd_size" {
   description = "Instance size of etcd Nodes"
 }
@@ -95,10 +83,6 @@ variable "aws_kube_worker_num" {
   description = "Number of Kubernetes Worker Nodes"
 }
 
-variable "aws_kube_worker_disk_size" {
-  description = "Disk size for Kubernetes Worker Nodes (in GiB)"
-}
-
 variable "aws_kube_worker_size" {
   description = "Instance size of Kubernetes Worker Nodes"
 }

contrib/terraform/openstack/README.md

@@ -273,7 +273,6 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
 |`wait_for_floatingip` | Let Terraform poll the instance until the floating IP has been associated, `false` by default. |
 |`node_root_volume_size_in_gb` | Size of the root volume for nodes, 0 to use ephemeral storage |
 |`master_root_volume_size_in_gb` | Size of the root volume for masters, 0 to use ephemeral storage |
-|`master_volume_type` | Volume type of the root volume for control_plane, 'Default' by default |
 |`gfs_root_volume_size_in_gb` | Size of the root volume for gluster, 0 to use ephemeral storage |
 |`etcd_root_volume_size_in_gb` | Size of the root volume for etcd nodes, 0 to use ephemeral storage |
 |`bastion_root_volume_size_in_gb` | Size of the root volume for bastions, 0 to use ephemeral storage |

contrib/terraform/openstack/kubespray.tf

@@ -83,7 +83,6 @@ module "compute" {
   use_server_groups                            = var.use_server_groups
   extra_sec_groups                             = var.extra_sec_groups
   extra_sec_groups_name                        = var.extra_sec_groups_name
-  group_vars_path                              = var.group_vars_path
 
   network_id = module.network.router_id
 }

contrib/terraform/openstack/modules/compute/main.tf

@@ -204,7 +204,7 @@ resource "openstack_compute_instance_v2" "bastion" {
   }
 
   provisioner "local-exec" {
-    command = "sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${var.bastion_fips[0]}/ > ${var.group_vars_path}/no_floating.yml"
+    command = "sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${var.bastion_fips[0]}/ > group_vars/no_floating.yml"
   }
 }
 
@@ -251,7 +251,7 @@ resource "openstack_compute_instance_v2" "k8s_master" {
   }
 
   provisioner "local-exec" {
-    command = "sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_master_fips), 0)}/ > ${var.group_vars_path}/no_floating.yml"
+    command = "sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_master_fips), 0)}/ > group_vars/no_floating.yml"
   }
 }
 
@@ -298,7 +298,7 @@ resource "openstack_compute_instance_v2" "k8s_master_no_etcd" {
   }
 
   provisioner "local-exec" {
-    command = "sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_master_fips), 0)}/ > ${var.group_vars_path}/no_floating.yml"
+    command = "sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_master_fips), 0)}/ > group_vars/no_floating.yml"
   }
 }
 
@@ -468,7 +468,7 @@ resource "openstack_compute_instance_v2" "k8s_node" {
   }
 
   provisioner "local-exec" {
-    command = "sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_node_fips), 0)}/ > ${var.group_vars_path}/no_floating.yml"
+    command = "sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, var.k8s_node_fips), 0)}/ > group_vars/no_floating.yml"
   }
 }
 
@@ -554,7 +554,7 @@ resource "openstack_compute_instance_v2" "k8s_nodes" {
   }
 
   provisioner "local-exec" {
-    command = "%{if each.value.floating_ip}sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, [for key, value in var.k8s_nodes_fips : value.address]), 0)}/ > ${var.group_vars_path}/no_floating.yml%{else}true%{endif}"
+    command = "%{if each.value.floating_ip}sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, [for key, value in var.k8s_nodes_fips : value.address]), 0)}/ > group_vars/no_floating.yml%{else}true%{endif}"
   }
 }
 

contrib/terraform/openstack/modules/compute/variables.tf

@@ -151,7 +151,3 @@ variable "image_master" {
 variable "image_master_uuid" {
   type = string
 }
-
-variable "group_vars_path" {
-  type = string
-}

contrib/terraform/openstack/variables.tf

@@ -278,9 +278,3 @@ variable "image_master_uuid" {
   description = "uuid of image to be used on master nodes. If empty defaults to image_uuid"
   default     = ""
 }
-
-variable "group_vars_path" {
-  description = "path to the inventory group vars directory"
-  type        = string
-  default     = "./group_vars"
-}

contrib/terraform/packet/README.md

@@ -1,16 +1,16 @@
-# Kubernetes on Equinix Metal with Terraform
+# Kubernetes on Packet with Terraform
 
 Provision a Kubernetes cluster with [Terraform](https://www.terraform.io) on
-[Equinix Metal](https://metal.equinix.com) ([formerly Packet](https://blog.equinix.com/blog/2020/10/06/equinix-metal-metal-and-more/)).
+[Packet](https://www.packet.com).
 
 ## Status
 
-This will install a Kubernetes cluster on Equinix Metal. It should work in all locations and on most server types.
+This will install a Kubernetes cluster on Packet bare metal. It should work in all locations and on most server types.
 
 ## Approach
 
 The terraform configuration inspects variables found in
-[variables.tf](variables.tf) to create resources in your Equinix Metal project.
+[variables.tf](variables.tf) to create resources in your Packet project.
 There is a [python script](../terraform.py) that reads the generated`.tfstate`
 file to generate a dynamic inventory that is consumed by [cluster.yml](../../..//cluster.yml)
 to actually install Kubernetes with Kubespray.
@@ -36,12 +36,12 @@ now six total etcd replicas.
 
 - [Install Terraform](https://www.terraform.io/intro/getting-started/install.html)
 - Install dependencies: `sudo pip install -r requirements.txt`
-- Account with Equinix Metal
+- Account with Packet Host
 - An SSH key pair
 
 ## SSH Key Setup
 
-An SSH keypair is required so Ansible can access the newly provisioned nodes (Equinix Metal hosts). By default, the public SSH key defined in cluster.tfvars will be installed in authorized_key on the newly provisioned nodes (~/.ssh/id_rsa.pub). Terraform will upload this public key and then it will be distributed out to all the nodes. If you have already set this public key in Equinix Metal (i.e. via the portal), then set the public keyfile name in cluster.tfvars to blank to prevent the duplicate key from being uploaded which will cause an error.
+An SSH keypair is required so Ansible can access the newly provisioned nodes (bare metal Packet hosts). By default, the public SSH key defined in cluster.tfvars will be installed in authorized_key on the newly provisioned nodes (~/.ssh/id_rsa.pub). Terraform will upload this public key and then it will be distributed out to all the nodes. If you have already set this public key in Packet (i.e. via the portal), then set the public keyfile name in cluster.tfvars to blank to prevent the duplicate key from being uploaded which will cause an error.
 
 If you don't already have a keypair generated (~/.ssh/id_rsa and ~/.ssh/id_rsa.pub), then a new keypair can be generated with the command:
 
@@ -51,7 +51,7 @@ ssh-keygen -f ~/.ssh/id_rsa
 
 ## Terraform
 
-Terraform will be used to provision all of the Equinix Metal resources with base software as appropriate.
+Terraform will be used to provision all of the Packet resources with base software as appropriate.
 
 ### Configuration
 
@@ -67,18 +67,18 @@ ln -s ../../contrib/terraform/packet/hosts
 
 This will be the base for subsequent Terraform commands.
 
-#### Equinix Metal API access
+#### Packet API access
 
-Your Equinix Metal API key must be available in the `PACKET_AUTH_TOKEN` environment variable.
+Your Packet API key must be available in the `PACKET_AUTH_TOKEN` environment variable.
 This key is typically stored outside of the code repo since it is considered secret.
 If someone gets this key, they can startup/shutdown hosts in your project!
 
 For more information on how to generate an API key or find your project ID, please see
-[Accounts Index](https://metal.equinix.com/developers/docs/accounts/).
+[API Integrations](https://support.packet.com/kb/articles/api-integrations)
 
-The Equinix Metal Project ID associated with the key will be set later in `cluster.tfvars`.
+The Packet Project ID associated with the key will be set later in cluster.tfvars.
 
-For more information about the API, please see [Equinix Metal API](https://metal.equinix.com/developers/api/).
+For more information about the API, please see [Packet API](https://www.packet.com/developers/api/)
 
 Example:
 
@@ -101,7 +101,7 @@ This helps when identifying which hosts are associated with each cluster.
 While the defaults in variables.tf will successfully deploy a cluster, it is recommended to set the following values:
 
 - cluster_name = the name of the inventory directory created above as $CLUSTER
-- packet_project_id = the Equinix Metal Project ID associated with the Equinix Metal API token above
+- packet_project_id = the Packet Project ID associated with the Packet API token above
 
 #### Enable localhost access
 

contrib/terraform/packet/kubespray.tf

@@ -1,4 +1,4 @@
-# Configure the Equinix Metal Provider
+# Configure the Packet Provider
 provider "packet" {
   version = "~> 2.0"
 }

contrib/terraform/packet/sample-inventory/cluster.tfvars

@@ -1,12 +1,12 @@
 # your Kubernetes cluster name here
 cluster_name = "mycluster"
 
-# Your Equinix Metal project ID. See hhttps://metal.equinix.com/developers/docs/accounts/
+# Your Packet project ID. See https://support.packet.com/kb/articles/api-integrations
 packet_project_id = "Example-API-Token"
 
-# The public SSH key to be uploaded into authorized_keys in bare metal Equinix Metal nodes provisioned
+# The public SSH key to be uploaded into authorized_keys in bare metal Packet nodes provisioned
-# leave this value blank if the public key is already setup in the Equinix Metal project
+# leave this value blank if the public key is already setup in the Packet project
-# Terraform will complain if the public key is setup in Equinix Metal
+# Terraform will complain if the public key is setup in Packet
 public_key_path = "~/.ssh/id_rsa.pub"
 
 # cluster location

contrib/terraform/packet/variables.tf

@@ -3,7 +3,7 @@ variable "cluster_name" {
 }
 
 variable "packet_project_id" {
-  description = "Your Equinix Metal project ID. See https://metal.equinix.com/developers/docs/accounts/"
+  description = "Your Packet project ID. See https://support.packet.com/kb/articles/api-integrations"
 }
 
 variable "operating_system" {

contrib/terraform/terraform.py

@@ -323,11 +323,11 @@ def openstack_host(resource, module_name):
     })
 
     # add groups based on attrs
-    groups.append('os_image=' + str(attrs['image']['id']))
+    groups.append('os_image=' + attrs['image']['id'])
-    groups.append('os_flavor=' + str(attrs['flavor']['name']))
+    groups.append('os_flavor=' + attrs['flavor']['name'])
     groups.extend('os_metadata_%s=%s' % item
                   for item in list(attrs['metadata'].items()))
-    groups.append('os_region=' + str(attrs['region']))
+    groups.append('os_region=' + attrs['region'])
 
     # groups specific to kubespray
     for group in attrs['metadata'].get('kubespray_groups', "").split(","):

contrib/terraform/upcloud/README.md

@@ -8,10 +8,10 @@ The setup looks like following
 
 ```text
    Kubernetes cluster
-+--------------------------+
++-----------------------+
 |   +--------------+    |
 |   | +--------------+  |
-| -->  | |              |  |
+|   | |              |  |
 |   | | Master/etcd  |  |
 |   | | node(s)      |  |
 |   +-+              |  |
@@ -21,16 +21,14 @@ The setup looks like following
 |           v           |
 |   +--------------+    |
 |   | +--------------+  |
-| -->  | |              |  |
+|   | |              |  |
 |   | |    Worker    |  |
 |   | |    node(s)   |  |
 |   +-+              |  |
 |     +--------------+  |
-+--------------------------+
++-----------------------+
 ```
 
-The nodes uses a private network for node to node communication and a public interface for all external communication.
-
 ## Requirements
 
 * Terraform 0.13.0 or newer
@@ -96,10 +94,9 @@ terraform destroy --var-file cluster-settings.tfvars \
 
 ## Variables
 
-* `prefix`: Prefix to add to all resources, if set to "" don't set any prefix
+* `hostname`: A valid domain name, e.g. example.com. The maximum length is 128 characters.
 * `template_name`: The name or UUID  of a base image
-* `username`: a user to access the nodes, defaults to "ubuntu"
+* `username`: a user to access the nodes
-* `private_network_cidr`: CIDR to use for the private network, defaults to "172.16.0.0/24"
 * `ssh_public_keys`: List of public SSH keys to install on all machines
 * `zone`: The zone where to run the cluster
 * `machines`: Machines to provision. Key of this object will be used as the name of the machine
@@ -107,6 +104,3 @@ terraform destroy --var-file cluster-settings.tfvars \
   * `cpu`: number of cpu cores
   * `mem`: memory size in MB
   * `disk_size`: The size of the storage in GB
-  * `additional_disks`: Additional disks to attach to the node.
-    * `size`: The size of the additional disk in GB
-    * `tier`: The tier of disk to use (`maxiops` is the only one you can choose atm)

contrib/terraform/upcloud/cluster-settings.tfvars

@@ -1,12 +1,13 @@
+
 # See: https://developers.upcloud.com/1.3/5-zones/
 zone     = "fi-hel1"
 username = "ubuntu"
 
-# Prefix to use for all resources to separate them from other resources
-prefix = "kubespray"
-
 inventory_file = "inventory.ini"
 
+#  A valid domain name, e.g. host.example.com. The maximum length is 128 characters.
+hostname = "example.com"
+
 #  Set the operating system using UUID or exact name
 template_name = "Ubuntu Server 20.04 LTS (Focal Fossa)"
 
@@ -16,7 +17,7 @@ ssh_public_keys = [
   "ssh-rsa public key 2",
 ]
 
-# check list of available plan https://developers.upcloud.com/1.3/7-plans/
+#check list of available plan https://developers.upcloud.com/1.3/7-plans/
 machines = {
   "master-0" : {
     "node_type" : "master",
@@ -26,7 +27,6 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
-    "additional_disks" : {}
   },
   "worker-0" : {
     "node_type" : "worker",
@@ -36,16 +36,6 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
-    "additional_disks" : {
-      # "some-disk-name-1": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # },
-      # "some-disk-name-2": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # }
-    }
   },
   "worker-1" : {
     "node_type" : "worker",
@@ -55,16 +45,6 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
-    "additional_disks" : {
-      # "some-disk-name-1": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # },
-      # "some-disk-name-2": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # }
-    }
   },
   "worker-2" : {
     "node_type" : "worker",
@@ -74,15 +54,5 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
-    "additional_disks" : {
-      # "some-disk-name-1": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # },
-      # "some-disk-name-2": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # }
-    }
   }
 }

contrib/terraform/upcloud/main.tf

@@ -11,14 +11,12 @@ provider "upcloud" {
 module "kubernetes" {
   source = "./modules/kubernetes-cluster"
 
-  prefix = var.prefix
   zone     = var.zone
+  hostname = var.hostname
 
   template_name = var.template_name
   username      = var.username
 
-  private_network_cidr = var.private_network_cidr
-
   machines = var.machines
 
   ssh_public_keys = var.ssh_public_keys
@@ -32,15 +30,13 @@ data "template_file" "inventory" {
   template = file("${path.module}/templates/inventory.tpl")
 
   vars = {
-    connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d",
+    connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s etcd_member_name=etcd%d",
       keys(module.kubernetes.master_ip),
-      values(module.kubernetes.master_ip).*.public_ip,
+      values(module.kubernetes.master_ip),
-      values(module.kubernetes.master_ip).*.private_ip,
     range(1, length(module.kubernetes.master_ip) + 1)))
-    connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s",
+    connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s",
       keys(module.kubernetes.worker_ip),
-      values(module.kubernetes.worker_ip).*.public_ip,
+    values(module.kubernetes.worker_ip)))
-    values(module.kubernetes.worker_ip).*.private_ip))
     list_master = join("\n", formatlist("%s",
     keys(module.kubernetes.master_ip)))
     list_worker = join("\n", formatlist("%s",

contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf

@@ -1,41 +1,3 @@
-locals {
-  # Create a list of all disks to create
-  disks = flatten([
-    for node_name, machine in var.machines : [
-      for disk_name, disk in machine.additional_disks : {
-        disk = disk
-        disk_name = disk_name
-        node_name = node_name
-      }
-    ]
-  ])
-
-  # If prefix is set, all resources will be prefixed with "${var.prefix}-"
-  # Else don't prefix with anything
-  resource-prefix = "%{ if var.prefix != ""}${var.prefix}-%{ endif }"
-}
-
-resource "upcloud_network" "private" {
-  name = "${local.resource-prefix}k8s-network"
-  zone = var.zone
-
-  ip_network {
-    address = var.private_network_cidr
-    dhcp    = true
-    family  = "IPv4"
-  }
-}
-
-resource "upcloud_storage" "additional_disks" {
-  for_each = {
-    for disk in local.disks: "${disk.node_name}_${disk.disk_name}" => disk.disk
-  }
-
-  size  = each.value.size
-  tier  = each.value.tier
-  title = "${local.resource-prefix}${each.key}"
-  zone  = var.zone
-}
 
 resource "upcloud_server" "master" {
   for_each = {
@@ -44,7 +6,7 @@ resource "upcloud_server" "master" {
     if machine.node_type == "master"
   }
 
-  hostname = "${local.resource-prefix}${each.key}"
+  hostname    = "${each.key}.${var.hostname}"
   cpu            = each.value.cpu
   mem       = each.value.mem
   zone            = var.zone
@@ -54,38 +16,25 @@ resource "upcloud_server" "master" {
   size = each.value.disk_size
   }
 
-  # Public network interface
+  # Network interfaces
  network_interface {
    type = "public"
  }
 
-  # Private network interface
  network_interface {
-    type    = "private"
+   type = "utility"
-    network = upcloud_network.private.id
  }
-
-  dynamic "storage_devices" {
-    for_each = {
-      for disk_key_name, disk in upcloud_storage.additional_disks :
-        disk_key_name => disk
-        # Only add the disk if it matches the node name in the start of its name
-        if length(regexall("^${each.key}_.+", disk_key_name)) > 0
-    }
-
-    content {
-      storage = storage_devices.value.id
-    }
-  }
-
  # Include at least one public SSH key
  login {
    user = var.username
    keys = var.ssh_public_keys
    create_password = false
+
  }
+
 }
 
+
 resource "upcloud_server" "worker" {
   for_each = {
     for name, machine in var.machines :
@@ -93,7 +42,7 @@ resource "upcloud_server" "worker" {
     if machine.node_type == "worker"
   }
 
-  hostname = "${local.resource-prefix}${each.key}"
+  hostname    = "${each.key}.${var.hostname}"
   cpu            = each.value.cpu
   mem       = each.value.mem
   zone            = var.zone
@@ -103,30 +52,11 @@ resource "upcloud_server" "worker" {
   size = each.value.disk_size
   }
 
-  # Public network interface
+  # Network interfaces
  network_interface {
    type = "public"
  }
 
-  # Private network interface
-  network_interface {
-    type    = "private"
-    network = upcloud_network.private.id
-  }
-
-  dynamic "storage_devices" {
-    for_each = {
-      for disk_key_name, disk in upcloud_storage.additional_disks :
-        disk_key_name => disk
-        # Only add the disk if it matches the node name in the start of its name
-        if length(regexall("^${each.key}_.+", disk_key_name)) > 0
-    }
-
-    content {
-      storage = storage_devices.value.id
-    }
-  }
-
  # Include at least one public SSH key
  login {
    user = var.username

contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf

@@ -2,19 +2,13 @@
 output "master_ip" {
   value = {
     for instance in upcloud_server.master :
-    instance.hostname => {
+    instance.hostname => instance.network_interface[0].ip_address
-      "public_ip": instance.network_interface[0].ip_address
-      "private_ip": instance.network_interface[1].ip_address
-    }
   }
 }
 
 output "worker_ip" {
   value = {
     for instance in upcloud_server.worker :
-    instance.hostname => {
+    instance.hostname => instance.network_interface[0].ip_address
-      "public_ip": instance.network_interface[0].ip_address
-      "private_ip": instance.network_interface[1].ip_address
-    }
   }
 }

contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf

@@ -1,16 +1,14 @@
-variable "prefix" {
-  type = string
-}
-
 variable "zone" {
   type = string
 }
 
-variable "template_name" {}
+variable "hostname"{
+ default ="example.com"
+}
 
-variable "username" {}
+variable "template_name"{}
 
-variable "private_network_cidr" {}
+variable "username"{}
 
 variable "machines" {
   description = "Cluster machines"
@@ -19,10 +17,6 @@ variable "machines" {
     cpu      = string
     mem      = string
     disk_size =  number
-    additional_disks = map(object({
-      size = number
-      tier = string
-    }))
   }))
 }
 

contrib/terraform/upcloud/sample-inventory/cluster.tfvars

@@ -2,21 +2,20 @@
 zone     = "fi-hel1"
 username = "ubuntu"
 
-# Prefix to use for all resources to separate them from other resources
-prefix = "kubespray"
-
 inventory_file = "inventory.ini"
 
+#  A valid domain name, e.g. host.example.com. The maximum length is 128 characters.
+hostname = "example.com"
+
 #  Set the operating system using UUID or exact name
 template_name = "Ubuntu Server 20.04 LTS (Focal Fossa)"
-
 ssh_public_keys = [
   # Put your public SSH key here
   "ssh-rsa I-did-not-read-the-docs",
   "ssh-rsa I-did-not-read-the-docs 2",
 ]
 
-# check list of available plan https://developers.upcloud.com/1.3/7-plans/
+check list of available plan https://developers.upcloud.com/1.3/7-plans/
 machines = {
   "master-0" : {
     "node_type" : "master",
@@ -26,7 +25,6 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
-    "additional_disks": {}
   },
   "worker-0" : {
     "node_type" : "worker",
@@ -36,16 +34,6 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
-    "additional_disks": {
-      # "some-disk-name-1": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # },
-      # "some-disk-name-2": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # }
-    }
   },
   "worker-1" : {
     "node_type" : "worker",
@@ -55,16 +43,6 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
-    "additional_disks": {
-      # "some-disk-name-1": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # },
-      # "some-disk-name-2": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # }
-    }
   },
   "worker-2" : {
     "node_type" : "worker",
@@ -74,15 +52,5 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
-    "additional_disks": {
-      # "some-disk-name-1": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # },
-      # "some-disk-name-2": {
-      #   "size": 100,
-      #   "tier": "maxiops",
-      # }
-    }
   }
 }

contrib/terraform/upcloud/variables.tf

@@ -1,40 +1,23 @@
-variable "prefix" {
-  type    = string
-  default = "kubespray"
-
-  description = "Prefix that is used to distinguish these resources from others"
-}
 
 variable "zone" {
   description = "The zone where to run the cluster"
 }
 
-variable "template_name" {
+variable "hostname" {
-  description = "Block describing the preconfigured operating system"
+  default = "example.com"
 }
 
-variable "username" {
+variable "template_name" {}
-  description = "The username to use for the nodes"
-  default     = "ubuntu"
-}
 
-variable "private_network_cidr" {
+variable "username" {}
-  description = "CIDR to use for the private network"
-  default     = "172.16.0.0/24"
-}
 
 variable "machines" {
   description = "Cluster machines"
-
   type = map(object({
     node_type = string
     cpu       = string
     mem       = string
     disk_size = number
-    additional_disks = map(object({
-      size = number
-      tier = string
-    }))
   }))
 }
 
@@ -47,10 +30,6 @@ variable "inventory_file" {
   description = "Where to store the generated inventory file"
 }
 
-variable "UPCLOUD_USERNAME" {
+variable "UPCLOUD_USERNAME" {}
-  description = "UpCloud username with API access"
-}
 
-variable "UPCLOUD_PASSWORD" {
+variable "UPCLOUD_PASSWORD" {}
-  description = "Password for UpCloud API user"
-}

docs/_sidebar.md

@@ -14,46 +14,30 @@
   * [Calico](docs/calico.md)
   * [Flannel](docs/flannel.md)
   * [Kube Router](docs/kube-router.md)
-  * [Kube OVN](docs/kube-ovn.md)
   * [Weave](docs/weave.md)
   * [Multus](docs/multus.md)
-  * [OVN4NFV](docs/ovn4nfv.md)
 * Ingress
-  * [ALB Ingress](docs/ingress_controller/alb_ingress_controller.md)
+  * [Ambassador](docs/ambassador.md)
-  * [Ambassador](docs/ingress_controller/ambassador.md)
-  * [MetalLB](docs/metallb.md)
-  * [Nginx Ingress](docs/ingress_controller/ingress_nginx.md)
 * [Cloud providers](docs/cloud.md)
   * [AWS](docs/aws.md)
   * [Azure](docs/azure.md)
   * [OpenStack](/docs/openstack.md)
-  * [Equinix Metal](/docs/equinix-metal.md)
+  * [Packet](/docs/packet.md)
   * [vSphere](/docs/vsphere.md)
-* [Operating Systems](docs/bootstrap-os.md)
+* Operating Systems
   * [Debian](docs/debian.md)
   * [Flatcar Container Linux](docs/flatcar.md)
   * [Fedora CoreOS](docs/fcos.md)
   * [OpenSUSE](docs/opensuse.md)
-  * [RedHat Enterprise Linux](docs/rhel.md)
-  * [CentOS/OracleLinux/AlmaLinux](docs/centos8.md)
-  * [Amazon Linux 2](docs/amazonlinux.md)
 * CRI
   * [Containerd](docs/containerd.md)
   * [CRI-O](docs/cri-o.md)
-  * [Kata Containers](docs/kata-containers.md)
-  * [gVisor](docs/gvisor.md)
 * Advanced
   * [Proxy](/docs/proxy.md)
   * [Downloads](docs/downloads.md)
   * [Netcheck](docs/netcheck.md)
-  * [Cert Manager](docs/cert_manager.md)
   * [DNS Stack](docs/dns-stack.md)
   * [Kubernetes reliability](docs/kubernetes-reliability.md)
-  * [Local Registry](docs/kubernetes-apps/registry.md)
-* External Storage Provisioners
-  * [RBD Provisioner](docs/kubernetes-apps/rbd_provisioner.md)
-  * [CEPHFS Provisioner](docs/kubernetes-apps/cephfs_provisioner.md)
-  * [Local Volume Provisioner](docs/kubernetes-apps/local_volume_provisioner.md)
 * Developers
   * [Test cases](docs/test_cases.md)
   * [Vagrant](docs/vagrant.md)

docs/ambassador.md

@@ -16,16 +16,6 @@ URL rewriting, CORS, rate limiting, and automatic metrics collection.
   installation/updates.
 * `ingress_ambassador_secure_port` (default: 443): HTTPS port to listen at.
 * `ingress_ambassador_insecure_port` (default: 80): HTTP port to listen at.
-* `ingress_ambassador_multi_namespaces` (default `false`): By default, Ambassador will only
-  watch the `ingress_ambassador_namespace` namespace for `AmbassadorInstallation` CRD resources.
-  When set to `true`, this value will tell the Ambassador Operator to watch **all** namespaces
-  for CRDs. If you want to run multiple Ambassador ingress instances, set this to `true`.
-
-### Ingress annotations
-
-The Ambassador API Gateway will automatically load balance `Ingress` resources
-that include the annotation `kubernetes.io/ingress.class=ambassador`. All the other
-resources will be just ignored.
 
 ### Ambassador Operator
 

docs/ansible.md

@@ -20,7 +20,7 @@ When _kube_node_ contains _etcd_, you define your etcd cluster to be as well sch
 If you want it a standalone, make sure those groups do not intersect.
 If you want the server to act both as control-plane and node, the server must be defined
 on both groups _kube_control_plane_ and _kube_node_. If you want a standalone and
-unschedulable control plane, the server must be defined only in the _kube_control_plane_ and
+unschedulable master, the server must be defined only in the _kube_control_plane_ and
 not _kube_node_.
 
 There are also two special groups:
@@ -67,7 +67,7 @@ The group variables to control main deployment options are located in the direct
 Optional variables are located in the `inventory/sample/group_vars/all.yml`.
 Mandatory variables that are common for at least one role (or a node group) can be found in the
 `inventory/sample/group_vars/k8s_cluster.yml`.
-There are also role vars for docker, kubernetes preinstall and control plane roles.
+There are also role vars for docker, kubernetes preinstall and master roles.
 According to the [ansible docs](https://docs.ansible.com/ansible/latest/playbooks_variables.html#variable-precedence-where-should-i-put-a-variable),
 those cannot be overridden from the group vars. In order to override, one should use
 the `-e` runtime flags (most simple way) or other layers described in the docs.
@@ -99,111 +99,45 @@ task vars (only for the task) | Unused for roles, but only for helper scripts
 The following tags are defined in playbooks:
 
 |                 Tag name | Used for
-|--------------------------------|---------
+|--------------------------|---------
-|                     ambassador | Ambassador Ingress Controller
-|                       annotate | Create kube-router annotation
 |                     apps | K8s apps definitions
-|                        asserts | Check tasks for download role
+|                    azure | Cloud-provider Azure
-|             aws-ebs-csi-driver | Configuring csi driver: aws-ebs
-|               azure-csi-driver | Configuring csi driver: azure
 |                  bastion | Setup ssh config for bastion
 |             bootstrap-os | Anything related to host OS configuration
 |                   calico | Network plugin Calico
-|                      calico_rr | Configuring Calico route reflector
 |                    canal | Network plugin Canal
-|             cephfs-provisioner | Configuring CephFS
-|                   cert-manager | Configuring certificate manager for K8s
-|                         cilium | Network plugin Cilium
-|              cinder-csi-driver | Configuring csi driver: cinder
-|                         client | Kubernetes clients role
 |           cloud-provider | Cloud-provider related tasks
-|                  cluster-roles | Configuring cluster wide application (psp ...)
+|                   docker | Configuring docker for hosts
-|                            cni | CNI plugins for Network Plugins
-|                     containerd | Configuring containerd engine runtime for hosts
-|   container_engine_accelerator | Enable nvidia accelerator for runtimes
-|               container-engine | Configuring container engines
-|             container-runtimes | Configuring container runtimes
-|                        coredns | Configuring coredns deployment
-|                           crio | Configuring crio container engine for hosts
-|                           crun | Configuring crun runtime
-|                     csi-driver | Configuring csi driver
-|                      dashboard | Installing and configuring the Kubernetes Dashboard
-|                            dns | Remove dns entries when resetting
-|                         docker | Configuring docker engine runtime for hosts
 |                 download | Fetching container images to a delegate host
 |                     etcd | Configuring etcd cluster
+|         etcd-pre-upgrade | Upgrading etcd cluster
 |             etcd-secrets | Configuring etcd certs/keys
 |                 etchosts | Configuring /etc/hosts entries for hosts
-|      external-cloud-controller | Configure cloud controllers
-|             external-openstack | Cloud controller : openstack
-|           external-provisioner | Configure external provisioners
-|               external-vsphere | Cloud controller : vsphere
 |                    facts | Gathering facts and misc check results
-|                          files | Remove files when resetting
 |                  flannel | Network plugin flannel
 |                      gce | Cloud-provider GCP
-|              gcp-pd-csi-driver | Configuring csi driver: gcp-pd
-|                         gvisor | Configuring gvisor runtime
-|                           helm | Installing and configuring Helm
-|             ingress-controller | Configure ingress controllers
-|                    ingress_alb | AWS ALB Ingress Controller
-|                           init | Windows kubernetes init nodes
-|                       iptables | Flush and clear iptable when resetting
 |          k8s-pre-upgrade | Upgrading K8s cluster
 |              k8s-secrets | Configuring K8s certs/keys
-|                 k8s-gen-tokens | Configuring K8s tokens
-|                kata-containers | Configuring kata-containers runtime
-|                           krew | Install and manage krew
-|                        kubeadm | Roles linked to kubeadm tasks
 |           kube-apiserver | Configuring static pod kube-apiserver
 |  kube-controller-manager | Configuring static pod kube-controller-manager
 |                  kubectl | Installing kubectl and bash completion
 |                  kubelet | Configuring kubelet service
-|                       kube-ovn | Network plugin kube-ovn
-|                    kube-router | Network plugin kube-router
 |               kube-proxy | Configuring static pod kube-proxy
+|           kube-scheduler | Configuring static pod kube-scheduler
 |                localhost | Special steps for the localhost (ansible runner)
-|         local-path-provisioner | Configure External provisioner: local-path
-|       local-volume-provisioner | Configure External provisioner: local-volume
-|                        macvlan | Network plugin macvlan
 |                   master | Configuring K8s master node role
-|                        metallb | Installing and configuring metallb
-|                 metrics_server | Configuring metrics_server
 |               netchecker | Installing netchecker K8s app
 |                  network | Configuring networking plugins for K8s
-|                         mounts | Umount kubelet dirs when reseting
-|                         multus | Network plugin multus
 |                    nginx | Configuring LB for kube-apiserver instances
 |                     node | Configuring K8s minion (compute) node role
-|                   nodelocaldns | Configuring nodelocaldns daemonset
+|                openstack | Cloud-provider OpenStack
-|                     node-label | Tasks linked to labeling of nodes
-|                   node-webhook | Tasks linked to webhook (grating access to resources)
-|                     nvidia_gpu | Enable nvidia accelerator for runtimes
-|                            oci | Cloud provider: oci
-|                        ovn4nfv | Network plugin ovn4nfv
-|             persistent_volumes | Configure csi volumes
-| persistent_volumes_aws_ebs_csi | Configuring csi driver: aws-ebs
-| persistent_volumes_cinder_csi  | Configuring csi driver: cinder
-| persistent_volumes_gcp_pd_csi  | Configuring csi driver: gcp-pd
-| persistent_volumes_openstack   | Configuring csi driver: openstack
-|              policy-controller | Configuring Calico policy controller
-|                    post-remove | Tasks running post-remove operation
-|                   post-upgrade | Tasks running post-upgrade operation
-|                     pre-remove | Tasks running pre-remove operation
-|                    pre-upgrade | Tasks running pre-upgrade operation
 |               preinstall | Preliminary configuration steps
-|                       registry | Configuring local docker registry
-|                          reset | Tasks running doing the node reset
 |               resolvconf | Configuring /etc/resolv.conf for hosts/apps
-|                rbd-provisioner | Configure External provisioner: rdb
-|                       services | Remove services (etcd, kubelet etc...) when resetting
-|                       snapshot | Enabling csi snapshot
-|            snapshot-controller | Configuring csi snapshot controller
 |                  upgrade | Upgrading, f.e. container images/binaries
 |                   upload | Distributing images/binaries across hosts
-|             vsphere-csi-driver | Configuring csi driver: vsphere
 |                    weave | Network plugin Weave
-|                      win_nodes | Running windows specific tasks
+|              ingress_alb | AWS ALB Ingress Controller
+|              ambassador  | Ambassador Ingress Controller
 
 Note: Use the ``bash scripts/gen_tags.sh`` command to generate a list of all
 tags found in the codebase. New tags will be listed with the empty "Used for"
@@ -253,28 +187,3 @@ For more information about Ansible and bastion hosts, read
 ## Mitogen
 
 You can use [mitogen](mitogen.md) to speed up kubespray.
-
-## Beyond ansible 2.9
-
-Ansible project has decided, in order to ease their maintenance burden, to split between
-two projects which are now joined under the Ansible umbrella.
-
-Ansible-base (2.10.x branch) will contain just the ansible language implementation while
-ansible modules that were previously bundled into a single repository will be part of the
-ansible 3.x package. Pleasee see [this blog post](https://blog.while-true-do.io/ansible-release-3-0-0/)
-that explains in detail the need and the evolution plan.
-
-**Note:** this change means that ansible virtual envs cannot be upgraded with `pip install -U`.
-You first need to uninstall your old ansible (pre 2.10) version and install the new one.
-
-```ShellSession
-pip uninstall ansible
-cd kubespray/
-pip install -U .
-```
-
-**Note:** some changes needed to support ansible 2.10+ are not backwards compatible with 2.9
-Kubespray needs to evolve and keep pace with upstream ansible and will be forced to eventually
-drop 2.9 support. Kubespray CIs use only the ansible version specified in the `requirements.txt`
-and while the `ansible_version.yml` may allow older versions to be used, these are not
-exercised in the CI and compatibility is not guaranteed.

docs/azure.md

@@ -10,7 +10,7 @@ Not all features are supported yet though, for a list of the current status have
 
 Before creating the instances you must first set the `azure_` variables in the `group_vars/all/all.yml` file.
 
-All of the values can be retrieved using the Azure CLI tool which can be downloaded here: <https://docs.microsoft.com/en-gb/cli/azure/install-azure-cli>
+All of the values can be retrieved using the azure cli tool which can be downloaded here: <https://docs.microsoft.com/en-gb/azure/xplat-cli-install>
 After installation you have to run `az login` to get access to your account.
 
 ### azure_cloud

docs/calico.md

@@ -189,7 +189,7 @@ To re-define default action please set the following variable in your inventory:
 calico_endpoint_to_host_action: "ACCEPT"
 ```
 
-### Optional : Define address on which Felix will respond to health requests
+## Optional : Define address on which Felix will respond to health requests
 
 Since Calico 3.2.0, HealthCheck default behavior changed from listening on all interfaces to just listening on localhost.
 
@@ -199,15 +199,6 @@ To re-define health host please set the following variable in your inventory:
 calico_healthhost: "0.0.0.0"
 ```
 
-### Optional : Configure Calico Node probe timeouts
-
-Under certain conditions a deployer may need to tune the Calico liveness and readiness probes timeout settings. These can be configured like this:
-
-```yml
-calico_node_livenessprobe_timeout: 10
-calico_node_readinessprobe_timeout: 10
-```
-
 ## Config encapsulation for cross server traffic
 
 Calico supports two types of encapsulation: [VXLAN and IP in IP](https://docs.projectcalico.org/v3.11/networking/vxlan-ipip). VXLAN is supported in some environments where IP in IP is not (for example, Azure).
@@ -228,19 +219,6 @@ calico_vxlan_mode: 'Never'
 
 If you use VXLAN mode, BGP networking is not required. You can disable BGP to reduce the moving parts in your cluster by `calico_network_backend: vxlan`
 
-## Configuring interface MTU
-
-This is an advanced topic and should usually not be modified unless you know exactly what you are doing. Calico is smart enough to deal with the defaults and calculate the proper MTU. If you do need to set up a custom MTU you can change `calico_veth_mtu` as follows:
-
-* If Wireguard is enabled, subtract 60 from your network MTU (i.e. 1500-60=1440)
-* If using VXLAN or BPF mode is enabled, subtract 50 from your network MTU (i.e. 1500-50=1450)
-* If using IPIP, subtract 20 from your network MTU (i.e. 1500-20=1480)
-* if not using any encapsulation, set to your network MTU (i.e. 1500 or 9000)
-
-```yaml
-calico_veth_mtu: 1440
-```
-
 ## Cloud providers configuration
 
 Please refer to the official documentation, for example [GCE configuration](http://docs.projectcalico.org/v1.5/getting-started/docker/installation/gce) requires a security rule for calico ip-ip tunnels. Note, calico is always configured with ``calico_ipip_mode: Always`` if the cloud provider was defined.
@@ -282,93 +260,3 @@ calico_ipam_host_local: true
 ```
 
 Refer to Project Calico section [Using host-local IPAM](https://docs.projectcalico.org/reference/cni-plugin/configuration#using-host-local-ipam) for further information.
-
-## eBPF Support
-
-Calico supports eBPF for its data plane see [an introduction to the Calico eBPF Dataplane](https://www.projectcalico.org/introducing-the-calico-ebpf-dataplane/) for further information.
-
-Note that it is advisable to always use the latest version of Calico when using the eBPF dataplane.
-
-### Enabling eBPF support
-
-To enable the eBPF dataplane support ensure you add the following to your inventory. Note that the `kube-proxy` is incompatible with running Calico in eBPF mode and the kube-proxy should be removed from the system.
-
-```yaml
-calico_bpf_enabled: true
-kube_proxy_remove: true
-```
-
-### Cleaning up after kube-proxy
-
-Calico node cannot clean up after kube-proxy has run in ipvs mode. If you are converting an existing cluster to eBPF you will need to ensure the `kube-proxy` DaemonSet is deleted and that ipvs rules are cleaned.
-
-To check that kube-proxy was running in ipvs mode:
-
-```ShellSession
-# ipvsadm -l
-```
-
-To clean up any ipvs leftovers:
-
-```ShellSession
-# ipvsadm -C
-```
-
-### Calico access to the kube-api
-
-Calico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.projectcalico.org/maintenance/ebpf/enabling-bpf) for guidelines on how to do this.
-
-Kubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `loadbalancer_apiserver` inventory variable documented in [HA Mode](./ha-mode.md).
-
-If no external loadbalancer is used, Calico eBPF can also use the localhost loadbalancer option. In this case Calico Automatic Host Endpoints need to be enabled to allow services like `coredns` and `metrics-server` to communicate with the kubernetes host endpoint. See [this blog post](https://www.projectcalico.org/securing-kubernetes-nodes-with-calico-automatic-host-endpoints/) on enabling automatic host endpoints.
-
-```yaml
-loadbalancer_apiserver_localhost: true
-use_localhost_as_kubeapi_loadbalancer: true
-```
-
-### Tunneled versus Direct Server Return
-
-By default Calico usese Tunneled service mode but it can use direct server return (DSR) in order to optimize the return path for a service.
-
-To configure DSR:
-
-```yaml
-calico_bpf_service_mode: "DSR"
-```
-
-### eBPF Logging and Troubleshooting
-
-In order to enable Calico eBPF mode logging:
-
-```yaml
-calico_bpf_log_level: "Debug"
-```
-
-To view the logs you need to use the `tc` command to read the kernel trace buffer:
-
-```ShellSession
-tc exec bpf debug
-```
-
-Please see [Calico eBPF troubleshooting guide](https://docs.projectcalico.org/maintenance/troubleshoot/troubleshoot-ebpf#ebpf-program-debug-logs).
-
-## Wireguard Encryption
-
-Calico supports using Wireguard for encryption. Please see the docs on [encryptiong cluster pod traffic](https://docs.projectcalico.org/security/encrypt-cluster-pod-traffic).
-
-To enable wireguard support:
-
-```yaml
-calico_wireguard_enabled: true
-```
-
-The following OSes will require enabling the EPEL repo in order to bring in wireguard tools:
-
-* CentOS 7 & 8
-* AlmaLinux 8
-* Amazon Linux 2
-
-```yaml
-epel_enabled: true
-```

docs/ci.md

@@ -10,10 +10,9 @@ amazon |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 centos7 |  :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :white_check_mark: |
 centos8 |  :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
 debian10 |  :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian11 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian9 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
+fedora32 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
 fedora33 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora34 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
 opensuse |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 oracle7 |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu16 |  :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :white_check_mark: |
@@ -28,10 +27,9 @@ amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 centos7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 centos8 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian10 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian11 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora32 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora33 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora34 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 opensuse |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 oracle7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu16 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
@@ -46,10 +44,9 @@ amazon |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 centos7 |  :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
 centos8 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian10 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian11 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora32 |  :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
 fedora33 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora34 |  :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
 opensuse |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 oracle7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu16 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |

docs/cilium.md

@@ -15,7 +15,7 @@ balancer deployed by Kubespray and **only contacts the first master**.
 ## Choose Cilium version
 
 ```yml
-cilium_version: v1.9.9
+cilium_version: v1.8.9 ## or v1.9.6
 ```
 
 ## Add variable to config

docs/comparisons.md

@@ -21,6 +21,7 @@ does generic configuration management tasks from the "OS operators" ansible
 world, plus some initial K8s clustering (with networking plugins included) and
 control plane bootstrapping.
 
-Kubespray has started using `kubeadm` internally for cluster creation since v2.3
+Kubespray supports `kubeadm` for cluster creation since v2.3
+(and deprecated non-kubeadm deployment starting from v2.8)
 in order to consume life cycle management domain knowledge from it
 and offload generic OS configuration things from it, which hopefully benefits both sides.

docs/coreos.md

@@ -0,0 +1,14 @@
+CoreOS bootstrap
+===============
+
+Example with Ansible:
+
+Before running the cluster playbook you must satisfy the following requirements:
+
+General CoreOS Pre-Installation Notes:
+
+- Ensure that the bin_dir is set to `/opt/bin`
+- ansible_python_interpreter should be `/opt/bin/python`. This will be laid down by the bootstrap task.
+- The default resolvconf_mode setting of `docker_dns` **does not** work for CoreOS. This is because we do not edit the systemd service file for docker on CoreOS nodes. Instead, just use the `host_resolvconf` mode. It should work out of the box.
+
+Then you can proceed to [cluster deployment](#run-deployment)

docs/fcos.md

@@ -1,6 +1,6 @@
 # Fedora CoreOS
 
-Tested with stable version 34.20210611.3.0
+Tested with stable version 31.20200223.3.0.
 
 Because package installation with `rpm-ostree` requires a reboot, playbook may fail while bootstrap.
 Restart playbook again.
@@ -9,9 +9,33 @@ Restart playbook again.
 
 Tested with
 
-- containerd
+- docker
 - crio
 
+### docker
+
+OS base packages contains docker.
+
+### cri-o
+
+To use `cri-o` disable docker service with ignition:
+
+```yaml
+#workaround, see https://github.com/coreos/fedora-coreos-tracker/issues/229
+systemd:
+  units:
+    - name: docker.service
+      enabled: false
+      contents: |
+        [Unit]
+        Description=disable docker
+
+        [Service]
+
+        [Install]
+        WantedBy=multi-user.target
+```
+
 ## Network
 
 ### calico
@@ -55,14 +79,11 @@ Prepare ignition and serve via http (a.e. python -m http.server )
 ### create guest
 
 ```shell script
-machine_name=myfcos1
+fcos_version=31.20200223.3.0
-ignition_url=http://mywebserver/fcos.ign
-
-fcos_version=34.20210611.3.0
 kernel=https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/${fcos_version}/x86_64/fedora-coreos-${fcos_version}-live-kernel-x86_64
 initrd=https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/${fcos_version}/x86_64/fedora-coreos-${fcos_version}-live-initramfs.x86_64.img
-rootfs=https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/${fcos_version}/x86_64/fedora-coreos-${fcos_version}-live-rootfs.x86_64.img
+ignition_url=http://mywebserver/fcos.ign
-kernel_args="console=ttyS0 coreos.live.rootfs_url=${rootfs} coreos.inst.install_dev=/dev/sda coreos.inst.stream=stable coreos.inst.ignition_url=${ignition_url}"
+kernel_args="ip=dhcp rd.neednet=1 console=tty0 coreos.liveiso=/ console=ttyS0 coreos.inst.install_dev=/dev/sda coreos.inst.stream=stable coreos.inst.ignition_url=${ignition_url}"
 sudo virt-install --name ${machine_name} --ram 4048 --graphics=none --vcpus 2 --disk size=20 \
                 --network bridge=virbr0 \
                 --install kernel=${kernel},initrd=${initrd},kernel_args_overwrite=yes,kernel_args="${kernel_args}"

docs/getting-started.md

@@ -11,7 +11,7 @@ You can use an
 to create or modify an Ansible inventory. Currently, it is limited in
 functionality and is only used for configuring a basic Kubespray cluster inventory, but it does
 support creating inventory file for large clusters as well. It now supports
-separated ETCD and Kubernetes control plane roles from node role if the size exceeds a
+separated ETCD and Kubernetes master roles from node role if the size exceeds a
 certain threshold. Run `python3 contrib/inventory_builder/inventory.py help` for more information.
 
 Example inventory generator usage:
@@ -40,7 +40,7 @@ See more details in the [ansible guide](/docs/ansible.md).
 
 ### Adding nodes
 
-You may want to add worker, control plane or etcd nodes to your existing cluster. This can be done by re-running the `cluster.yml` playbook, or you can target the bare minimum needed to get kubelet installed on the worker and talking to your control planes. This is especially helpful when doing something like autoscaling your clusters.
+You may want to add worker, master or etcd nodes to your existing cluster. This can be done by re-running the `cluster.yml` playbook, or you can target the bare minimum needed to get kubelet installed on the worker and talking to your masters. This is especially helpful when doing something like autoscaling your clusters.
 
 - Add the new worker node to your inventory in the appropriate group (or utilize a [dynamic inventory](https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html)).
 - Run the ansible-playbook command, substituting `cluster.yml` for `scale.yml`:
@@ -52,7 +52,7 @@ ansible-playbook -i inventory/mycluster/hosts.yml scale.yml -b -v \
 
 ### Remove nodes
 
-You may want to remove **control plane**, **worker**, or **etcd** nodes from your
+You may want to remove **master**, **worker**, or **etcd** nodes from your
 existing cluster. This can be done by re-running the `remove-node.yml`
 playbook. First, all specified nodes will be drained, then stop some
 kubernetes services and delete some certificates,
@@ -108,11 +108,11 @@ Accessing through Ingress is highly recommended. For proxy access, please note t
 
 For token authentication, guide to create Service Account is provided in [dashboard sample user](https://github.com/kubernetes/dashboard/blob/master/docs/user/access-control/creating-sample-user.md) doc. Still take care of default namespace.
 
-Access can also by achieved via ssh tunnel on a control plane :
+Access can also by achieved via ssh tunnel on a master :
 
 ```bash
-# localhost:8081 will be sent to control-plane-1's own localhost:8081
+# localhost:8081 will be sent to master-1's own localhost:8081
-ssh -L8001:localhost:8001 user@control-plane-1
+ssh -L8001:localhost:8001 user@master-1
 sudo -i
 kubectl proxy
 ```

docs/gvisor.md

@@ -1,16 +0,0 @@
-# gVisor
-
-[gVisor](https://gvisor.dev/docs/) is an application kernel, written in Go, that implements a substantial portion of the Linux system call interface. It provides an additional layer of isolation between running applications and the host operating system.
-
-gVisor includes an Open Container Initiative (OCI) runtime called runsc that makes it easy to work with existing container tooling. The runsc runtime integrates with Docker and Kubernetes, making it simple to run sandboxed containers.
-
-## Usage
-
-To enable gVisor you should be using a container manager that is compatible with selecting the [RuntimeClass](https://kubernetes.io/docs/concepts/containers/runtime-class/) such as `containerd`.
-
-Containerd support:
-
-```yaml
-container_manager: containerd
-gvisor_enabled: true
-```

docs/integration.md

@@ -95,7 +95,7 @@ If you made useful changes or fixed a bug in existent kubespray repo, use this f
 
 3. Setup desired user.name and user.email for submodule.
 If kubespray is only one submodule in your repo you could use something like:
-```git submodule foreach --recursive 'git config user.name "First Last" && git config user.email "your-email-address@used.for.cncf"'```
+```git submodule foreach --recursive 'git config user.name "First Last" && git config user.email "your-email-addres@used.for.cncf"'```
 
 4. Sync with upstream master:
 

docs/kubernetes-reliability.md

@@ -21,7 +21,7 @@ By default the normal behavior looks like:
 
 > Kubernetes controller manager and Kubelet work asynchronously. It means that
 > the delay may include any network latency, API Server latency, etcd latency,
-> latency caused by load on one's control plane nodes and so on. So if
+> latency caused by load on one's master nodes and so on. So if
 > `--node-status-update-frequency` is set to 5s in reality it may appear in
 > etcd in 6-7 seconds or even longer when etcd cannot commit data to quorum
 > nodes.
@@ -56,7 +56,7 @@ services so pods from failed node won't be accessible anymore.
 
 ## Fast Update and Fast Reaction
 
-If `--node-status-update-frequency` is set to **4s** (10s is default).
+If `-–node-status-update-frequency` is set to **4s** (10s is default).
 `--node-monitor-period` to **2s** (5s is default).
 `--node-monitor-grace-period` to **20s** (40s is default).
 `--default-not-ready-toleration-seconds` and ``--default-unreachable-toleration-seconds`` are set to **30**
@@ -78,7 +78,7 @@ minute which may require large etcd containers or even dedicated nodes for etcd.
 
 ## Medium Update and Average Reaction
 
-Let's set `--node-status-update-frequency` to **20s**
+Let's set `-–node-status-update-frequency` to **20s**
 `--node-monitor-grace-period` to **2m** and `--default-not-ready-toleration-seconds` and
 ``--default-unreachable-toleration-seconds`` to **60**.
 In that case, Kubelet will try to update status every 20s. So, it will be 6 * 5
@@ -94,7 +94,7 @@ etcd updates per minute.
 
 ## Low Update and Slow reaction
 
-Let's set `--node-status-update-frequency` to **1m**.
+Let's set `-–node-status-update-frequency` to **1m**.
 `--node-monitor-grace-period` will set to **5m** and `--default-not-ready-toleration-seconds` and
 ``--default-unreachable-toleration-seconds`` to **60**. In this scenario, every kubelet will try to update the status
 every minute. There will be 5 * 5 = 25 attempts before unhealthy status. After 5m,

docs/large-deployments.md

@@ -11,8 +11,8 @@ For a large scaled deployments, consider the following configuration changes:
 * Override the ``download_run_once: true`` and/or ``download_localhost: true``.
   See download modes for details.
 
-* Adjust the `retry_stagger` global var as appropriate. It should provide same
+* Adjust the `retry_stagger` global var as appropriate. It should provide sane
-  load on a delegate (the first K8s control plane node) then retrying failed
+  load on a delegate (the first K8s master node) then retrying failed
   push or download operations.
 
 * Tune parameters for DNS related applications

docs/metallb.md

@@ -4,14 +4,6 @@ MetalLB hooks into your Kubernetes cluster, and provides a network load-balancer
 It allows you to create Kubernetes services of type "LoadBalancer" in clusters that don't run on a cloud provider, and thus cannot simply hook into 3rd party products to provide load-balancers.
 The default operationg mode of MetalLB is in ["Layer2"](https://metallb.universe.tf/concepts/layer2/) but it can also operate in ["BGP"](https://metallb.universe.tf/concepts/bgp/) mode.
 
-## Prerequisites
-
-You have to configure arp_ignore and arp_announce to avoid answering ARP queries from kube-ipvs0 interface for MetalLB to work.
-
-```yaml
-kube_proxy_strict_arp: true
-```
-
 ## Install
 
 You have to explicitly enable the MetalLB extension and set an IP address range from which to allocate LoadBalancer IPs.

docs/nodes.md

@@ -6,9 +6,9 @@ Modified from [comments in #3471](https://github.com/kubernetes-sigs/kubespray/i
 
 Currently you can't remove the first node in your kube_control_plane and etcd-master list. If you still want to remove this node you have to:
 
-### 1) Change order of current control planes
+### 1) Change order of current masters
 
-Modify the order of your control plane list by pushing your first entry to any other position. E.g. if you want to remove `node-1` of the following example:
+Modify the order of your master list by pushing your first entry to any other position. E.g. if you want to remove `node-1` of the following example:
 
 ```yaml
   children:
@@ -70,20 +70,20 @@ Before using `--limit` run playbook `facts.yml` without the limit to refresh fac
 
 With the old node still in the inventory, run `remove-node.yml`. You need to pass `-e node=NODE_NAME` to the playbook to limit the execution to the node being removed.
   
-If the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars: `-e node=NODE_NAME -e reset_nodes=false -e allow_ungraceful_removal=true`.
+If the node you want to remove is not online, you should add `reset_nodes=false` to your extra-vars: `-e node=NODE_NAME -e reset_nodes=false`.
-Use this flag even when you remove other types of nodes like a control plane or etcd nodes.
+Use this flag even when you remove other types of nodes like a master or etcd nodes.
 
-### 4) Remove the node from the inventory
+### 5) Remove the node from the inventory
 
 That's it.
 
-## Adding/replacing a control plane node
+## Adding/replacing a master node
 
 ### 1) Run `cluster.yml`
 
 Append the new host to the inventory and run `cluster.yml`. You can NOT use `scale.yml` for that.
 
-### 2) Restart kube-system/nginx-proxy
+### 3) Restart kube-system/nginx-proxy
 
 In all hosts, restart nginx-proxy pod. This pod is a local proxy for the apiserver. Kubespray will update its static config, but it needs to be restarted in order to reload.
 
@@ -92,49 +92,10 @@ In all hosts, restart nginx-proxy pod. This pod is a local proxy for the apiserv
 docker ps | grep k8s_nginx-proxy_nginx-proxy | awk '{print $1}' | xargs docker restart
 ```
 
-### 3) Remove old control plane nodes
+### 4) Remove old master nodes
 
 With the old node still in the inventory, run `remove-node.yml`. You need to pass `-e node=NODE_NAME` to the playbook to limit the execution to the node being removed.
-If the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars.
+If the node you want to remove is not online, you should add `reset_nodes=false` to your extra-vars.
-
-## Replacing a first control plane node
-
-### 1) Change control plane nodes order in inventory
-
-from
-
-```ini
-[kube_control_plane]
- node-1
- node-2
- node-3
-```
-
-to
-
-```ini
-[kube_control_plane]
- node-2
- node-3
- node-1
-```
-
-### 2) Remove old first control plane node from cluster
-
-With the old node still in the inventory, run `remove-node.yml`. You need to pass `-e node=node-1` to the playbook to limit the execution to the node being removed.
-If the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars.
-
-### 3) Edit cluster-info configmap in kube-system namespace
-
-`kubectl  edit cm -n kube-public cluster-info`
-
-Change ip of old kube_control_plane node with ip of live kube_control_plane node (`server` field). Also, update `certificate-authority-data` field if you changed certs.
-
-### 4) Add new control plane node
-
-Update inventory (if needed)
-
-Run `cluster.yml` with `--limit=kube_control_plane`
 
 ## Adding an etcd node
 
@@ -143,7 +104,7 @@ You need to make sure there are always an odd number of etcd nodes in the cluste
 ### 1) Add the new node running cluster.yml
 
 Update the inventory and run `cluster.yml` passing `--limit=etcd,kube_control_plane -e ignore_assert_errors=yes`.
-If the node you want to add as an etcd node is already a worker or control plane node in your cluster, you have to remove him first using `remove-node.yml`.
+If the node you want to add as an etcd node is already a worker or master node in your cluster, you have to remove him first using `remove-node.yml`.
 
 Run `upgrade-cluster.yml` also passing `--limit=etcd,kube_control_plane -e ignore_assert_errors=yes`. This is necessary to update all etcd configuration in the cluster.  
 
@@ -151,19 +112,15 @@ At this point, you will have an even number of nodes.
 Everything should still be working, and you should only have problems if the cluster decides to elect a new etcd leader before you remove a node.
 Even so, running applications should continue to be available.
 
-If you add multiple etcd nodes with one run, you might want to append `-e etcd_retries=10` to increase the amount of retries between each etcd node join.
+If you add multiple ectd nodes with one run, you might want to append `-e etcd_retries=10` to increase the amount of retries between each ectd node join.
 Otherwise the etcd cluster might still be processing the first join and fail on subsequent nodes. `etcd_retries=10` might work to join 3 new nodes.
 
-### 2) Add the new node to apiserver config
-
-In every control plane node, edit `/etc/kubernetes/manifests/kube-apiserver.yaml`. Make sure the new etcd nodes are present in the apiserver command line parameter `--etcd-servers=...`.
-
 ## Removing an etcd node
 
 ### 1) Remove an old etcd node
 
 With the node still in the inventory, run `remove-node.yml` passing `-e node=NODE_NAME` as the name of the node that should be removed.
-If the node you want to remove is not online, you should add `reset_nodes=false` and `allow_ungraceful_removal=true` to your extra-vars.
+If the node you want to remove is not online, you should add `reset_nodes=false` to your extra-vars.
 
 ### 2) Make sure only remaining nodes are in your inventory
 
@@ -173,10 +130,6 @@ Remove `NODE_NAME` from your inventory file.
 
 Run `cluster.yml` to regenerate the configuration files on all remaining nodes.
 
-### 4) Remove the old etcd node from apiserver config
+### 4) Shutdown the old instance
-
-In every control plane node, edit `/etc/kubernetes/manifests/kube-apiserver.yaml`. Make sure only active etcd nodes are still present in the apiserver command line parameter `--etcd-servers=...`.
-
-### 5) Shutdown the old instance
 
 That's it.

docs/openstack.md

@@ -12,7 +12,6 @@ Kubespray has been tested on a number of OpenStack Public Clouds including (in a
 - [ELASTX](https://elastx.se/)
 - [EnterCloudSuite](https://www.entercloudsuite.com/)
 - [FugaCloud](https://fuga.cloud/)
-- [Infomaniak](https://infomaniak.com)
 - [Open Telekom Cloud](https://cloud.telekom.de/) : requires to set the variable `wait_for_floatingip = "true"` in your cluster.tfvars
 - [OVHcloud](https://www.ovhcloud.com/)
 - [Rackspace](https://www.rackspace.com/)
@@ -105,12 +104,6 @@ The new cloud provider is configured to have Octavia by default in Kubespray.
   cinder_topology: true
   ```
 
-- Enabling `cinder_csi_ignore_volume_az: true`, ignores volumeAZ and schedules on any of the available node AZ.
-
-  ```yaml
-  cinder_csi_ignore_volume_az: true
-  ```
-
 - If you are using OpenStack loadbalancer(s) replace the `openstack_lbaas_subnet_id` with the new `external_openstack_lbaas_subnet_id`. **Note** The new cloud provider is using Octavia instead of Neutron LBaaS by default!
 - Enable 3 feature gates to allow migration of all volumes and storage classes (if you have any feature gates already set just add the 3 listed below):
 

docs/packet.md

@@ -1,15 +1,15 @@
-# Equinix Metal
+# Packet
 
-Kubespray provides support for bare metal deployments using the [Equinix Metal](http://metal.equinix.com).
+Kubespray provides support for bare metal deployments using the [Packet bare metal cloud](http://www.packet.com).
 Deploying upon bare metal allows Kubernetes to run at locations where an existing public or private cloud might not exist such
-as cell tower, edge collocated installations. The deployment mechanism used by Kubespray for Equinix Metal is similar to that used for
+as cell tower, edge collocated installations. The deployment mechanism used by Kubespray for Packet is similar to that used for
-AWS and OpenStack clouds (notably using Terraform to deploy the infrastructure). Terraform uses the Equinix Metal provider plugin
+AWS and OpenStack clouds (notably using Terraform to deploy the infrastructure). Terraform uses the Packet provider plugin
 to provision and configure hosts which are then used by the Kubespray Ansible playbooks. The Ansible inventory is generated
 dynamically from the Terraform state file.
 
 ## Local Host Configuration
 
-To perform this installation, you will need a localhost to run Terraform/Ansible (laptop, VM, etc) and an account with Equinix Metal.
+To perform this installation, you will need a localhost to run Terraform/Ansible (laptop, VM, etc) and an account with Packet.
 In this example, we're using an m1.large CentOS 7 OpenStack VM as the localhost to kickoff the Kubernetes installation.
 You'll need Ansible, Git, and PIP.
 
@@ -64,7 +64,7 @@ ln -s ../../contrib/terraform/packet/hosts
 ```
 
 Details about the cluster, such as the name, as well as the authentication tokens and project ID
-for Equinix Metal need to be defined. To find these values see [Equinix Metal API Accounts](https://metal.equinix.com/developers/docs/accounts/).
+for Packet need to be defined. To find these values see [Packet API Integration](https://support.packet.com/kb/articles/api-integrations)
 
 ```bash
 vi cluster.tfvars

docs/proxy.md

@@ -18,6 +18,6 @@ If you set http and https proxy, all nodes and loadbalancer will be excluded fro
 ## Exclude workers from no_proxy
 
 Since workers are included in the no_proxy variable, by default, docker engine will be restarted on all nodes (all
-pods will restart) when adding or removing workers.  To override this behaviour by only including control plane nodes in the
+pods will restart) when adding or removing workers.  To override this behaviour by only including master nodes in the
 no_proxy variable, set:
 `no_proxy_exclude_workers: true`

docs/recover-control-plane.md

@@ -20,7 +20,7 @@ __Note that you need at least one functional node to be able to recover using th
 ## Runbook
 
 * Move any broken etcd nodes into the "broken\_etcd" group, make sure the "etcd\_member\_name" variable is set.
-* Move any broken control plane nodes into the "broken\_kube\_control\_plane" group.
+* Move any broken master nodes into the "broken\_kube\_control\_plane" group.
 
 Then run the playbook with ```--limit etcd,kube_control_plane``` and increase the number of ETCD retries by setting ```-e etcd_retries=10``` or something even larger. The amount of retries required is difficult to predict.
 

docs/roadmap.md

@@ -14,7 +14,7 @@
   - [ ] GCE
   - [x] AWS (contrib/terraform/aws)
   - [x] OpenStack (contrib/terraform/openstack)
-  - [x] Equinix Metal
+  - [x] Packet
   - [ ] Digital Ocean
   - [ ] Azure
 - [ ] On AWS autoscaling, multi AZ
@@ -28,7 +28,7 @@
 - [x] Run kubernetes e2e tests
 - [ ] Test idempotency on single OS but for all network plugins/container engines
 - [ ] single test on AWS per day
-- [ ] test scale up cluster:  +1 etcd, +1 control plane, +1 node
+- [ ] test scale up cluster:  +1 etcd, +1 master, +1 node
 - [x] Reorganize CI test vars into group var files
 
 ## Lifecycle

docs/test_cases.md

@@ -8,7 +8,7 @@ and the `etcd` group merged with the `kube_control_plane`.
 `separate` layout is when there is only node of each type, which includes
  a kube_control_plane, kube_node, and etcd cluster member.
 
-`ha` layout consists of two etcd nodes, two control planes and a single worker node,
+`ha` layout consists of two etcd nodes, two masters and a single worker node,
 with role intersection.
 
 `scale` layout can be combined with above layouts (`ha-scale`, `separate-scale`). It includes 200 fake hosts

docs/upgrades.md

@@ -62,29 +62,6 @@ If you want to manually control the upgrade procedure, you can use the variables
 `upgrade_node_confirm: true` - waiting to confirmation to upgrade next node
 `upgrade_node_pause_seconds: 60` - pause 60 seconds before upgrade next node
 
-## Node-based upgrade
-
-If you don't want to upgrade all nodes in one run, you can use `--limit` [patterns](https://docs.ansible.com/ansible/latest/user_guide/intro_patterns.html#patterns-and-ansible-playbook-flags).
-
-Before using `--limit` run playbook `facts.yml` without the limit to refresh facts cache for all nodes:
-
-```ShellSession
-ansible-playbook facts.yml -b -i inventory/sample/hosts.ini
-```
-
-After this upgrade control plane and etcd groups [#5147](https://github.com/kubernetes-sigs/kubespray/issues/5147):
-
-```ShellSession
-ansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=v1.20.7 --limit "kube_control_plane:etcd"
-```
-
-Now you can upgrade other nodes in any order and quantity:
-
-```ShellSession
-ansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=v1.20.7 --limit "node4:node6:node7:node12"
-ansible-playbook upgrade-cluster.yml -b -i inventory/sample/hosts.ini -e kube_version=v1.20.7 --limit "node5*"
-```
-
 ## Multiple upgrades
 
 :warning: [Do not skip releases when upgrading--upgrade by one tag at a time.](https://github.com/kubernetes-sigs/kubespray/issues/3849#issuecomment-451386515) :warning:
@@ -336,12 +313,6 @@ Upgrade etcd:
 ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=etcd
 ```
 
-Upgrade etcd without rotating etcd certs:
-
-```ShellSession
-ansible-playbook -b -i inventory/sample/hosts.ini cluster.yml --tags=etcd --limit=etcd --skip-tags=etcd-secrets
-```
-
 Upgrade kubelet:
 
 ```ShellSession

docs/vars.md

@@ -26,7 +26,7 @@ Some variables of note include:
 * *kube_version* - Specify a given Kubernetes version
 * *searchdomains* - Array of DNS domains to search when looking up hostnames
 * *nameservers* - Array of nameservers to use for DNS lookup
-* *preinstall_selinux_state* - Set selinux state, permitted values are permissive, enforcing and disabled.
+* *preinstall_selinux_state* - Set selinux state, permitted values are permissive and disabled.
 
 ## Addressing variables
 
@@ -136,12 +136,14 @@ Stack](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/dns-stack.m
   [kubelet-rubber-stamp](https://github.com/kontena/kubelet-rubber-stamp).
 * *node_labels* - Labels applied to nodes via kubelet --node-labels parameter.
   For example, labels can be set in the inventory as variables or more widely in group_vars.
-  *node_labels* can only be defined as a dict:
+  *node_labels* can be defined either as a dict or a comma-separated labels string:
 
 ```yml
 node_labels:
   label1_name: label1_value
   label2_name: label2_value
+
+node_labels: "label1_name=label1_value,label2_name=label2_value"
 ```
 
 * *node_taints* - Taints applied to nodes via kubelet --register-with-taints parameter.
@@ -178,7 +180,7 @@ node_taints:
 For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments.
 
 Extra flags for the kubelet can be specified using these variables,
-in the form of dicts of key-value pairs of configuration parameters that will be inserted into the kubelet YAML config file. The `kubelet_node_config_extra_args` apply kubelet settings only to nodes and not control planes. Example:
+in the form of dicts of key-value pairs of configuration parameters that will be inserted into the kubelet YAML config file. The `kubelet_node_config_extra_args` apply kubelet settings only to nodes and not masters. Example:
 
 ```yml
 kubelet_config_extra_args:
@@ -200,7 +202,7 @@ Previously, the same parameters could be passed as flags to kubelet binary with
 * *kubelet_custom_flags*
 * *kubelet_node_custom_flags*
 
-The `kubelet_node_custom_flags` apply kubelet settings only to nodes and not control planes. Example:
+The `kubelet_node_custom_flags` apply kubelet settings only to nodes and not masters. Example:
 
 ```yml
 kubelet_custom_flags:

docs/vsphere-csi.md

@@ -19,8 +19,8 @@ You need to source the vSphere credentials you use to deploy your machines that
 | external_vsphere_vcenter_ip                 | TRUE     | string  |                            |                           | IP/URL of the vCenter                                                                                               |
 | external_vsphere_vcenter_port               | TRUE     | string  |                            | "443"                     | Port of the vCenter API                                                                                             |
 | external_vsphere_insecure                   | TRUE     | string  | "true", "false"            | "true"                    | set to "true" if the host above uses a self-signed cert                                                             |
-| external_vsphere_user                       | TRUE     | string  |                            |                           | User name for vCenter with required privileges (Can also be specified with the `VSPHERE_USER` environment variable) |
+| external_vsphere_user                       | TRUE     | string  |                            |                           | User name for vCenter with required privileges |
-| external_vsphere_password                   | TRUE     | string  |                            |                           | Password for vCenter (Can also be specified with the `VSPHERE_PASSWORD` environment variable)                       |
+| external_vsphere_password                   | TRUE     | string  |                            |                           | Password for vCenter                       |
 | external_vsphere_datacenter                 | TRUE     | string  |                            |                           | Datacenter name to use                                                                                              |
 | external_vsphere_kubernetes_cluster_id      | TRUE     | string  |                            | "kubernetes-cluster-id"   | Kubernetes cluster ID to use                                                                                        |
 | external_vsphere_version                    | TRUE     | string  |                            | "6.7u3"                   | Vmware Vsphere version where located all VMs                                                                        |
@@ -34,9 +34,6 @@ You need to source the vSphere credentials you use to deploy your machines that
 | vsphere_csi_node_driver_registrar_image_tag | TRUE     | string  |                            | "v1.1.0"                  | CSI node driver registrat image tag to use                                                                          |
 | vsphere_csi_driver_image_tag                | TRUE     | string  |                            | "v1.0.2"                  | CSI driver image tag to use                                                                                         |
 | vsphere_csi_resizer_tag                     | TRUE     | string  |                            | "v1.1.0"                  | CSI resizer image tag to use
-| vsphere_csi_aggressive_node_drain           | FALSE    | boolean |                            | false                     | Enable aggressive node drain strategy                                                                               |
-| vsphere_csi_aggressive_node_unreachable_timeout            | FALSE     | int  | 300   |                           | Timeout till node will be drained when it in an unreachable state                                                           |
-| vsphere_csi_aggressive_node_not_ready_timeout              | FALSE     | int  | 300   |                           | Timeout till node will be drained when it in not-ready state                                                                |
 
 ## Usage example
 

docs/vsphere.md

@@ -31,12 +31,12 @@ external_cloud_provider: "vsphere"
 Then, `inventory/sample/group_vars/vsphere.yml`, you need to declare your vCenter credentials and enable the vSphere CSI following the description below.
 
 | Variable                               | Required | Type    | Choices                    | Default | Comment                                                                   |
-|----------------------------------------|----------|---------|----------------------------|---------------------------|---------------------------------------------------------------------------------------------------------------------|
+|----------------------------------------|----------|---------|----------------------------|---------|---------------------------------------------------------------------------|
 | external_vsphere_vcenter_ip            | TRUE     | string  |                            |                           | IP/URL of the vCenter                                   |
 | external_vsphere_vcenter_port          | TRUE     | string  |                            | "443"                     | Port of the vCenter API                                 |
 | external_vsphere_insecure              | TRUE     | string  | "true", "false"            | "true"                    | set to "true" if the host above uses a self-signed cert |
-| external_vsphere_user                  | TRUE     | string  |                            |                           | User name for vCenter with required privileges (Can also be specified with the `VSPHERE_USER` environment variable) |
+| external_vsphere_user                  | TRUE     | string  |                            |                           | User name for vCenter with required privileges          |
-| external_vsphere_password              | TRUE     | string  |                            |                           | Password for vCenter (Can also be specified with the `VSPHERE_PASSWORD` environment variable)                       |
+| external_vsphere_password              | TRUE     | string  |                            |                           | Password for vCenter                                    |
 | external_vsphere_datacenter            | TRUE     | string  |                            |                           | Datacenter name to use                                  |
 | external_vsphere_kubernetes_cluster_id | TRUE     | string  |                            | "kubernetes-cluster-id"   | Kubernetes cluster ID to use                            |
 | vsphere_csi_enabled                    | TRUE     | boolean |                            | false                     | Enable vSphere CSI                                      |

facts.yml

@@ -2,7 +2,6 @@
 - name: Gather facts
   hosts: k8s_cluster:etcd:calico_rr
   gather_facts: False
-  tags: always
   tasks:
     - name: Gather minimal facts
       setup:

inventory/sample/group_vars/all/containerd.yml

@@ -32,8 +32,3 @@
 #   "docker.io": "https://registry-1.docker.io"
 
 # containerd_max_container_log_line_size: -1
-
-# containerd_registry_auth:
-#   - registry: 10.0.0.2:5000
-#     username: user
-#     password: pass

inventory/sample/group_vars/all/cri-o.yml

@@ -1,6 +0,0 @@
-# crio_insecure_registries:
-#   - 10.0.0.2:5000
-# crio_registry_auth:
-#   - registry: 10.0.0.2:5000
-#     username: user
-#     password: pass

inventory/sample/group_vars/all/offline.yml

@@ -13,7 +13,6 @@
 ## Container Registry overrides
 # kube_image_repo: "{{ registry_host }}"
 # gcr_image_repo: "{{ registry_host }}"
-# github_image_repo: "{{ registry_host }}"
 # docker_image_repo: "{{ registry_host }}"
 # quay_image_repo: "{{ registry_host }}"
 
@@ -45,10 +44,6 @@
 # [Optional] kata: only if you set kata_containers_enabled: true
 # kata_containers_download_url: "{{ files_repo }}/kata-containers/runtime/releases/download/{{ kata_containers_version }}/kata-static-{{ kata_containers_version }}-{{ ansible_architecture }}.tar.xz"
 
-# [Optional] cri-o: only if you set container_manager: crio
-# crio_download_base: "download.opensuse.org/repositories/devel:kubic:libcontainers:stable"
-# crio_download_crio: "http://{{ crio_download_base }}:/cri-o:/"
-
 ## CentOS/Redhat/AlmaLinux
 ### For EL7, base and extras repo must be available, for EL8, baseos and appstream
 ### By default we enable those repo automatically

inventory/sample/group_vars/all/vsphere.yml

@@ -2,8 +2,8 @@
 # external_vsphere_vcenter_ip: "myvcenter.domain.com"
 # external_vsphere_vcenter_port: "443"
 # external_vsphere_insecure: "true"
-# external_vsphere_user: "administrator@vsphere.local" # Can also be set via the `VSPHERE_USER` environment variable
+# external_vsphere_user: "administrator@vsphere.local"
-# external_vsphere_password: "K8s_admin" # Can also be set via the `VSPHERE_PASSWORD` environment variable
+# external_vsphere_password: "K8s_admin"
 # external_vsphere_datacenter: "DATACENTER_name"
 # external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id"
 

inventory/sample/group_vars/k8s_cluster/addons.yml

@@ -14,9 +14,8 @@ registry_enabled: false
 
 # Metrics Server deployment
 metrics_server_enabled: false
-# metrics_server_resizer: false
 # metrics_server_kubelet_insecure_tls: true
-# metrics_server_metric_resolution: 15s
+# metrics_server_metric_resolution: 60s
 # metrics_server_kubelet_preferred_address_types: "InternalIP"
 
 # Rancher Local Path Provisioner
@@ -53,11 +52,6 @@ local_volume_provisioner_enabled: false
 #     volume_mode: Filesystem
 #     fs_type: ext4
 
-# CSI Volume Snapshot Controller deployment, set this to true if your CSI is able to manage snapshots
-# currently, setting cinder_csi_enabled=true would automatically enable the snapshot controller
-# Longhorn is an extenal CSI that would also require setting this to true but it is not included in kubespray
-# csi_snapshot_controller_enabled: false
-
 # CephFS provisioner deployment
 cephfs_provisioner_enabled: false
 # cephfs_provisioner_namespace: "cephfs-provisioner"
@@ -163,10 +157,11 @@ metallb_speaker_enabled: true
 #     operator: "Equal"
 #     value: ""
 #     effect: "NoSchedule"
-# metallb_version: v0.10.2
+# metallb_version: v0.9.6
 # metallb_protocol: "layer2"
 # metallb_port: "7472"
-# metallb_memberlist_port: "7946"
+# metallb_limits_cpu: "100m"
+# metallb_limits_mem: "100Mi"
 # metallb_additional_address_pools:
 #   kube_service_pool:
 #     ip_range:

inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml

@@ -2,7 +2,7 @@
 # Kubernetes configuration dirs and system namespace.
 # Those are where all the additional config stuff goes
 # the kubernetes normally puts in /srv/kubernetes.
-# This puts them in a same location and namespace.
+# This puts them in a sane location and namespace.
 # Editing those values will almost surely break something.
 kube_config_dir: /etc/kubernetes
 kube_script_dir: "{{ bin_dir }}/kubernetes-scripts"
@@ -17,7 +17,7 @@ kube_token_dir: "{{ kube_config_dir }}/tokens"
 kube_api_anonymous_auth: true
 
 ## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.21.6
+kube_version: v1.20.7
 
 # Where the binaries will be downloaded.
 # Note: ensure that you've enough disk space (about 1G)
@@ -149,10 +149,6 @@ kube_proxy_nodeport_addresses: >-
 ## Encrypting Secret Data at Rest (experimental)
 kube_encrypt_secret_data: false
 
-# Graceful Node Shutdown (Kubernetes >= 1.21.0), see https://kubernetes.io/blog/2021/04/21/graceful-node-shutdown-beta/
-# kubelet_shutdown_grace_period: 60s
-# kubelet_shutdown_grace_period_critical_pods: 20s
-
 # DNS configuration.
 # Kubernetes cluster name, also will be used as DNS domain
 cluster_name: cluster.local
@@ -166,7 +162,6 @@ dns_mode: coredns
 enable_nodelocaldns: true
 nodelocaldns_ip: 169.254.25.10
 nodelocaldns_health_port: 9254
-nodelocaldns_bind_metrics_host_ip: false
 # nodelocaldns_external_zones:
 # - zones:
 #   - example.com
@@ -211,9 +206,6 @@ k8s_image_pull_policy: IfNotPresent
 kubernetes_audit: false
 
 # dynamic kubelet configuration
-# Note: Feature DynamicKubeletConfig is deprecated in 1.22 and will not move to GA.
-# It is planned to be removed from Kubernetes in the version 1.23.
-# Please use alternative ways to update kubelet configuration.
 dynamic_kubelet_configuration: false
 
 # define kubelet config dir for dynamic kubelet

inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml

@@ -36,10 +36,10 @@
 # calico_mtu: 1500
 
 # Configure the MTU to use for workload interfaces and tunnels.
-# - If Wireguard is enabled, subtract 60 from your network MTU (i.e 1500-60=1440)
+# - If Wireguard is enabled, set to your network MTU - 60
-# - Otherwise, if VXLAN or BPF mode is enabled, subtract 50 from your network MTU (i.e. 1500-50=1450)
+# - Otherwise, if VXLAN or BPF mode is enabled, set to your network MTU - 50
-# - Otherwise, if IPIP is enabled, subtract 20 from your network MTU (i.e. 1500-20=1480)
+# - Otherwise, if IPIP is enabled, set to your network MTU - 20
-# - Otherwise, if not using any encapsulation, set to your network MTU (i.e. 1500)
+# - Otherwise, if not using any encapsulation, set to your network MTU.
 # calico_veth_mtu: 1440
 
 # Advertise Cluster IPs
@@ -100,10 +100,3 @@
 # If you want use the default route interface when you use multiple interface with dynamique route (iproute2)
 # see https://docs.projectcalico.org/reference/node/configuration : FELIX_DEVICEROUTESOURCEADDRESS
 # calico_use_default_route_src_ipaddr: false
-
-# Enable calico traffic encryption with wireguard
-# calico_wireguard_enabled: false
-
-# Under certain situations liveness and readiness probes may need tunning
-# calico_node_livenessprobe_timeout: 10
-# calico_node_readinessprobe_timeout: 10

legacy_groups.yml

@@ -4,7 +4,6 @@
 - name: Add kube-master nodes to kube_control_plane
   hosts: kube-master
   gather_facts: false
-  tags: always
   tasks:
     - name: add nodes to kube_control_plane group
       group_by:
@@ -13,7 +12,6 @@
 - name: Add kube-node nodes to kube_node
   hosts: kube-node
   gather_facts: false
-  tags: always
   tasks:
     - name: add nodes to kube_node group
       group_by:
@@ -22,7 +20,6 @@
 - name: Add k8s-cluster nodes to k8s_cluster
   hosts: k8s-cluster
   gather_facts: false
-  tags: always
   tasks:
     - name: add nodes to k8s_cluster group
       group_by:
@@ -31,7 +28,6 @@
 - name: Add calico-rr nodes to calico_rr
   hosts: calico-rr
   gather_facts: false
-  tags: always
   tasks:
     - name: add nodes to calico_rr group
       group_by:
@@ -40,7 +36,6 @@
 - name: Add no-floating nodes to no_floating
   hosts: no-floating
   gather_facts: false
-  tags: always
   tasks:
     - name: add nodes to no-floating group
       group_by:

mitogen.yml

@@ -5,7 +5,7 @@
 - hosts: localhost
   strategy: linear
   vars:
-    mitogen_version: 0.3.0rc1
+    mitogen_version: 0.2.9
     mitogen_url: https://github.com/dw/mitogen/archive/v{{ mitogen_version }}.tar.gz
     ansible_connection: local
   tasks:
@@ -13,7 +13,6 @@
       file:
         path: "{{ item }}"
         state: directory
-        mode: 0755
       become: false
       loop:
         - "{{ playbook_dir }}/plugins/mitogen"
@@ -41,4 +40,3 @@
         section: defaults
         option: strategy
         value: mitogen_linear
-        mode: 0644

recover-control-plane.yml

@@ -12,13 +12,13 @@
     - { role: kubespray-defaults}
     - { role: bastion-ssh-config, tags: ["localhost", "bastion"]}
 
-- hosts: etcd[0]
+- hosts: "{{ groups['etcd'] | first }}"
   environment: "{{ proxy_disable_env }}"
   roles:
     - { role: kubespray-defaults}
     - { role: recover_control_plane/etcd }
 
-- hosts: kube_control_plane[0]
+- hosts: "{{ groups['kube_control_plane'] | first }}"
   environment: "{{ proxy_disable_env }}"
   roles:
     - { role: kubespray-defaults}
@@ -26,7 +26,7 @@
 
 - include: cluster.yml
 
-- hosts: kube_control_plane
+- hosts: "{{ groups['kube_control_plane'] }}"
   environment: "{{ proxy_disable_env }}"
   roles:
     - { role: kubespray-defaults}

remove-node.yml

@@ -7,25 +7,25 @@
 
 - hosts: "{{ node | default('etcd:k8s_cluster:calico_rr') }}"
   gather_facts: no
-  tasks:
+  environment: "{{ proxy_disable_env }}"
-    - name: Confirm Execution
+  vars_prompt:
-      pause:
+    name: "delete_nodes_confirmation"
     prompt: "Are you sure you want to delete nodes state? Type 'yes' to delete nodes."
-      register: pause_result
+    default: "no"
-      run_once: True
+    private: no
-      when:
-        - not (skip_confirmation | default(false) | bool)
 
-    - name: Fail if user does not confirm deletion
+  pre_tasks:
+    - name: check confirmation
       fail:
         msg: "Delete nodes confirmation failed"
-      when: pause_result.user_input | default('yes') != 'yes'
+      when: delete_nodes_confirmation != "yes"
 
 - hosts: kube_control_plane[0]
   gather_facts: no
   environment: "{{ proxy_disable_env }}"
   roles:
     - { role: kubespray-defaults }
+    - { role: bootstrap-os, tags: bootstrap-os }
     - { role: remove-node/pre-remove, tags: pre-remove }
 
 - hosts: "{{ node | default('kube_node') }}"
@@ -33,6 +33,7 @@
   environment: "{{ proxy_disable_env }}"
   roles:
     - { role: kubespray-defaults, when: reset_nodes|default(True)|bool }
+    - { role: bootstrap-os, tags: bootstrap-os, when: reset_nodes|default(True)|bool }
     - { role: remove-node/remove-etcd-node }
     - { role: reset, tags: reset, when: reset_nodes|default(True)|bool }
 
@@ -42,4 +43,5 @@
   environment: "{{ proxy_disable_env }}"
   roles:
     - { role: kubespray-defaults, when: reset_nodes|default(True)|bool }
+    - { role: bootstrap-os, tags: bootstrap-os, when: reset_nodes|default(True)|bool }
     - { role: remove-node/post-remove, tags: post-remove }