Enable kubeadm etcd mode (#4818)

* Enable kubeadm etcd mode

Uses cert commands from kubeadm experimental control plane to
enable non-master nodes to obtain etcd certs.

Related story: PROD-29434

Change-Id: Idafa1d223e5c6ceadf819b6f9c06adf4c4f74178

* Add validation checks and exclude calico kdd mode

Change-Id: Ic234f5e71261d33191376e70d438f9f6d35f358c

* Move etcd mode test to ubuntu flannel HA job

Change-Id: I9af6fd80a1bbb1692ab10d6da095eb368f6bc732

* rename etcd_mode to etcd_kubeadm_enabled

Change-Id: Ib196d6c8a52f48cae370b026f7687ff9ca69c172

roles/kubernetes/kubeadm/defaults/main.yml

@@ -10,3 +10,9 @@ kube_override_hostname: >-
   {%- else -%}
   {{ inventory_hostname }}
   {%- endif -%}
+
+# Requests a fresh upload of certificates from first master
+kubeadm_etcd_refresh_cert_key: false
+
+# Experimental kubeadm etcd deployment mode. Available only for new deployment
+etcd_kubeadm_enabled: false

roles/kubernetes/kubeadm/tasks/kubeadm_etcd_node.yml

@@ -0,0 +1,74 @@
+---
+- name: Refresh certificates so they are fresh and not expired
+  command: >-
+    {{ bin_dir }}/kubeadm init phase
+    --config {{ kube_config_dir }}/kubeadm-config.yaml
+    upload-certs --experimental-upload-certs
+    {% if kubeadm_certificate_key is defined %}
+    --certificate-key={{ kubeadm_certificate_key }}
+    {% endif %}
+  register: kubeadm_upload_cert
+  delegate_to: "{{ groups['kube-master'][0] }}"
+  when: kubeadm_etcd_refresh_cert_key
+  run_once: yes
+
+- name: Parse certificate key if not set
+  set_fact:
+    kubeadm_certificate_key: "{{ hostvars[groups['kube-master'][0]]['kubeadm_upload_cert'].stdout_lines[-1] | trim }}"
+  when: kubeadm_certificate_key is undefined
+
+- name: Pull control plane certs down
+  shell: >-
+    {{ bin_dir }}/kubeadm join phase
+    control-plane-prepare download-certs
+    --certificate-key {{ kubeadm_certificate_key }}
+    --experimental-control-plane
+    --token {{ kubeadm_token }}
+    --discovery-token-unsafe-skip-ca-verification
+    {{ kubeadm_discovery_address }}
+    &&
+    {{ bin_dir }}/kubeadm join phase
+    control-plane-prepare certs
+    --experimental-control-plane
+    --token {{ kubeadm_token }}
+    --discovery-token-unsafe-skip-ca-verification
+    {{ kubeadm_discovery_address }}
+  args:
+    creates: "{{ kube_cert_dir }}/apiserver-etcd-client.key"
+
+- name: Delete unneeded certificates
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+    - "{{ kube_cert_dir }}/apiserver.crt"
+    - "{{ kube_cert_dir }}/apiserver.key"
+    - "{{ kube_cert_dir }}/ca.key"
+    - "{{ kube_cert_dir }}/etcd/ca.key"
+    - "{{ kube_cert_dir }}/etcd/healthcheck-client.crt"
+    - "{{ kube_cert_dir }}/etcd/healthcheck-client.key"
+    - "{{ kube_cert_dir }}/etcd/peer.crt"
+    - "{{ kube_cert_dir }}/etcd/peer.key"
+    - "{{ kube_cert_dir }}/etcd/server.crt"
+    - "{{ kube_cert_dir }}/etcd/server.key"
+    - "{{ kube_cert_dir }}/front-proxy-ca.crt"
+    - "{{ kube_cert_dir }}/front-proxy-ca.key"
+    - "{{ kube_cert_dir }}/front-proxy-client.crt"
+    - "{{ kube_cert_dir }}/front-proxy-client.key"
+    - "{{ kube_cert_dir }}/sa.key"
+    - "{{ kube_cert_dir }}/sa.pub"
+
+- name: Calculate etcd cert serial
+  command: "openssl x509 -in {{ kube_cert_dir }}/apiserver-etcd-client.crt -noout -serial"
+  register: "etcd_client_cert_serial_result"
+  changed_when: false
+  when:
+    - inventory_hostname in groups['k8s-cluster']|union(groups['calico-rr']|default([]))|unique|sort
+  tags:
+    - network
+
+- name: Set etcd_client_cert_serial
+  set_fact:
+    etcd_client_cert_serial: "{{ etcd_client_cert_serial_result.stdout.split('=')[1] }}"
+  tags:
+    - network

roles/kubernetes/kubeadm/tasks/main.yml

@@ -10,6 +10,7 @@
   tags:
     - facts
 
+
 - name: Check if kubelet.conf exists
   stat:
     path: "{{ kube_config_dir }}/kubelet.conf"
@@ -168,3 +169,12 @@
     - kubeadm_discovery_address != kube_apiserver_endpoint
   tags:
     - kube-proxy
+
+- name: Extract etcd certs from control plane if using etcd kubeadm mode
+  include_tasks: kubeadm_etcd_node.yml
+  when:
+    - etcd_kubeadm_enabled
+    - kubeadm_control_plane
+    - inventory_hostname not in groups['kube-master']
+    - kube_network_plugin in ["calico", "flannel", "canal", "cilium"]
+    - kube_network_plugin != "calico" or calico_datastore == "etcd"