Enable kubeadm etcd mode (#4818)

* Enable kubeadm etcd mode

Uses cert commands from kubeadm experimental control plane to
enable non-master nodes to obtain etcd certs.

Related story: PROD-29434

Change-Id: Idafa1d223e5c6ceadf819b6f9c06adf4c4f74178

* Add validation checks and exclude calico kdd mode

Change-Id: Ic234f5e71261d33191376e70d438f9f6d35f358c

* Move etcd mode test to ubuntu flannel HA job

Change-Id: I9af6fd80a1bbb1692ab10d6da095eb368f6bc732

* rename etcd_mode to etcd_kubeadm_enabled

Change-Id: Ib196d6c8a52f48cae370b026f7687ff9ca69c172

roles/kubernetes/master/defaults/main/etcd.yml

@@ -0,0 +1,32 @@
+---
+# Note: This does not set up DNS entries. It simply adds the following DNS
+# entries to the certificate
+etcd_cert_alt_names:
+  - "etcd.kube-system.svc.{{ dns_domain }}"
+  - "etcd.kube-system.svc"
+  - "etcd.kube-system"
+  - "etcd"
+etcd_cert_alt_ips: []
+
+etcd_heartbeat_interval: "250"
+etcd_election_timeout: "5000"
+
+# etcd_snapshot_count: "10000"
+
+# Parameters for ionice
+# -c takes an integer between 0 and 3 or one of the strings none, realtime, best-effort or idle.
+# -n takes an integer between 0 (highest priority) and 7 (lowest priority)
+# etcd_ionice: "-c2 -n0"
+
+etcd_metrics: "basic"
+
+## A dictionary of extra environment variables to add to etcd.env, formatted like:
+##  etcd_extra_vars:
+##    var1: "value1"
+##    var2: "value2"
+## Note this is different from the etcd role with ETCD_ prfexi, caps, and underscores
+etcd_extra_vars: {}
+
+# etcd_quota_backend_bytes: "2G"
+
+etcd_compaction_retention: "8"

roles/kubernetes/master/defaults/main/main.yml

@@ -2,6 +2,12 @@
 # disable upgrade cluster
 upgrade_cluster_setup: false
 
+# Enable kubeadm experimental control plane
+kubeadm_control_plane: false
+
+# Experimental kubeadm etcd deployment mode. Available only for new deployment
+etcd_kubeadm_enabled: false
+
 # An experimental dev/test only dynamic volumes provisioner,
 # for PetSets. Works for kube>=v1.3 only.
 kube_hostpath_dynamic_provisioner: "false"

roles/kubernetes/master/tasks/kubeadm-etcd.yml

@@ -0,0 +1,18 @@
+---
+- name: Calculate etcd cert serial
+  command: "openssl x509 -in {{ kube_cert_dir }}/apiserver-etcd-client.crt -noout -serial"
+  register: "etcd_client_cert_serial_result"
+  changed_when: false
+  tags:
+    - network
+
+- name: Set etcd_client_cert_serial
+  set_fact:
+    etcd_client_cert_serial: "{{ etcd_client_cert_serial_result.stdout.split('=')[1] }}"
+  tags:
+    - network
+
+- name: Ensure etcdctl binary is installed
+  include_tasks: "{{ role_path }}/../../etcd/tasks/install_host.yml"
+  vars:
+    etcd_cluster_setup: true

roles/kubernetes/master/tasks/kubeadm-secondary-experimental.yml

@@ -43,6 +43,10 @@
     kubeadm_certificate_key: "{{ hostvars[groups['kube-master'][0]]['kubeadm_upload_cert'].stdout_lines[-1] | trim }}"
   when: kubeadm_certificate_key is undefined
 
+- name: check already run
+  debug:
+    msg: "{{ kubeadm_already_run.stat.exists }}"
+
 - name: Joining control plane node to the cluster.
   command: >-
     {{ bin_dir }}/kubeadm join
@@ -52,9 +56,11 @@
     --certificate-key={{ kubeadm_certificate_key }}
     {% endif %}
   register: kubeadm_join_control_plane
+  retries: 3
+  until: kubeadm_join_control_plane is succeeded
   when:
     - inventory_hostname != groups['kube-master']|first
-    - not kubeadm_already_run.stat.exists
+    - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists
   environment:
     PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
 

roles/kubernetes/master/tasks/main.yml

@@ -75,3 +75,7 @@
 
 - name: Include kubeadm setup
   import_tasks: kubeadm-setup.yml
+
+- name: Include kubeadm etcd extra tasks
+  include_tasks: kubeadm-etcd.yml
+  when: etcd_kubeadm_enabled

roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2

@@ -24,6 +24,7 @@ apiVersion: kubeadm.k8s.io/v1beta1
 kind: ClusterConfiguration
 clusterName: {{ cluster_name }}
 etcd:
+{% if not etcd_kubeadm_enabled %}
   external:
       endpoints:
 {% for endpoint in etcd_access_addresses.split(',') %}
@@ -32,6 +33,46 @@ etcd:
       caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }}
       certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }}
       keyFile: {{ etcd_cert_dir }}/{{ kube_etcd_key_file }}
+{% elif etcd_kubeadm_enabled %}
+  local:
+    imageRepository: "{{ etcd_image_repo | regex_replace("/etcd$","") }}"
+    imageTag: "{{ etcd_image_tag }}"
+    dataDir: "/var/lib/etcd"
+    extraArgs:
+      metrics: {{ etcd_metrics }}
+      election-timeout: "{{ etcd_election_timeout }}"
+      heartbeat-interval: "{{ etcd_heartbeat_interval }}"
+      auto-compaction-retention: "{{ etcd_compaction_retention }}"
+{% if etcd_snapshot_count is defined %}
+      snapshot-count: "{{ etcd_snapshot_count }}"
+{% endif %}
+{% if etcd_quota_backend_bytes is defined %}
+      quota-backend-bytes: "{{ etcd_quota_backend_bytes }}"
+{% endif %}
+{% if etcd_log_package_levels is defined %}
+      log-package_levels: "{{ etcd_log_package_levels }}"
+{% endif %}
+{% for key, value in etcd_extra_vars.items() %}
+      {{ key }}: "{{ value }}"
+{% endfor %}
+{% if host_architecture != "amd64" -%}
+      etcd-unsupported-arch: {{host_architecture}}
+{% endif %}
+    serverCertSANs:
+{% for san in etcd_cert_alt_names %}
+      - {{ san }}
+{% endfor %}
+{% for san in etcd_cert_alt_ips %}
+      - {{ san }}
+{% endfor %}
+    peerCertSANs:
+{% for san in etcd_cert_alt_names %}
+      - {{ san }}
+{% endfor %}
+{% for san in etcd_cert_alt_ips %}
+      - {{ san }}
+{% endfor %}
+{% endif %}
 networking:
   dnsDomain: {{ dns_domain }}
   serviceSubnet: {{ kube_service_addresses }}