Use K8s 1.14 and add kubeadm experimental control plane mode (#4514)

* Use K8s 1.14 and add kubeadm experimental control plane mode

This reverts commit d39c273d96.

* Cleanup kubeadm setup run on first master

* pin kubeadm_certificate_key in test

* Remove kubelet autolabel of kube-node, add symlink for pki dir

Change-Id: Id5e74dd667c60675dbfe4193b0bc9fb44380e1ca

roles/kubernetes/master/tasks/kubeadm-setup.yml

@@ -10,11 +10,10 @@
   import_tasks: kubeadm-migrate-certs.yml
   when: old_apiserver_cert.stat.exists
 
-- name: kubeadm | Check apiserver key
+- name: kubeadm | Check serviceaccount key
   stat:
-    path: "{{ kube_cert_dir }}/apiserver.key"
-  register: apiserver_key_before
-  delegate_to: "{{groups['kube-master']|first}}"
+    path: "{{ kube_cert_dir }}/sa.key"
+  register: sa_key_before
   run_once: true
 
 - name: kubeadm | Check if kubeadm has already run
@@ -62,10 +61,6 @@
     sans_address: "{{ groups['kube-master'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | list | select('defined') | list }}"
   tags: facts
 
-- name: kubeadm | Copy etcd cert dir under k8s cert dir
-  command: "cp -TR {{ etcd_cert_dir }} {{ kube_config_dir }}/ssl/etcd"
-  changed_when: false
-
 - name: Create audit-policy directory
   file:
     path: "{{ audit_policy_file | dirname }}"
@@ -94,7 +89,18 @@
     - kubeadm_already_run.stat.exists
 
 - name: kubeadm | Initialize first master
-  command: timeout -k 600s 600s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all
+  command: >-
+    timeout -k 600s 600s
+    {{ bin_dir }}/kubeadm init
+    --config={{ kube_config_dir }}/kubeadm-config.yaml
+    --ignore-preflight-errors=all
+    {% if kubeadm_version is version('v1.14.0', '>=') %}
+    --experimental-upload-certs
+    {% endif %}
+    --skip-phases=addon/coredns
+    {% if kubeadm_certificate_key is defined %}
+    --certificate-key={{ kubeadm_certificate_key }}
+    {% endif %}
   register: kubeadm_init
   # Retry is because upload config sometimes fails
   retries: 3
@@ -105,76 +111,73 @@
     PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
   notify: Master | restart kubelet
 
-- name: slurp kubeadm certs
-  slurp:
-    src: "{{ item }}"
-  with_items:
-    - "{{ kube_cert_dir }}/apiserver.crt"
-    - "{{ kube_cert_dir }}/apiserver.key"
-    - "{{ kube_cert_dir }}/apiserver-kubelet-client.crt"
-    - "{{ kube_cert_dir }}/apiserver-kubelet-client.key"
-    - "{{ kube_cert_dir }}/ca.crt"
-    - "{{ kube_cert_dir }}/ca.key"
-    - "{{ kube_cert_dir }}/front-proxy-ca.crt"
-    - "{{ kube_cert_dir }}/front-proxy-ca.key"
-    - "{{ kube_cert_dir }}/front-proxy-client.crt"
-    - "{{ kube_cert_dir }}/front-proxy-client.key"
-    - "{{ kube_cert_dir }}/sa.key"
-    - "{{ kube_cert_dir }}/sa.pub"
-  register: kubeadm_certs
-  delegate_to: "{{ groups['kube-master']|first }}"
-  run_once: true
-
-- name: kubeadm | write out kubeadm certs
-  copy:
-    dest: "{{ item.item }}"
-    content: "{{ item.content | b64decode }}"
-    owner: root
-    group: root
-    mode: 0600
-  no_log: true
-  register: copy_kubeadm_certs
-  with_items: "{{ kubeadm_certs.results }}"
-  when: inventory_hostname != groups['kube-master']|first
-
-- name: kubeadm | Kubeconfig management with kubeadm
-  import_tasks: kubeadm-kubeconfig.yml
+- name: set kubeadm certificate key
+  set_fact:
+    kubeadm_certificate_key: "{{ item | regex_search('--certificate-key ([^ ]+)','\\1') | first }}"
+  with_items: "{{ (hostvars['kube-master'][0]['kubeadm_init']|default({'stdout_lines': []}))['stdout_lines'] }}"
   when:
-    - not upgrade_cluster_setup
-    - kubeadm_already_run.stat.exists
+    - kubeadm_version is version('v1.14.0', '>=')
+    - kubeadm_certificate_key is not defined
+    - item | trim | match('.*--certificate-key .*')
+    - hostvars['kube-master'][0]['kubeadm_init']['stdout_lines'] is defined
 
-- name: kubeadm | Init other uninitialized masters
-  command: timeout -k 600s 600s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all
-  register: kubeadm_init
-  retries: 10
-  until: kubeadm_init is succeeded or "field is immutable" in kubeadm_init.stderr
-  when: inventory_hostname != groups['kube-master']|first and not kubeadm_already_run.stat.exists
-  failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
-  environment:
-    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
-  notify: Master | restart kubelet
+- name: Create kubeadm token for joining nodes with 24h expiration (default)
+  command: "{{ bin_dir }}/kubeadm --kubeconfig /etc/kubernetes/admin.conf token create"
+  register: temp_token
+  retries: 5
+  delay: 5
+  until: temp_token is succeeded
+  delegate_to: "{{groups['kube-master']|first}}"
+  when: kubeadm_token is not defined
+  tags:
+    - kubeadm_token
 
-- name: kubeadm | upgrage kubernetes cluster
+- name: Set kubeadm_token
+  set_fact:
+    kubeadm_token: "{{ temp_token.stdout }}"
+  when: temp_token.stdout is defined
+  tags:
+    - kubeadm_token
+
+- name: Create hardcoded kubeadm token for joining nodes with 24h expiration (if defined)
+  shell: >-
+    {{ bin_dir }}/kubeadm --kubeconfig /etc/kubernetes/admin.conf token delete {{ kubeadm_token }} || :;
+    {{ bin_dir }}/kubeadm --kubeconfig /etc/kubernetes/admin.conf token create {{ kubeadm_token }}
+  when:
+    - inventory_hostname == groups['kube-master']|first
+    - kubeadm_token is defined
+  tags:
+    - kubeadm_token
+
+- name: kubeadm | Initialize other masters (experimental control plane)
+  include: kubeadm-secondary-experimental.yml
+  when: kubeadm_control_plane
+
+- name: kubeadm | Initialize other masters (experimental control plane)
+  include: kubeadm-secondary-legacy.yml
+  when: not kubeadm_control_plane
+
+- name: kubeadm | upgrade kubernetes cluster
   import_tasks: kubeadm-upgrade.yml
   when: upgrade_cluster_setup
 
-- name: kubeadm | Check apiserver key again
+- name: kubeadm | Check serviceaccount key again
   stat:
-    path: "{{ kube_cert_dir }}/apiserver.key"
-  register: apiserver_key_after
-  delegate_to: "{{groups['kube-master']|first}}"
+    path: "{{ kube_cert_dir }}/sa.key"
+  register: sa_key_after
   run_once: true
 
 - name: kubeadm | Set secret_changed if service account key was updated
   command: /bin/true
   notify: Master | set secret_changed
-  when: apiserver_key_before.stat.checksum|default("") != apiserver_key_after.stat.checksum
+  when: sa_key_before.stat.checksum|default("") != sa_key_after.stat.checksum
 
 - name: kubeadm | cleanup old certs if necessary
   import_tasks: kubeadm-cleanup-old-certs.yml
   when:
     - old_apiserver_cert.stat.exists
 
+# FIXME(mattymo): from docs: If you don't want to taint your control-plane node, set this field to an empty slice, i.e. `taints: {}` in the YAML file.
 - name: kubeadm | Remove taint for master with node role
   command: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf taint node {{ inventory_hostname }} node-role.kubernetes.io/master:NoSchedule-"
   delegate_to: "{{groups['kube-master']|first}}"