Use K8s 1.14 and add kubeadm experimental control plane mode (#4514)

* Use K8s 1.14 and add kubeadm experimental control plane mode

This reverts commit d39c273d96.

* Cleanup kubeadm setup run on first master

* pin kubeadm_certificate_key in test

* Remove kubelet autolabel of kube-node, add symlink for pki dir

Change-Id: Id5e74dd667c60675dbfe4193b0bc9fb44380e1ca

roles/kubernetes/master/tasks/kubeadm-certificate.yml

@@ -12,33 +12,3 @@
     - {src: front-proxy-client.crt, dest: front-proxy-client.crt.old}
     - {src: front-proxy-client.key, dest: front-proxy-client.key.old}
   ignore_errors: yes
-
-- name: Remove old certs and keys
-  file:
-    path: "{{ kube_cert_dir }}/{{ item }}"
-    state: absent
-  with_items:
-    - apiserver.crt
-    - apiserver.key
-    - apiserver-kubelet-client.crt
-    - apiserver-kubelet-client.key
-    - front-proxy-client.crt
-    - front-proxy-client.key
-
-- name: Generate new certs and keys
-  command: "{{ bin_dir }}/kubeadm init phase certs {{ item }} --config={{ kube_config_dir }}/kubeadm-config.yaml"
-  environment: "{{ proxy_env }}"
-  with_items:
-    - apiserver
-    - apiserver-kubelet-client
-    - front-proxy-client
-  when: inventory_hostname == groups['kube-master']|first and kubeadm_version is version('v1.13.0', '>=')
-
-- name: Generate new certs and keys
-  command: "{{ bin_dir }}/kubeadm alpha phase certs {{ item }} --config={{ kube_config_dir }}/kubeadm-config.yaml"
-  environment: "{{ proxy_env }}"
-  with_items:
-    - apiserver
-    - apiserver-kubelet-client
-    - front-proxy-client
-  when: inventory_hostname == groups['kube-master']|first and kubeadm_version is version('v1.13.0', '<')

roles/kubernetes/master/tasks/kubeadm-kubeconfig.yml

@@ -1,34 +0,0 @@
----
-- name: Backup old configuration files
-  copy:
-    src: "{{ kube_config_dir }}/{{ item.src }}"
-    dest: "{{ kube_config_dir }}/{{ item.dest }}"
-    remote_src: yes
-  with_items:
-    - {src: admin.conf, dest: admin.conf.old}
-    - {src: kubelet.conf, dest: kubelet.conf.old}
-    - {src: controller-manager.conf, dest: controller-manager.conf.old}
-    - {src: scheduler.conf, dest: scheduler.conf.old}
-  ignore_errors: yes
-
-- name: Remove old configuration files
-  file:
-    path: "{{ kube_config_dir }}/{{ item }}"
-    state: absent
-  with_items:
-    - admin.conf
-    - kubelet.conf
-    - controller-manager.conf
-    - scheduler.conf
-
-- name: Generate new configuration files
-  command: "{{ bin_dir }}/kubeadm init phase kubeconfig all --config={{ kube_config_dir }}/kubeadm-config.yaml"
-  environment: "{{ proxy_env }}"
-  when: kubeadm_version is version('v1.13.0', '>=')
-  ignore_errors: yes
-
-- name: Generate new configuration files
-  command: "{{ bin_dir }}/kubeadm alpha phase kubeconfig all --config={{ kube_config_dir }}/kubeadm-config.yaml"
-  environment: "{{ proxy_env }}"
-  when: kubeadm_version is version('v1.13.0', '<')
-  ignore_errors: yes

roles/kubernetes/master/tasks/kubeadm-migrate-certs.yml

@@ -15,6 +15,6 @@
     - {src: front-proxy-client-key.pem, dest: front-proxy-client.key}
     - {src: service-account-key.pem, dest: sa.pub}
     - {src: service-account-key.pem, dest: sa.key}
-    - {src: "node-{{ inventory_hostname }}.pem", dest: apiserver-kubelet-client.crt }
-    - {src: "node-{{ inventory_hostname }}-key.pem", dest: apiserver-kubelet-client.key }
+    - {src: "node-{{ inventory_hostname }}.pem", dest: apiserver-kubelet-client.crt}
+    - {src: "node-{{ inventory_hostname }}-key.pem", dest: apiserver-kubelet-client.key}
   register: kubeadm_copy_old_certs

roles/kubernetes/master/tasks/kubeadm-secondary-experimental.yml

@@ -0,0 +1,45 @@
+---
+- name: Set kubeadm_discovery_address
+  set_fact:
+    kubeadm_discovery_address: >-
+      {%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%}
+      {{ first_kube_master }}:{{ kube_apiserver_port }}
+      {%- else -%}
+      {{ kube_apiserver_endpoint }}
+      {%- endif %}
+  tags:
+    - facts
+
+- name: Create kubeadm ControlPlane config
+  template:
+    src: "kubeadm-controlplane.{{ kubeadmConfig_api_version }}.yaml.j2"
+    dest: "{{ kube_config_dir }}/kubeadm-controlplane.yaml"
+    backup: yes
+  when:
+    - inventory_hostname != groups['kube-master']|first
+    - not kubeadm_already_run.stat.exists
+
+- name: Wait for k8s apiserver
+  wait_for:
+    host: "{{kubeadm_discovery_address.split(':')[0]}}"
+    port: "{{kubeadm_discovery_address.split(':')[1]}}"
+    timeout: 180
+
+- name: Joining control plane node to the cluster.
+  command: >-
+    {{ bin_dir }}/kubeadm join
+    --config {{ kube_config_dir}}/kubeadm-controlplane.yaml
+    --ignore-preflight-errors=all
+    {% if kubeadm_certificate_key is defined %}
+    --certificate-key={{ kubeadm_certificate_key }}
+    {% endif %}
+  register: kubeadm_join_control_plane
+  when:
+    - inventory_hostname != groups['kube-master']|first
+    - not kubeadm_already_run.stat.exists
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
+
+- name: Set secret_changed to false to avoid extra token rotation
+  set_fact:
+    secret_changed: false

roles/kubernetes/master/tasks/kubeadm-secondary-legacy.yml

@@ -0,0 +1,44 @@
+---
+- name: slurp kubeadm certs
+  slurp:
+    src: "{{ item }}"
+  with_items:
+    - "{{ kube_cert_dir }}/apiserver.crt"
+    - "{{ kube_cert_dir }}/apiserver.key"
+    - "{{ kube_cert_dir }}/apiserver-kubelet-client.crt"
+    - "{{ kube_cert_dir }}/apiserver-kubelet-client.key"
+    - "{{ kube_cert_dir }}/ca.crt"
+    - "{{ kube_cert_dir }}/ca.key"
+    - "{{ kube_cert_dir }}/front-proxy-ca.crt"
+    - "{{ kube_cert_dir }}/front-proxy-ca.key"
+    - "{{ kube_cert_dir }}/front-proxy-client.crt"
+    - "{{ kube_cert_dir }}/front-proxy-client.key"
+    - "{{ kube_cert_dir }}/sa.key"
+    - "{{ kube_cert_dir }}/sa.pub"
+  register: kubeadm_certs
+  delegate_to: "{{ groups['kube-master']|first }}"
+
+- name: kubeadm | write out kubeadm certs
+  copy:
+    dest: "{{ item.item }}"
+    content: "{{ item.content | b64decode }}"
+    owner: root
+    group: root
+    mode: 0600
+  no_log: true
+  register: copy_kubeadm_certs
+  with_items: "{{ kubeadm_certs.results }}"
+  when: inventory_hostname != groups['kube-master']|first
+
+- name: kubeadm | Init other uninitialized masters
+  command: timeout -k 600s 600s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all
+  register: kubeadm_init
+  retries: 10
+  until: kubeadm_init is succeeded or "field is immutable" in kubeadm_init.stderr
+  when:
+    - inventory_hostname != groups['kube-master']|first
+    - not kubeadm_already_run.stat.exists
+  failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
+  notify: Master | restart kubelet

roles/kubernetes/master/tasks/kubeadm-setup.yml

@@ -10,11 +10,10 @@
   import_tasks: kubeadm-migrate-certs.yml
   when: old_apiserver_cert.stat.exists
 
-- name: kubeadm | Check apiserver key
+- name: kubeadm | Check serviceaccount key
   stat:
-    path: "{{ kube_cert_dir }}/apiserver.key"
-  register: apiserver_key_before
-  delegate_to: "{{groups['kube-master']|first}}"
+    path: "{{ kube_cert_dir }}/sa.key"
+  register: sa_key_before
   run_once: true
 
 - name: kubeadm | Check if kubeadm has already run
@@ -62,10 +61,6 @@
     sans_address: "{{ groups['kube-master'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | list | select('defined') | list }}"
   tags: facts
 
-- name: kubeadm | Copy etcd cert dir under k8s cert dir
-  command: "cp -TR {{ etcd_cert_dir }} {{ kube_config_dir }}/ssl/etcd"
-  changed_when: false
-
 - name: Create audit-policy directory
   file:
     path: "{{ audit_policy_file | dirname }}"
@@ -94,7 +89,18 @@
     - kubeadm_already_run.stat.exists
 
 - name: kubeadm | Initialize first master
-  command: timeout -k 600s 600s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all
+  command: >-
+    timeout -k 600s 600s
+    {{ bin_dir }}/kubeadm init
+    --config={{ kube_config_dir }}/kubeadm-config.yaml
+    --ignore-preflight-errors=all
+    {% if kubeadm_version is version('v1.14.0', '>=') %}
+    --experimental-upload-certs
+    {% endif %}
+    --skip-phases=addon/coredns
+    {% if kubeadm_certificate_key is defined %}
+    --certificate-key={{ kubeadm_certificate_key }}
+    {% endif %}
   register: kubeadm_init
   # Retry is because upload config sometimes fails
   retries: 3
@@ -105,76 +111,73 @@
     PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
   notify: Master | restart kubelet
 
-- name: slurp kubeadm certs
-  slurp:
-    src: "{{ item }}"
-  with_items:
-    - "{{ kube_cert_dir }}/apiserver.crt"
-    - "{{ kube_cert_dir }}/apiserver.key"
-    - "{{ kube_cert_dir }}/apiserver-kubelet-client.crt"
-    - "{{ kube_cert_dir }}/apiserver-kubelet-client.key"
-    - "{{ kube_cert_dir }}/ca.crt"
-    - "{{ kube_cert_dir }}/ca.key"
-    - "{{ kube_cert_dir }}/front-proxy-ca.crt"
-    - "{{ kube_cert_dir }}/front-proxy-ca.key"
-    - "{{ kube_cert_dir }}/front-proxy-client.crt"
-    - "{{ kube_cert_dir }}/front-proxy-client.key"
-    - "{{ kube_cert_dir }}/sa.key"
-    - "{{ kube_cert_dir }}/sa.pub"
-  register: kubeadm_certs
-  delegate_to: "{{ groups['kube-master']|first }}"
-  run_once: true
-
-- name: kubeadm | write out kubeadm certs
-  copy:
-    dest: "{{ item.item }}"
-    content: "{{ item.content | b64decode }}"
-    owner: root
-    group: root
-    mode: 0600
-  no_log: true
-  register: copy_kubeadm_certs
-  with_items: "{{ kubeadm_certs.results }}"
-  when: inventory_hostname != groups['kube-master']|first
-
-- name: kubeadm | Kubeconfig management with kubeadm
-  import_tasks: kubeadm-kubeconfig.yml
+- name: set kubeadm certificate key
+  set_fact:
+    kubeadm_certificate_key: "{{ item | regex_search('--certificate-key ([^ ]+)','\\1') | first }}"
+  with_items: "{{ (hostvars['kube-master'][0]['kubeadm_init']|default({'stdout_lines': []}))['stdout_lines'] }}"
   when:
-    - not upgrade_cluster_setup
-    - kubeadm_already_run.stat.exists
+    - kubeadm_version is version('v1.14.0', '>=')
+    - kubeadm_certificate_key is not defined
+    - item | trim | match('.*--certificate-key .*')
+    - hostvars['kube-master'][0]['kubeadm_init']['stdout_lines'] is defined
 
-- name: kubeadm | Init other uninitialized masters
-  command: timeout -k 600s 600s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all
-  register: kubeadm_init
-  retries: 10
-  until: kubeadm_init is succeeded or "field is immutable" in kubeadm_init.stderr
-  when: inventory_hostname != groups['kube-master']|first and not kubeadm_already_run.stat.exists
-  failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
-  environment:
-    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
-  notify: Master | restart kubelet
+- name: Create kubeadm token for joining nodes with 24h expiration (default)
+  command: "{{ bin_dir }}/kubeadm --kubeconfig /etc/kubernetes/admin.conf token create"
+  register: temp_token
+  retries: 5
+  delay: 5
+  until: temp_token is succeeded
+  delegate_to: "{{groups['kube-master']|first}}"
+  when: kubeadm_token is not defined
+  tags:
+    - kubeadm_token
 
-- name: kubeadm | upgrage kubernetes cluster
+- name: Set kubeadm_token
+  set_fact:
+    kubeadm_token: "{{ temp_token.stdout }}"
+  when: temp_token.stdout is defined
+  tags:
+    - kubeadm_token
+
+- name: Create hardcoded kubeadm token for joining nodes with 24h expiration (if defined)
+  shell: >-
+    {{ bin_dir }}/kubeadm --kubeconfig /etc/kubernetes/admin.conf token delete {{ kubeadm_token }} || :;
+    {{ bin_dir }}/kubeadm --kubeconfig /etc/kubernetes/admin.conf token create {{ kubeadm_token }}
+  when:
+    - inventory_hostname == groups['kube-master']|first
+    - kubeadm_token is defined
+  tags:
+    - kubeadm_token
+
+- name: kubeadm | Initialize other masters (experimental control plane)
+  include: kubeadm-secondary-experimental.yml
+  when: kubeadm_control_plane
+
+- name: kubeadm | Initialize other masters (experimental control plane)
+  include: kubeadm-secondary-legacy.yml
+  when: not kubeadm_control_plane
+
+- name: kubeadm | upgrade kubernetes cluster
   import_tasks: kubeadm-upgrade.yml
   when: upgrade_cluster_setup
 
-- name: kubeadm | Check apiserver key again
+- name: kubeadm | Check serviceaccount key again
   stat:
-    path: "{{ kube_cert_dir }}/apiserver.key"
-  register: apiserver_key_after
-  delegate_to: "{{groups['kube-master']|first}}"
+    path: "{{ kube_cert_dir }}/sa.key"
+  register: sa_key_after
   run_once: true
 
 - name: kubeadm | Set secret_changed if service account key was updated
   command: /bin/true
   notify: Master | set secret_changed
-  when: apiserver_key_before.stat.checksum|default("") != apiserver_key_after.stat.checksum
+  when: sa_key_before.stat.checksum|default("") != sa_key_after.stat.checksum
 
 - name: kubeadm | cleanup old certs if necessary
   import_tasks: kubeadm-cleanup-old-certs.yml
   when:
     - old_apiserver_cert.stat.exists
 
+# FIXME(mattymo): from docs: If you don't want to taint your control-plane node, set this field to an empty slice, i.e. `taints: {}` in the YAML file.
 - name: kubeadm | Remove taint for master with node role
   command: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf taint node {{ inventory_hostname }} node-role.kubernetes.io/master:NoSchedule-"
   delegate_to: "{{groups['kube-master']|first}}"

roles/kubernetes/master/tasks/kubeadm-upgrade.yml

@@ -17,6 +17,8 @@
   failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
   notify: Master | restart kubelet
 
+# FIXME: https://github.com/kubernetes/kubeadm/issues/1498 remove stdout_lines
+# check after issue is fixed
 - name: kubeadm | Upgrade other masters
   command: >-
     timeout -k 600s 600s
@@ -29,5 +31,8 @@
     --etcd-upgrade=false
   register: kubeadm_upgrade
   when: inventory_hostname != groups['kube-master']|first
-  failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
+  failed_when:
+    - kubeadm_upgrade.rc != 0
+    - '"field is immutable" not in kubeadm_upgrade.stderr'
+    - kubeadm_upgrade.stdout_lines | length > 1
   notify: Master | restart kubelet

roles/kubernetes/master/tasks/kubeadm-version.yml

@@ -3,11 +3,6 @@
   command: "{{ bin_dir }}/kubeadm version -o short"
   register: kubeadm_output
 
-- name: sets kubeadm api version to v1alpha1
-  set_fact:
-    kubeadmConfig_api_version: v1alpha1
-  when: kubeadm_output.stdout is version('v1.11.0', '<')
-
 - name: sets kubeadm api version to v1alpha2
   set_fact:
     kubeadmConfig_api_version: v1alpha2

roles/kubernetes/master/tasks/pre-upgrade.yml

@@ -3,8 +3,8 @@
   command: "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} ls /registry/minions"
   environment:
     ETCDCTL_API: 2
-    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
-    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/{{ kube_etcd_cert_file }}"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/{{ kube_etcd_key_file }}"
   register: old_data_exists
   delegate_to: "{{groups['etcd'][0]}}"
   changed_when: false