Upgrade to kubeadm (#1667)

* Enable upgrade to kubeadm

* fix kubedns upgrade

* try upgrade route

* use init/upgrade strategy for kubeadm and ignore kubedns svc

* Use bin_dir for kubeadm

* delete more secrets

* fix waiting for terminating pods

* Manually enforce kube-proxy for kubeadm deploy

* remove proxy. update to kubeadm 1.8.0rc1

roles/kubernetes/kubeadm/tasks/main.yml

@@ -24,7 +24,7 @@
   register: kubeadm_client_conf
 
 - name: Join to cluster if needed
-  command: kubeadm join --config {{ kube_config_dir}}/kubeadm-client.conf --skip-preflight-checks
+  command: "{{ bin_dir }}/kubeadm join --config {{ kube_config_dir}}/kubeadm-client.conf --skip-preflight-checks"
   register: kubeadm_join
   when: not is_kube_master and (kubeadm_client_conf.changed or not kubelet_conf.stat.exists)
 

roles/kubernetes/master/tasks/kubeadm-cleanup-old-certs.yml

@@ -0,0 +1,3 @@
+---
+- name: kubeadm | Purge old certs
+  command: "rm -f {{kube_cert_dir }}/*.pem"

roles/kubernetes/master/tasks/kubeadm-migrate-certs.yml

@@ -0,0 +1,12 @@
+---
+- name: Copy old certs to the kubeadm expected path
+  copy:
+    src: "{{ kube_cert_dir }}/{{ item.src }}"
+    dest: "{{ kube_cert_dir }}/{{ item.dest }}"
+    remote_src: yes
+  with_items:
+    - {src: apiserver.pem, dest: apiserver.crt}
+    - {src: apiserver.pem, dest: apiserver.key}
+    - {src: ca.pem, dest: ca.crt}
+    - {src: ca-key.pem, dest: ca.key}
+  register: kubeadm_copy_old_certs

roles/kubernetes/master/tasks/kubeadm-setup.yml

@@ -1,4 +1,35 @@
 ---
+- name: kubeadm | Check if old apiserver cert exists on host
+  stat:
+    path: "{{ kube_cert_dir }}/apiserver.pem"
+  register: old_apiserver_cert
+  delegate_to: "{{groups['kube-master']|first}}"
+  run_once: true
+
+- name: kubeadm | Check service account key
+  stat:
+    path: "{{ kube_cert_dir }}/sa.key"
+  register: sa_key_before
+  delegate_to: "{{groups['kube-master']|first}}"
+  run_once: true
+
+- name: kubeadm | Check if kubeadm has already run
+  stat:
+    path: "{{ kube_config_dir }}/admin.conf"
+  register: admin_conf
+
+- name: kubeadm | Delete old static pods
+  file:
+    path: "{{ kube_config_dir }}/manifests/{{item}}.manifest"
+    state: absent
+  with_items: ["kube-apiserver", "kube-controller-manager", "kube-scheduler", "kube-proxy"]
+  when: old_apiserver_cert.stat.exists
+
+- name: kubeadm | Forcefully delete old static pods
+  shell: "docker ps -f name=k8s_{{item}} -q | xargs --no-run-if-empty docker rm -f"
+  with_items: ["kube-apiserver", "kube-controller-manager", "kube-scheduler"]
+  when: old_apiserver_cert.stat.exists
+
 - name: kubeadm | aggregate all SANs
   set_fact:
     apiserver_sans: >-
@@ -29,18 +60,29 @@
     dest: "{{ kube_config_dir }}/kubeadm-config.yaml"
   register: kubeadm_config
 
-- name: Check if kubeadm has already run
-  stat:
-    path: "{{ kube_config_dir }}/admin.conf"
-  register: admin_conf
-
-
 - name: kubeadm | Initialize first master
-  command: timeout -k 240s 240s kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --skip-preflight-checks
+  command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --skip-preflight-checks
   register: kubeadm_init
   #Retry is because upload config sometimes fails
   retries: 3
-  when: inventory_hostname == groups['kube-master']|first and (kubeadm_config.changed or not admin_conf.stat.exists)
+  when: inventory_hostname == groups['kube-master']|first and not admin_conf.stat.exists
+  failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
+  notify: Master | restart kubelet
+
+- name: kubeadm | Upgrade first master
+  command: timeout -k 240s 240s {{ bin_dir }}/kubeadm upgrade apply --config={{ kube_config_dir }}/kubeadm-config.yaml {{ kube_version }} --skip-preflight-checks
+  register: kubeadm_upgrade
+  #Retry is because upload config sometimes fails
+  retries: 3
+  when: inventory_hostname == groups['kube-master']|first and (kubeadm_config.changed and admin_conf.stat.exists)
+  failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
+  notify: Master | restart kubelet
+
+# FIXME(mattymo): remove when https://github.com/kubernetes/kubeadm/issues/433 is fixed
+- name: kubeadm | Enable kube-proxy
+  command: "{{ bin_dir }}/kubeadm alpha phase addon kube-proxy --config={{ kube_config_dir }}/kubeadm-config.yaml"
+  when: inventory_hostname == groups['kube-master']|first
+  changed_when: false
 
 - name: slurp kubeadm certs
   slurp:
@@ -62,7 +104,7 @@
   delegate_to: "{{ groups['kube-master']|first }}"
   run_once: true
 
-- name: write out kubeadm certs
+- name: kubeadm | write out kubeadm certs
   copy:
     dest: "{{ item.item }}"
     content: "{{ item.content | b64decode }}"
@@ -74,9 +116,32 @@
   with_items: "{{ kubeadm_certs.results }}"
   when: inventory_hostname != groups['kube-master']|first
 
-- name: kubeadm | Initialize other masters
-  command: timeout -k 240s 240s kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --skip-preflight-checks
+- name: kubeadm | Init other uninitialized masters
+  command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --skip-preflight-checks
   register: kubeadm_init
-  #Retry is because upload config sometimes fails
-  retries: 3
-  when: inventory_hostname != groups['kube-master']|first and (kubeadm_config.changed or not admin_conf.stat.exists or copy_kubeadm_certs.changed)
+  when: inventory_hostname != groups['kube-master']|first and not admin_conf.stat.exists
+  failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
+  notify: Master | restart kubelet
+
+- name: kubeadm | Upgrade other masters
+  command: timeout -k 240s 240s {{ bin_dir }}/kubeadm upgrade apply --config={{ kube_config_dir }}/kubeadm-config.yaml {{ kube_version }} --skip-preflight-checks
+  register: kubeadm_upgrade
+  when: inventory_hostname != groups['kube-master']|first and (kubeadm_config.changed and admin_conf.stat.exists)
+  failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
+  notify: Master | restart kubelet
+
+- name: kubeadm | Check service account key again
+  stat:
+    path: "{{ kube_cert_dir }}/sa.key"
+  register: sa_key_after
+  delegate_to: "{{groups['kube-master']|first}}"
+  run_once: true
+
+- name: kubeadm | Set secret_changed if service account key was updated
+  command: /bin/true
+  notify: Master | set secret_changed
+  when: sa_key_before.stat.checksum|default("") != sa_key_after.stat.checksum
+
+- name: kubeadm | cleanup old certs if necessary
+  include: kubeadm-cleanup-old-certs.yml
+  when: old_apiserver_cert.stat.exists

roles/kubernetes/master/templates/kubeadm-config.yaml.j2

@@ -17,7 +17,6 @@ networking:
   podSubnet: {{ kube_pods_subnet }}
 kubernetesVersion: {{ kube_version }}
 cloudProvider: {{ cloud_provider|default('') }}
-#TODO: cloud provider conf file
 authorizationModes:
 - Node
 {% for mode in authorization_modes %}
@@ -53,7 +52,6 @@ apiServerExtraArgs:
   runtime-config: {{ kube_api_runtime_config }}
 {% endif %}
   allow-privileged: "true"
-#TODO: Custom flags compatible with kubeadm
 controllerManagerExtraArgs:
   node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
   node-monitor-period: {{ kube_controller_node_monitor_period }}
@@ -61,7 +59,6 @@ controllerManagerExtraArgs:
 {% if kube_feature_gates %}
   feature-gates: {{ kube_feature_gates|join(',') }}
 {% endif %}
-#schedulerExtraArgs:
 apiServerCertSANs:
 {% for san in  apiserver_sans.split(' ') | unique %}
   - {{ san }}

roles/kubernetes/node/tasks/install.yml

@@ -19,12 +19,20 @@
   when: kubeadm_enabled
   tags: kubeadm
 
-- name: install | Copy binary from download dir
+- name: install | Copy kubeadm binary from download dir
   command: rsync -piu "{{ local_release_dir }}/kubeadm" "{{ bin_dir }}/kubeadm"
   changed_when: false
   when: kubeadm_enabled
   tags: kubeadm
 
+- name: install | Set kubeadm binary permissions
+  file:
+    path: "{{ bin_dir }}/kubeadm"
+    mode: "0755"
+    state: file
+  when: kubeadm_enabled
+  tags: kubeadm
+
 - include: "install_{{ kubelet_deployment_type }}.yml"
 
 - name: install | Write kubelet systemd init file

roles/kubernetes/node/tasks/main.yml

@@ -96,6 +96,13 @@
   when: not kubeadm_enabled
   tags: kube-proxy
 
+- name: Purge proxy manifest for kubeadm
+  file:
+    path: "{{ kube_manifest_dir }}/kube-proxy.manifest"
+    state: absent
+  when: kubeadm_enabled
+  tags: kube-proxy
+
 # reload-systemd
 - meta: flush_handlers