Files
Ali Afsharzadeh 15ea3a9bba Upload kubeadm ClusterConfiguration before the control plane upgrade (#13594)
kubeadm upgrade reads ClusterConfiguration only from the kubeadm-config
ConfigMap, so it keeps the old etcd version. The etcd version then
changes on all control plane nodes at the same time, and the control
plane stops. Upload the ClusterConfiguration first. Then kubeadm
upgrades etcd on one node at a time.

Signed-off-by: Ali Afsharzadeh <afsharzadeh8@gmail.com>
2026-09-30 09:06:38 +00:00

130 lines
5.3 KiB
YAML

---
- name: Ensure kube-apiserver is up before upgrade
import_tasks: check-api.yml
- name: Kubeadm | Preflight admission readiness check
command:
cmd: "{{ kubectl }} create --dry-run=server -f -"
stdin: >-
{"apiVersion":"batch/v1","kind":"Job","metadata":{"generateName":"upgrade-preflight-probe-","namespace":"kube-system"},"spec":{"template":{"spec":{"containers":[{"name":"pause","image":"{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"}],"restartPolicy":"Never"}}}}
register: preflight_check_result
retries: "{{ kubeadm_upgrade_preflight_check_retries }}"
delay: "{{ kubeadm_upgrade_preflight_check_delay }}"
until: preflight_check_result.rc == 0
changed_when: false
failed_when: false
when:
- kubeadm_upgrade_preflight_check_enabled
- inventory_hostname == first_kube_control_plane
environment:
PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
- name: Kubeadm | Warn if preflight admission check failed
debug:
msg: >-
WARNING: Preflight admission dry-run failed after {{ kubeadm_upgrade_preflight_check_retries }} retries.
The admission pipeline may not be ready. kubeadm upgrade apply may fail with [ERROR CreateJob].
See Kubernetes issue https://github.com/kubernetes/kubernetes/issues/124237 for details.
when:
- kubeadm_upgrade_preflight_check_enabled
- inventory_hostname == first_kube_control_plane
- preflight_check_result.rc is defined and preflight_check_result.rc != 0
# kubeadm upgrade reads ClusterConfiguration only from the kubeadm-config ConfigMap.
# Upload it first. Then kubeadm upgrades etcd on one node at a time.
- name: Kubeadm | Upload ClusterConfiguration before upgrade
command: "{{ bin_dir }}/kubeadm init phase upload-config kubeadm --config {{ kube_config_dir }}/kubeadm-config.yaml"
register: kubeadm_upload_cluster_config
retries: 3
until: kubeadm_upload_cluster_config.rc == 0
when:
- inventory_hostname == first_kube_control_plane
- name: Kubeadm | Upgrade first control plane node to {{ kube_version }}
command: >-
timeout -k 600s 600s
{{ bin_dir }}/kubeadm upgrade apply -y v{{ kube_version }}
--config={{ kube_config_dir }}/kubeadm-config.yaml
register: kubeadm_upgrade
when: inventory_hostname == first_kube_control_plane
failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
environment:
PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
- name: Kubeadm | Upgrade other control plane nodes to {{ kube_version }}
command: >-
{{ bin_dir }}/kubeadm upgrade node
--config={{ kube_config_dir }}/kubeadm-config.yaml
register: kubeadm_upgrade
when: inventory_hostname != first_kube_control_plane
failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
environment:
PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
# kubeadm upgrade no longer reconciles ClusterConfiguration and KubeProxyConfiguration changes, this must be done separately after upgrade to ensure the latest config is applied
- name: Update kubeadm and kubelet configmaps after upgrade
command: "{{ bin_dir }}/kubeadm init phase upload-config all --config {{ kube_config_dir }}/kubeadm-config.yaml"
register: kubeadm_upload_config
# Retry is because upload config sometimes fails
retries: 3
until: kubeadm_upload_config.rc == 0
when:
- inventory_hostname == first_kube_control_plane
- name: Update kube-proxy configmap after upgrade
command: "{{ bin_dir }}/kubeadm init phase addon kube-proxy --config {{ kube_config_dir }}/kubeadm-config.yaml"
register: kube_proxy_upload_config
# Retry is because upload config sometimes fails
retries: 3
until: kube_proxy_upload_config.rc == 0
when:
- inventory_hostname == first_kube_control_plane
- ('addon/kube-proxy' not in kubeadm_init_phases_skip)
- name: Rewrite kubeadm managed etcd static pod manifests with updated configmap
command: "{{ bin_dir }}/kubeadm init phase etcd local --config {{ kube_config_dir }}/kubeadm-config.yaml"
when:
- etcd_deployment_type == "kubeadm"
notify: Control plane | restart kubelet
- name: Rewrite kubernetes control plane static pod manifests with updated configmap
command: "{{ bin_dir }}/kubeadm init phase control-plane all --config {{ kube_config_dir }}/kubeadm-config.yaml"
notify: Control plane | restart kubelet
- name: Flush kubelet handlers
meta: flush_handlers
- name: Ensure kube-apiserver is up after upgrade and control plane configuration updates
import_tasks: check-api.yml
- name: Kubeadm | Remove binding to anonymous user
command: "{{ kubectl }} -n kube-public delete rolebinding kubeadm:bootstrap-signer-clusterinfo --ignore-not-found"
when: remove_anonymous_access
- name: Kubeadm | clean kubectl cache to refresh api types
file:
path: "{{ item }}"
state: absent
with_items:
- /root/.kube/cache
- /root/.kube/http-cache
# FIXME: https://github.com/kubernetes/kubeadm/issues/1318
- name: Kubeadm | scale down coredns replicas to 0 if not using coredns dns_mode
command: >-
{{ kubectl }}
-n kube-system
scale deployment/coredns --replicas 0
register: scale_down_coredns
retries: 6
delay: 5
until: scale_down_coredns is succeeded
run_once: true
when:
- kubeadm_scale_down_coredns_enabled
- dns_mode not in ['coredns', 'coredns_dual']
changed_when: false
- name: Kubeadm | cleanup old upgrade backups
import_tasks: kubeadm-backup-cleanup.yml