Files
felipe88alves f6b046bcac feat: add preflight admission readiness check before kubeadm upgrade (#13140)
* feat: add preflight admission readiness check before kubeadm upgrade

  - linked to issue #13139
  - Add opt-in dry-run Job probe to verify the admission pipeline is ready before running kubeadm upgrade apply (workaround for Kubernetes #124237)
  - Gate the check behind kubeadm_upgrade_preflight_check_enabled (default false) with configurable retries and delay
  - Emit a warning instead of failing if the preflight check does not pass

* fix: refine preflight admission check Job and warning

- Use generateName for the dry-run preflight Job to avoid fixed-name collisions
- Expand the Kubernetes issue reference in the warning to a full GitHub URL

Created with AI-assistance
2026-09-10 08:14:36 +00:00

120 lines
4.8 KiB
YAML

---
- name: Ensure kube-apiserver is up before upgrade
import_tasks: check-api.yml
- name: Kubeadm | Preflight admission readiness check
command:
cmd: "{{ kubectl }} create --dry-run=server -f -"
stdin: >-
{"apiVersion":"batch/v1","kind":"Job","metadata":{"generateName":"upgrade-preflight-probe-","namespace":"kube-system"},"spec":{"template":{"spec":{"containers":[{"name":"pause","image":"{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"}],"restartPolicy":"Never"}}}}
register: preflight_check_result
retries: "{{ kubeadm_upgrade_preflight_check_retries }}"
delay: "{{ kubeadm_upgrade_preflight_check_delay }}"
until: preflight_check_result.rc == 0
changed_when: false
failed_when: false
when:
- kubeadm_upgrade_preflight_check_enabled
- inventory_hostname == first_kube_control_plane
environment:
PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
- name: Kubeadm | Warn if preflight admission check failed
debug:
msg: >-
WARNING: Preflight admission dry-run failed after {{ kubeadm_upgrade_preflight_check_retries }} retries.
The admission pipeline may not be ready. kubeadm upgrade apply may fail with [ERROR CreateJob].
See Kubernetes issue https://github.com/kubernetes/kubernetes/issues/124237 for details.
when:
- kubeadm_upgrade_preflight_check_enabled
- inventory_hostname == first_kube_control_plane
- preflight_check_result.rc is defined and preflight_check_result.rc != 0
- name: Kubeadm | Upgrade first control plane node to {{ kube_version }}
command: >-
timeout -k 600s 600s
{{ bin_dir }}/kubeadm upgrade apply -y v{{ kube_version }}
--config={{ kube_config_dir }}/kubeadm-config.yaml
register: kubeadm_upgrade
when: inventory_hostname == first_kube_control_plane
failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
environment:
PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
- name: Kubeadm | Upgrade other control plane nodes to {{ kube_version }}
command: >-
{{ bin_dir }}/kubeadm upgrade node
--config={{ kube_config_dir }}/kubeadm-config.yaml
register: kubeadm_upgrade
when: inventory_hostname != first_kube_control_plane
failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
environment:
PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
# kubeadm upgrade no longer reconciles ClusterConfiguration and KubeProxyConfiguration changes, this must be done separately after upgrade to ensure the latest config is applied
- name: Update kubeadm and kubelet configmaps after upgrade
command: "{{ bin_dir }}/kubeadm init phase upload-config all --config {{ kube_config_dir }}/kubeadm-config.yaml"
register: kubeadm_upload_config
# Retry is because upload config sometimes fails
retries: 3
until: kubeadm_upload_config.rc == 0
when:
- inventory_hostname == first_kube_control_plane
- name: Update kube-proxy configmap after upgrade
command: "{{ bin_dir }}/kubeadm init phase addon kube-proxy --config {{ kube_config_dir }}/kubeadm-config.yaml"
register: kube_proxy_upload_config
# Retry is because upload config sometimes fails
retries: 3
until: kube_proxy_upload_config.rc == 0
when:
- inventory_hostname == first_kube_control_plane
- ('addon/kube-proxy' not in kubeadm_init_phases_skip)
- name: Rewrite kubeadm managed etcd static pod manifests with updated configmap
command: "{{ bin_dir }}/kubeadm init phase etcd local --config {{ kube_config_dir }}/kubeadm-config.yaml"
when:
- etcd_deployment_type == "kubeadm"
notify: Control plane | restart kubelet
- name: Rewrite kubernetes control plane static pod manifests with updated configmap
command: "{{ bin_dir }}/kubeadm init phase control-plane all --config {{ kube_config_dir }}/kubeadm-config.yaml"
notify: Control plane | restart kubelet
- name: Flush kubelet handlers
meta: flush_handlers
- name: Ensure kube-apiserver is up after upgrade and control plane configuration updates
import_tasks: check-api.yml
- name: Kubeadm | Remove binding to anonymous user
command: "{{ kubectl }} -n kube-public delete rolebinding kubeadm:bootstrap-signer-clusterinfo --ignore-not-found"
when: remove_anonymous_access
- name: Kubeadm | clean kubectl cache to refresh api types
file:
path: "{{ item }}"
state: absent
with_items:
- /root/.kube/cache
- /root/.kube/http-cache
# FIXME: https://github.com/kubernetes/kubeadm/issues/1318
- name: Kubeadm | scale down coredns replicas to 0 if not using coredns dns_mode
command: >-
{{ kubectl }}
-n kube-system
scale deployment/coredns --replicas 0
register: scale_down_coredns
retries: 6
delay: 5
until: scale_down_coredns is succeeded
run_once: true
when:
- kubeadm_scale_down_coredns_enabled
- dns_mode not in ['coredns', 'coredns_dual']
changed_when: false
- name: Kubeadm | cleanup old upgrade backups
import_tasks: kubeadm-backup-cleanup.yml