From f6b046bcac4e2eb413ec56f3af0f5fb08a4ec7e8 Mon Sep 17 00:00:00 2001 From: felipe88alves Date: Thu, 10 Sep 2026 09:14:36 +0100 Subject: [PATCH] feat: add preflight admission readiness check before kubeadm upgrade (#13140) * feat: add preflight admission readiness check before kubeadm upgrade - linked to issue #13139 - Add opt-in dry-run Job probe to verify the admission pipeline is ready before running kubeadm upgrade apply (workaround for Kubernetes #124237) - Gate the check behind kubeadm_upgrade_preflight_check_enabled (default false) with configurable retries and delay - Emit a warning instead of failing if the preflight check does not pass * fix: refine preflight admission check Job and warning - Use generateName for the dry-run preflight Job to avoid fixed-name collisions - Expand the Kubernetes issue reference in the warning to a full GitHub URL Created with AI-assistance --- .../control-plane/defaults/main/main.yml | 8 ++++++ .../control-plane/tasks/kubeadm-upgrade.yml | 28 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/roles/kubernetes/control-plane/defaults/main/main.yml b/roles/kubernetes/control-plane/defaults/main/main.yml index 34671cd02..ed49545ab 100644 --- a/roles/kubernetes/control-plane/defaults/main/main.yml +++ b/roles/kubernetes/control-plane/defaults/main/main.yml @@ -37,6 +37,14 @@ kube_controller_manager_bind_address: "::" ## Control plane health check settings control_plane_health_retries: 60 # Default retries for apiserver, scheduler, controller-manager health checks +# Preflight admission readiness check before kubeadm upgrade (opt-in) +# Dry-runs a batch/v1 Job to verify the admission pipeline is ready after API server restart +# Workaround for Kubernetes #124237: admission controllers may be stale after API server restart +# Enable if your cluster uses admission controllers (e.g. ValidatingAdmissionPolicy) with CRD paramKind references +kubeadm_upgrade_preflight_check_enabled: false +kubeadm_upgrade_preflight_check_retries: 6 +kubeadm_upgrade_preflight_check_delay: 5 + # Leader election lease durations and timeouts for controller-manager kube_controller_manager_leader_elect_lease_duration: 15s kube_controller_manager_leader_elect_renew_deadline: 10s diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml b/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml index 1ffd18153..a602c4895 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml @@ -2,6 +2,34 @@ - name: Ensure kube-apiserver is up before upgrade import_tasks: check-api.yml +- name: Kubeadm | Preflight admission readiness check + command: + cmd: "{{ kubectl }} create --dry-run=server -f -" + stdin: >- + {"apiVersion":"batch/v1","kind":"Job","metadata":{"generateName":"upgrade-preflight-probe-","namespace":"kube-system"},"spec":{"template":{"spec":{"containers":[{"name":"pause","image":"{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"}],"restartPolicy":"Never"}}}} + register: preflight_check_result + retries: "{{ kubeadm_upgrade_preflight_check_retries }}" + delay: "{{ kubeadm_upgrade_preflight_check_delay }}" + until: preflight_check_result.rc == 0 + changed_when: false + failed_when: false + when: + - kubeadm_upgrade_preflight_check_enabled + - inventory_hostname == first_kube_control_plane + environment: + PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}" + +- name: Kubeadm | Warn if preflight admission check failed + debug: + msg: >- + WARNING: Preflight admission dry-run failed after {{ kubeadm_upgrade_preflight_check_retries }} retries. + The admission pipeline may not be ready. kubeadm upgrade apply may fail with [ERROR CreateJob]. + See Kubernetes issue https://github.com/kubernetes/kubernetes/issues/124237 for details. + when: + - kubeadm_upgrade_preflight_check_enabled + - inventory_hostname == first_kube_control_plane + - preflight_check_result.rc is defined and preflight_check_result.rc != 0 + - name: Kubeadm | Upgrade first control plane node to {{ kube_version }} command: >- timeout -k 600s 600s