feat: add preflight admission readiness check before kubeadm upgrade (#13140)

* feat: add preflight admission readiness check before kubeadm upgrade

  - linked to issue #13139
  - Add opt-in dry-run Job probe to verify the admission pipeline is ready before running kubeadm upgrade apply (workaround for Kubernetes #124237)
  - Gate the check behind kubeadm_upgrade_preflight_check_enabled (default false) with configurable retries and delay
  - Emit a warning instead of failing if the preflight check does not pass

* fix: refine preflight admission check Job and warning

- Use generateName for the dry-run preflight Job to avoid fixed-name collisions
- Expand the Kubernetes issue reference in the warning to a full GitHub URL

Created with AI-assistance

roles/kubernetes/control-plane/defaults/main/main.yml

@@ -37,6 +37,14 @@ kube_controller_manager_bind_address: "::"
 ## Control plane health check settings
 control_plane_health_retries: 60  # Default retries for apiserver, scheduler, controller-manager health checks
 
+# Preflight admission readiness check before kubeadm upgrade (opt-in)
+# Dry-runs a batch/v1 Job to verify the admission pipeline is ready after API server restart
+# Workaround for Kubernetes #124237: admission controllers may be stale after API server restart
+# Enable if your cluster uses admission controllers (e.g. ValidatingAdmissionPolicy) with CRD paramKind references
+kubeadm_upgrade_preflight_check_enabled: false
+kubeadm_upgrade_preflight_check_retries: 6
+kubeadm_upgrade_preflight_check_delay: 5
+
 # Leader election lease durations and timeouts for controller-manager
 kube_controller_manager_leader_elect_lease_duration: 15s
 kube_controller_manager_leader_elect_renew_deadline: 10s

roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml

@@ -2,6 +2,34 @@
 - name: Ensure kube-apiserver is up before upgrade
   import_tasks: check-api.yml
 
+- name: Kubeadm | Preflight admission readiness check
+  command:
+    cmd: "{{ kubectl }} create --dry-run=server -f -"
+    stdin: >-
+      {"apiVersion":"batch/v1","kind":"Job","metadata":{"generateName":"upgrade-preflight-probe-","namespace":"kube-system"},"spec":{"template":{"spec":{"containers":[{"name":"pause","image":"{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"}],"restartPolicy":"Never"}}}}
+  register: preflight_check_result
+  retries: "{{ kubeadm_upgrade_preflight_check_retries }}"
+  delay: "{{ kubeadm_upgrade_preflight_check_delay }}"
+  until: preflight_check_result.rc == 0
+  changed_when: false
+  failed_when: false
+  when:
+  - kubeadm_upgrade_preflight_check_enabled
+  - inventory_hostname == first_kube_control_plane
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
+
+- name: Kubeadm | Warn if preflight admission check failed
+  debug:
+    msg: >-
+      WARNING: Preflight admission dry-run failed after {{ kubeadm_upgrade_preflight_check_retries }} retries.
+      The admission pipeline may not be ready. kubeadm upgrade apply may fail with [ERROR CreateJob].
+      See Kubernetes issue https://github.com/kubernetes/kubernetes/issues/124237 for details.
+  when:
+  - kubeadm_upgrade_preflight_check_enabled
+  - inventory_hostname == first_kube_control_plane
+  - preflight_check_result.rc is defined and preflight_check_result.rc != 0
+
 - name: Kubeadm | Upgrade first control plane node to {{ kube_version }}
   command: >-
     timeout -k 600s 600s