diff --git a/roles/kubernetes/control-plane/defaults/main/main.yml b/roles/kubernetes/control-plane/defaults/main/main.yml index 34671cd02..ed49545ab 100644 --- a/roles/kubernetes/control-plane/defaults/main/main.yml +++ b/roles/kubernetes/control-plane/defaults/main/main.yml @@ -37,6 +37,14 @@ kube_controller_manager_bind_address: "::" ## Control plane health check settings control_plane_health_retries: 60 # Default retries for apiserver, scheduler, controller-manager health checks +# Preflight admission readiness check before kubeadm upgrade (opt-in) +# Dry-runs a batch/v1 Job to verify the admission pipeline is ready after API server restart +# Workaround for Kubernetes #124237: admission controllers may be stale after API server restart +# Enable if your cluster uses admission controllers (e.g. ValidatingAdmissionPolicy) with CRD paramKind references +kubeadm_upgrade_preflight_check_enabled: false +kubeadm_upgrade_preflight_check_retries: 6 +kubeadm_upgrade_preflight_check_delay: 5 + # Leader election lease durations and timeouts for controller-manager kube_controller_manager_leader_elect_lease_duration: 15s kube_controller_manager_leader_elect_renew_deadline: 10s diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml b/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml index 1ffd18153..a602c4895 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml @@ -2,6 +2,34 @@ - name: Ensure kube-apiserver is up before upgrade import_tasks: check-api.yml +- name: Kubeadm | Preflight admission readiness check + command: + cmd: "{{ kubectl }} create --dry-run=server -f -" + stdin: >- + {"apiVersion":"batch/v1","kind":"Job","metadata":{"generateName":"upgrade-preflight-probe-","namespace":"kube-system"},"spec":{"template":{"spec":{"containers":[{"name":"pause","image":"{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"}],"restartPolicy":"Never"}}}} + register: preflight_check_result + retries: "{{ kubeadm_upgrade_preflight_check_retries }}" + delay: "{{ kubeadm_upgrade_preflight_check_delay }}" + until: preflight_check_result.rc == 0 + changed_when: false + failed_when: false + when: + - kubeadm_upgrade_preflight_check_enabled + - inventory_hostname == first_kube_control_plane + environment: + PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}" + +- name: Kubeadm | Warn if preflight admission check failed + debug: + msg: >- + WARNING: Preflight admission dry-run failed after {{ kubeadm_upgrade_preflight_check_retries }} retries. + The admission pipeline may not be ready. kubeadm upgrade apply may fail with [ERROR CreateJob]. + See Kubernetes issue https://github.com/kubernetes/kubernetes/issues/124237 for details. + when: + - kubeadm_upgrade_preflight_check_enabled + - inventory_hostname == first_kube_control_plane + - preflight_check_result.rc is defined and preflight_check_result.rc != 0 + - name: Kubeadm | Upgrade first control plane node to {{ kube_version }} command: >- timeout -k 600s 600s