fix: configure Kubernetes feature gates with dictionaries (#13608)

* fix: configure Kubernetes feature gates with dictionaries

* docs: update feature gate settings to dictionaries

docs/ansible/vars.md

@@ -132,8 +132,8 @@ following default cluster parameters:
 
 * *cloud_provider* - The provider for cloud services. (default is unset, Set to `external` for running with an external cloud provider)
 
-* *kube_feature_gates* - A list of key=value pairs that describe feature gates for
+* *kube_feature_gates* - A dictionary of feature gate names and boolean values for
-  alpha/experimental Kubernetes features. (defaults is `[]`).
+  alpha/experimental Kubernetes features (defaults to `{}`).
   Additionally, you can use also the following variables to individually customize your kubernetes components installation (they works exactly like `kube_feature_gates`):
   * *kube_apiserver_feature_gates*
   * *kube_controller_feature_gates*
@@ -141,8 +141,8 @@ following default cluster parameters:
   * *kube_proxy_feature_gates*
   * *kubelet_feature_gates*
 
-* *kubeadm_feature_gates* - A list of key=value pairs that describe feature gates for
+* *kubeadm_feature_gates* - A dictionary of feature gate names and boolean values for
-  alpha/experimental Kubeadm features. (defaults is `[]`)
+  alpha/experimental Kubeadm features (defaults to `{}`).
 
 * *authorization_modes* - A list of [authorization mode](
   https://kubernetes.io/docs/reference/access-authn-authz/authorization/#using-flags-for-your-authorization-module)

docs/developers/ci-setup.md

@@ -152,7 +152,7 @@ metrics_server_enabled: True
 # Enable ZSWAP
 kubelet_fail_swap_on: False
 kube_feature_gates:
-  - "NodeSwap=True"
+  NodeSwap: true
 ```
 
 ## Additional files

docs/operations/hardening.md

@@ -76,7 +76,8 @@ remove_anonymous_access: true
 ## kube-controller-manager
 kube_controller_manager_bind_address: 127.0.0.1
 kube_controller_terminated_pod_gc_threshold: 50
-kube_controller_feature_gates: ["RotateKubeletServerCertificate=true"]
+kube_controller_feature_gates:
+  RotateKubeletServerCertificate: true
 
 ## kube-scheduler
 kube_scheduler_bind_address: 127.0.0.1
@@ -98,7 +99,8 @@ kubelet_event_record_qps: 1
 kubelet_rotate_certificates: true
 kubelet_streaming_connection_idle_timeout: "5m"
 kubelet_make_iptables_util_chains: true
-kubelet_feature_gates: ["RotateKubeletServerCertificate=true"]
+kubelet_feature_gates:
+  RotateKubeletServerCertificate: true
 kubelet_seccomp_default: true
 kubelet_systemd_hardening: true
 # In case you have multiple interfaces in your

roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2

@@ -111,9 +111,7 @@ networking:
 {% endif %}
 {% if kubeadm_feature_gates %}
 featureGates:
-{%   for feature in kubeadm_feature_gates %}
+  {{ kubeadm_feature_gates | to_nice_yaml(indent=2) | indent(2) }}
-  {{ feature | replace("=", ": ") }}
-{%   endfor %}
 {% endif %}
 kubernetesVersion: v{{ kube_version }}
 {% if kube_apiserver_endpoint is defined %}
@@ -264,7 +262,7 @@ apiServer:
 {% endfor %}
 {% if kube_apiserver_feature_gates or kube_feature_gates %}
   - name: feature-gates
-    value: "{{ kube_apiserver_feature_gates | default(kube_feature_gates, true) | join(',') }}"
+    value: "{% for key, value in (kube_apiserver_feature_gates | default(kube_feature_gates, true)).items() %}{{ key }}={{ value | lower }}{{ ',' if not loop.last else '' }}{% endfor %}"
 {% endif %}
 {% if tls_min_version is defined %}
   - name: tls-min-version
@@ -402,7 +400,7 @@ controllerManager:
     value: "{{ kube_controller_manager_leader_elect_renew_deadline }}"
 {% if kube_controller_feature_gates or kube_feature_gates %}
   - name: feature-gates
-    value: "{{ kube_controller_feature_gates | default(kube_feature_gates, true) | join(',') }}"
+    value: "{% for key, value in (kube_controller_feature_gates | default(kube_feature_gates, true)).items() %}{{ key }}={{ value | lower }}{{ ',' if not loop.last else '' }}{% endfor %}"
 {% endif %}
 {% for key in kube_kubeadm_controller_extra_args %}
   - name: "{{ key }}"
@@ -441,7 +439,7 @@ scheduler:
     value: "{{ kube_config_dir }}/kubescheduler-config.yaml"
 {% if kube_scheduler_feature_gates or kube_feature_gates %}
   - name: feature-gates
-    value: "{{ kube_scheduler_feature_gates | default(kube_feature_gates, true) | join(',') }}"
+    value: "{% for key, value in (kube_scheduler_feature_gates | default(kube_feature_gates, true)).items() %}{{ key }}={{ value | lower }}{{ ',' if not loop.last else '' }}{% endfor %}"
 {% endif %}
   - name: profiling
     value: "{{ kube_profiling }}"
@@ -564,9 +562,7 @@ portRange: {{ kube_proxy_port_range }}
 {% if kube_proxy_feature_gates or kube_feature_gates %}
 {% set feature_gates = ( kube_proxy_feature_gates | default(kube_feature_gates, true) ) %}
 featureGates:
-{%   for feature in feature_gates %}
+  {{ feature_gates | to_nice_yaml(indent=2) | indent(2) }}
-  {{ feature | replace("=", ": ") }}
-{%   endfor %}
 {% endif %}
 {# DNS settings for kubelet #}
 {% if enable_nodelocaldns %}
@@ -590,19 +586,7 @@ clusterDNS:
 {% for dns_address in kubelet_cluster_dns %}
 - {{ dns_address }}
 {% endfor %}
-{% set disable_websockets_to_kubelet = (container_manager == 'docker' and kube_version is version('1.36.0', '>=')) %}
+{% if _kubelet_feature_gates %}
-{% set feature_gates = ( kubelet_feature_gates | default(kube_feature_gates, true) ) %}
-{% if disable_websockets_to_kubelet %}
-{% set feature_gates = feature_gates | reject('search', '^ExtendWebSocketsToKubelet=') | list %}
-{% endif %}
-{% if feature_gates or disable_websockets_to_kubelet %}
 featureGates:
-{%   if disable_websockets_to_kubelet %}
+  {{ _kubelet_feature_gates | to_nice_yaml(indent=2) | indent(2) }}
-  # cri-dockerd does not support WebSocket streaming
-  # https://github.com/kubernetes-sigs/kubespray/issues/13360
-  ExtendWebSocketsToKubelet: false
-{%   endif %}
-{%   for feature in feature_gates %}
-  {{ feature | replace("=", ": ") }}
-{%   endfor %}
 {% endif %}

roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2

@@ -81,21 +81,9 @@ resolvConf: "{{ kube_resolv_conf }}"
 {% if kubelet_config_extra_args %}
 {{ kubelet_config_extra_args | to_nice_yaml(indent=2) }}
 {% endif %}
-{% set disable_websockets_to_kubelet = (container_manager == 'docker' and kube_version is version('1.36.0', '>=')) %}
+{% if _kubelet_feature_gates %}
-{% set kubelet_gates = (kubelet_feature_gates | default(kube_feature_gates, true)) %}
-{% if disable_websockets_to_kubelet %}
-{% set kubelet_gates = kubelet_gates | reject('search', '^ExtendWebSocketsToKubelet=') | list %}
-{% endif %}
-{% if kubelet_gates or disable_websockets_to_kubelet %}
 featureGates:
-{% if disable_websockets_to_kubelet %}
+  {{ _kubelet_feature_gates | to_nice_yaml(indent=2) | indent(2) }}
-  # cri-dockerd does not support WebSocket streaming
-  # https://github.com/kubernetes-sigs/kubespray/issues/13360
-  ExtendWebSocketsToKubelet: false
-{% endif %}
-{% for feature in kubelet_gates %}
-  {{ feature | replace("=", ": ") }}
-{% endfor %}
 {% endif %}
 {% if tls_min_version is defined %}
 tlsMinVersion: {{ tls_min_version }}

roles/kubespray_defaults/defaults/main/main.yml

@@ -633,15 +633,14 @@ additional_sysctl: []
 #   kernel.pid_max: 0
 additional_sysctl_min: {}
 
-## List of key=value pairs that describe feature gates for
+## Dictionaries of feature gate names and boolean values for the k8s cluster.
-## the k8s cluster.
+kube_feature_gates: {}
-kube_feature_gates: []
+kube_apiserver_feature_gates: {}
-kube_apiserver_feature_gates: []
+kube_controller_feature_gates: {}
-kube_controller_feature_gates: []
+kube_scheduler_feature_gates: {}
-kube_scheduler_feature_gates: []
+kube_proxy_feature_gates: {}
-kube_proxy_feature_gates: []
+kubelet_feature_gates: {}
-kubelet_feature_gates: []
+kubeadm_feature_gates: {}
-kubeadm_feature_gates: []
 
 local_volume_provisioner_storage_class: local-storage
 local_volume_provisioner_base_dir: /mnt/disks

roles/kubespray_defaults/vars/main/main.yml

@@ -20,6 +20,15 @@ etcd_supported_versions:
 
 kube_proxy_deployed: "{{ 'addon/kube-proxy' not in kubeadm_init_phases_skip }}"
 
+# cri-dockerd does not support WebSocket streaming to the kubelet.
+# https://github.com/kubernetes-sigs/kubespray/issues/13360
+_kubelet_feature_gates: >-
+    {{ (kubelet_feature_gates | default(kube_feature_gates, true)) | combine(
+         {'ExtendWebSocketsToKubelet': false}
+         if container_manager == 'docker' and kube_version is version('1.36.0', '>=') else
+         {}
+       ) }}
+
 # Sysctl floors: only applied when the node's current value is lower than
 # the desired one, so pre-tuned images and newer kernels are not regressed.
 # Values are aligned with OpenShift Node Tuning Operator and kops, and

roles/validate_inventory/tasks/main.yml

@@ -84,6 +84,24 @@
   run_once: true
   when: not ignore_assert_errors
 
+- name: Stop if feature gates are not dictionaries of booleans
+  assert:
+    that:
+      - lookup('ansible.builtin.vars', item) is mapping
+      - lookup('ansible.builtin.vars', item) | dict2items | rejectattr('value', 'boolean') | length == 0
+    fail_msg: >-
+      {{ item }} must be a dictionary of feature gate names and boolean values.
+      Replace lists such as ["NodeSwap=true"] with {NodeSwap: true}.
+  loop:
+    - kube_feature_gates
+    - kube_apiserver_feature_gates
+    - kube_controller_feature_gates
+    - kube_scheduler_feature_gates
+    - kube_proxy_feature_gates
+    - kubelet_feature_gates
+    - kubeadm_feature_gates
+  when: not ignore_assert_errors
+
 - name: Stop if even number of etcd hosts
   assert:
     that: groups.get('etcd', groups.kube_control_plane) | length is not divisibleby 2

tests/files/fedora43-calico-swap-selinux.yml

@@ -11,4 +11,4 @@ preinstall_selinux_state: enforcing
 # Test Alpha swap feature by leveraging zswap default config in Fedora 35
 kubelet_fail_swap_on: false
 kube_feature_gates:
-  - "NodeSwap=True"
+  NodeSwap: true

tests/files/fedora44-calico-swap-selinux.yml

@@ -11,4 +11,4 @@ preinstall_selinux_state: enforcing
 # Test Alpha swap feature by leveraging zswap default config in Fedora 35
 kubelet_fail_swap_on: false
 kube_feature_gates:
-  - "NodeSwap=True"
+  NodeSwap: true

tests/files/ubuntu24-calico-all-in-one-hardening.yml

@@ -64,7 +64,8 @@ kube_profiling: false
 ## kube-controller-manager
 kube_controller_manager_bind_address: 127.0.0.1
 kube_controller_terminated_pod_gc_threshold: 50
-kube_controller_feature_gates: ["RotateKubeletServerCertificate=true"]
+kube_controller_feature_gates:
+  RotateKubeletServerCertificate: true
 
 ## kube-scheduler
 kube_scheduler_bind_address: 127.0.0.1
@@ -86,7 +87,8 @@ kubelet_event_record_qps: 1
 kubelet_rotate_certificates: true
 kubelet_streaming_connection_idle_timeout: "5m"
 kubelet_make_iptables_util_chains: true
-kubelet_feature_gates: ["RotateKubeletServerCertificate=true"]
+kubelet_feature_gates:
+  RotateKubeletServerCertificate: true
 kubelet_seccomp_default: true
 kubelet_systemd_hardening: true
 # In case you have multiple interfaces in your