Add support for ipv6 only cluster via "enable_ipv6only_stack_networks" (#11831)

roles/kubernetes/client/tasks/main.yml

@@ -71,7 +71,7 @@
     user_certs: "{{ admin_kubeconfig['users'][0]['user'] }}"
     username: "kubernetes-admin-{{ cluster_name }}"
     context: "kubernetes-admin-{{ cluster_name }}@{{ cluster_name }}"
-    override_cluster_name: "{{ {'clusters': [{'cluster': (cluster_infos | combine({'server': 'https://' + external_apiserver_address + ':' + (external_apiserver_port | string)})), 'name': cluster_name}]} }}"
+    override_cluster_name: "{{ {'clusters': [{'cluster': (cluster_infos | combine({'server': 'https://' + (external_apiserver_address | ansible.utils.ipwrap) + ':' + (external_apiserver_port | string)})), 'name': cluster_name}]} }}"
     override_context: "{{ {'contexts': [{'context': {'user': username, 'cluster': cluster_name}, 'name': context}], 'current-context': context} }}"
     override_user: "{{ {'users': [{'name': username, 'user': user_certs}]} }}"
   when: kubeconfig_localhost

roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml

@@ -43,8 +43,8 @@
 
 - name: Wait for k8s apiserver
   wait_for:
-    host: "{{ kubeadm_discovery_address.split(':')[0] }}"
-    port: "{{ kubeadm_discovery_address.split(':')[1] }}"
+    host: "{{ kubeadm_discovery_address | regex_replace('\\]?:\\d+$', '') | regex_replace('^\\[', '') }}"
+    port: "{{ kubeadm_discovery_address.split(':')[-1] }}"
     timeout: 180
 
 

roles/kubernetes/control-plane/tasks/kubeadm-setup.yml

@@ -35,6 +35,7 @@
       - "{{ kube_apiserver_ip }}"
       - "localhost"
       - "127.0.0.1"
+      - "::1"
     sans_lb: "{{ [apiserver_loadbalancer_domain_name] if apiserver_loadbalancer_domain_name is defined else [] }}"
     sans_lb_ip: "{{ [loadbalancer_apiserver.address] if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined else [] }}"
     sans_supp: "{{ supplementary_addresses_in_ssl_keys if supplementary_addresses_in_ssl_keys is defined else [] }}"

roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2

@@ -94,9 +94,9 @@ dns:
   imageTag: {{ coredns_image_tag }}
 networking:
   dnsDomain: {{ dns_domain }}
-  serviceSubnet: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks else '' }}"
+  serviceSubnet: "{{ kube_service_addresses_range }}"
 {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
-  podSubnet: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks else '' }}"
+  podSubnet: "{{ kube_pods_subnet_range }}"
 {% endif %}
 {% if kubeadm_feature_gates %}
 featureGates:
@@ -108,7 +108,7 @@ kubernetesVersion: {{ kube_version }}
 {% if kubeadm_config_api_fqdn is defined %}
 controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
 {% else %}
-controlPlaneEndpoint: {{ ip | default(fallback_ip) }}:{{ kube_apiserver_port }}
+controlPlaneEndpoint: "{{ lookup('ansible.builtin.vars', 'ip' + default_net_mode, default=hostvars[inventory_hostname]['fallback_ip' + default_net_mode]) | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}"
 {% endif %}
 certificatesDir: {{ kube_cert_dir }}
 imageRepository: {{ kube_image_repo }}
@@ -147,7 +147,7 @@ apiServer:
     etcd-servers-overrides: "/events#{{ etcd_events_access_addresses_semicolon }}"
 {% endif %}
     service-node-port-range: {{ kube_apiserver_node_port_range }}
-    service-cluster-ip-range: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks else '' }}"
+    service-cluster-ip-range: "{{ kube_service_addresses_range }}"
     kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}"
     profiling: "{{ kube_profiling }}"
     request-timeout: "{{ kube_apiserver_request_timeout }}"
@@ -294,7 +294,7 @@ apiServer:
 {% endif %}
   certSANs:
 {% for san in apiserver_sans %}
-  - "{{ san }}"
+  - {{ san }}
 {% endfor %}
   timeoutForControlPlane: 5m0s
 controllerManager:
@@ -302,18 +302,18 @@ controllerManager:
     node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
     node-monitor-period: {{ kube_controller_node_monitor_period }}
 {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
-    cluster-cidr: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks else '' }}"
+    cluster-cidr: "{{ kube_pods_subnet_range }}"
 {% endif %}
-    service-cluster-ip-range: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks else '' }}"
+    service-cluster-ip-range: "{{ kube_service_addresses_range }}"
 {% if kube_network_plugin is defined and kube_network_plugin == "calico" and not calico_ipam_host_local %}
     allocate-node-cidrs: "false"
-{% else %}
-{% if enable_dual_stack_networks %}
+{% elif enable_ipv6only_stack_networks %}
+    node-cidr-mask-size-ipv6: "{{ kube_network_node_prefix_ipv6 }}"
+{% elif enable_dual_stack_networks %}
     node-cidr-mask-size-ipv4: "{{ kube_network_node_prefix }}"
     node-cidr-mask-size-ipv6: "{{ kube_network_node_prefix_ipv6 }}"
 {% else %}
     node-cidr-mask-size: "{{ kube_network_node_prefix }}"
-{% endif %}
 {% endif %}
     profiling: "{{ kube_profiling }}"
     terminated-pod-gc-threshold: "{{ kube_controller_terminated_pod_gc_threshold }}"
@@ -392,7 +392,7 @@ clientConnection:
   kubeconfig: {{ kube_proxy_client_kubeconfig }}
   qps: {{ kube_proxy_client_qps }}
 {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
-clusterCIDR: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks else '' }}"
+clusterCIDR: "{{ kube_pods_subnet_range }}"
 {% endif %}
 configSyncPeriod: {{ kube_proxy_config_sync_period }}
 conntrack:

roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2

@@ -106,9 +106,9 @@ dns:
   imageTag: {{ coredns_image_tag }}
 networking:
   dnsDomain: {{ dns_domain }}
-  serviceSubnet: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks else '' }}"
+  serviceSubnet: "{{ kube_service_addresses_range }}"
 {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
-  podSubnet: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks else '' }}"
+  podSubnet: "{{ kube_pods_subnet_range }}"
 {% endif %}
 {% if kubeadm_feature_gates %}
 featureGates:
@@ -120,7 +120,7 @@ kubernetesVersion: {{ kube_version }}
 {% if kubeadm_config_api_fqdn is defined %}
 controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
 {% else %}
-controlPlaneEndpoint: {{ ip | default(fallback_ip) }}:{{ kube_apiserver_port }}
+controlPlaneEndpoint: "{{ lookup('ansible.builtin.vars', 'ip' + default_net_mode, default=hostvars[inventory_hostname]['fallback_ip' + default_net_mode]) | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}"
 {% endif %}
 certificatesDir: {{ kube_cert_dir }}
 imageRepository: {{ kube_image_repo }}
@@ -174,7 +174,7 @@ apiServer:
   - name: service-node-port-range
     value: "{{ kube_apiserver_node_port_range }}"
   - name: service-cluster-ip-range
-    value: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks else '' }}"
+    value: "{{ kube_service_addresses_range }}"
   - name: kubelet-preferred-address-types
     value: "{{ kubelet_preferred_address_types }}"
   - name: profiling
@@ -351,7 +351,7 @@ apiServer:
 {% endif %}
   certSANs:
 {% for san in apiserver_sans %}
-  - "{{ san }}"
+  - {{ san }}
 {% endfor %}
 controllerManager:
   extraArgs:
@@ -361,15 +361,17 @@ controllerManager:
     value: "{{ kube_controller_node_monitor_period }}"
 {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
   - name: cluster-cidr
-    value: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks else '' }}"
+    value: "{{ kube_pods_subnet_range }}"
 {% endif %}
   - name: service-cluster-ip-range
-    value: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks else '' }}"
+    value: "{{ kube_service_addresses_range }}"
 {% if kube_network_plugin is defined and kube_network_plugin == "calico" and not calico_ipam_host_local %}
   - name: allocate-node-cidrs
     value: "false"
-{% else %}
-{% if enable_dual_stack_networks %}
+{% elif enable_ipv6only_stack_networks %}
+  - name: node-cidr-mask-size-ipv6
+    value: "{{ kube_network_node_prefix_ipv6 }}"
+{% elif enable_dual_stack_networks %}
   - name: node-cidr-mask-size-ipv4
     value: "{{ kube_network_node_prefix }}"
   - name: node-cidr-mask-size-ipv6
@@ -377,7 +379,6 @@ controllerManager:
 {% else %}
   - name: node-cidr-mask-size
     value: "{{ kube_network_node_prefix }}"
-{% endif %}
 {% endif %}
   - name: profiling
     value: "{{ kube_profiling }}"
@@ -488,7 +489,7 @@ clientConnection:
   kubeconfig: {{ kube_proxy_client_kubeconfig }}
   qps: {{ kube_proxy_client_qps }}
 {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
-clusterCIDR: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks else '' }}"
+clusterCIDR: "{{ kube_pods_subnet_range }}"
 {% endif %}
 configSyncPeriod: {{ kube_proxy_config_sync_period }}
 conntrack:

roles/kubernetes/control-plane/templates/kubeadm-controlplane.yaml.j2

@@ -7,9 +7,9 @@ discovery:
 {% else %}
   bootstrapToken:
 {% if kubeadm_config_api_fqdn is defined %}
-    apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
+    apiServerEndpoint: "{{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}"
 {% else %}
-    apiServerEndpoint: {{ kubeadm_discovery_address }}
+    apiServerEndpoint: "{{ kubeadm_discovery_address }}"
 {% endif %}
     token: {{ kubeadm_token }}
     unsafeSkipCAVerification: true

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2

@@ -8,9 +8,9 @@ discovery:
 {% else %}
   bootstrapToken:
 {% if kubeadm_config_api_fqdn is defined %}
-    apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
+    apiServerEndpoint: "{{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}"
 {% else %}
-    apiServerEndpoint: {{ kubeadm_discovery_address }}
+    apiServerEndpoint: "{{ kubeadm_discovery_address }}"
 {% endif %}
     token: {{ kubeadm_token }}
 {% if ca_cert_content is defined %}

roles/kubernetes/node/defaults/main.yml

@@ -1,9 +1,9 @@
 ---
 # advertised host IP for kubelet. This affects network plugin config. Take caution
-kubelet_address: "{{ ip | default(fallback_ip) }}{{ (',' + ip6) if enable_dual_stack_networks and ip6 is defined else '' }}"
+kubelet_address: "{{ [ip | default(fallback_ip) if not enable_ipv6only_stack_networks, ip6 | default(fallback_ip6) if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}"
 
 # bind address for kubelet. Set to 0.0.0.0 to listen on all interfaces
-kubelet_bind_address: "{{ ip | default('0.0.0.0') }}"
+kubelet_bind_address: "{{ (ip6 if enable_ipv6only_stack_networks else ip) | default('0.0.0.0') }}"
 
 # resolv.conf to base dns config
 kube_resolv_conf: "/etc/resolv.conf"
@@ -29,7 +29,7 @@ kubelet_systemd_wants_dependencies: []
 # List of secure IPs for kubelet
 kube_node_addresses: >-
   {%- for host in (groups['k8s_cluster'] | union(groups['etcd'])) -%}
-    {{ hostvars[host]['ip'] | default(hostvars[host]['fallback_ip']) }}{{ ' ' if not loop.last else '' }}
+    {{ hostvars[host]['ip' + default_net_mode] | default(hostvars[host]['fallback_ip' + default_net_mode]) | ansible.utils.ipwrap }}{{ ' ' if not loop.last else '' }}
   {%- endfor -%}
 kubelet_secure_addresses: "localhost link-local {{ kube_pods_subnet }} {{ kube_node_addresses }}"
 

roles/kubernetes/node/templates/loadbalancer/haproxy.cfg.j2

@@ -22,7 +22,7 @@ defaults
 {% if loadbalancer_apiserver_healthcheck_port is defined -%}
 frontend healthz
   bind 0.0.0.0:{{ loadbalancer_apiserver_healthcheck_port }}
-  {% if enable_dual_stack_networks -%}
+  {% if enable_dual_stack_networks or enable_ipv6only_stack_networks -%}
   bind :::{{ loadbalancer_apiserver_healthcheck_port }}
   {% endif -%}
   mode http
@@ -31,7 +31,7 @@ frontend healthz
 
 frontend kube_api_frontend
   bind 127.0.0.1:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }}
-  {% if enable_dual_stack_networks -%}
+  {% if enable_dual_stack_networks or enable_ipv6only_stack_networks -%}
   bind [::1]:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};
   {% endif -%}
   mode tcp

roles/kubernetes/node/templates/loadbalancer/nginx.conf.j2

@@ -20,7 +20,7 @@ stream {
 
   server {
     listen        127.0.0.1:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};
-    {% if enable_dual_stack_networks -%}
+    {% if enable_dual_stack_networks or enable_ipv6only_stack_networks -%}
     listen        [::1]:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};
     {% endif -%}
     proxy_pass    kube_apiserver;
@@ -44,7 +44,7 @@ http {
   {% if loadbalancer_apiserver_healthcheck_port is defined -%}
   server {
     listen {{ loadbalancer_apiserver_healthcheck_port }};
-    {% if enable_dual_stack_networks -%}
+    {% if enable_dual_stack_networks or enable_ipv6only_stack_networks -%}
     listen [::]:{{ loadbalancer_apiserver_healthcheck_port }};
     {% endif -%}
     location /healthz {

roles/kubernetes/preinstall/tasks/0080-system-configurations.yml

@@ -84,7 +84,7 @@
     value: "1"
     state: present
     reload: true
-  when: enable_dual_stack_networks | bool
+  when: (enable_dual_stack_networks or enable_ipv6only_stack_networks) | bool
 
 - name: Check if we need to set fs.may_detach_mounts
   stat:

roles/kubernetes/preinstall/tasks/0090-etchosts.yml

@@ -3,10 +3,15 @@
   set_fact:
     etc_hosts_inventory_block: |-
       {% for item in (groups['k8s_cluster'] + groups['etcd'] | default([]) + groups['calico_rr'] | default([])) | unique -%}
-      {% if 'access_ip' in hostvars[item] or 'ip' in hostvars[item] or 'ansible_default_ipv4' in hostvars[item] -%}
+      {% if 'access_ip' in hostvars[item] or 'ip' in hostvars[item] or hostvars[item]['ansible_default_ipv4'] -%}
       {{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item]['ansible_default_ipv4']['address'])) }}
       {%- if ('ansible_hostname' in hostvars[item] and item != hostvars[item]['ansible_hostname']) %} {{ hostvars[item]['ansible_hostname'] }}.{{ dns_domain }} {{ hostvars[item]['ansible_hostname'] }} {% else %} {{ item }}.{{ dns_domain }} {{ item }} {% endif %}
 
+      {% endif %}
+      {% if ('access_ip6' in hostvars[item] or 'ip6' in hostvars[item] or hostvars[item]['ansible_default_ipv6']) and (enable_ipv6only_stack_networks or enable_dual_stack_networks) -%}
+      {{ hostvars[item]['access_ip6'] | default(hostvars[item]['ip6'] | default(hostvars[item]['ansible_default_ipv6']['address'])) }}
+      {%- if ('ansible_hostname' in hostvars[item] and item != hostvars[item]['ansible_hostname']) %} {{ hostvars[item]['ansible_hostname'] }}.{{ dns_domain }} {{ hostvars[item]['ansible_hostname'] }} {% else %} {{ item }}.{{ dns_domain }} {{ item }} {% endif %}
+
       {% endif %}
       {% endfor %}
   delegate_to: localhost