refact ip stack (#11953)

roles/kubernetes/client/tasks/main.yml

@@ -71,7 +71,7 @@
     user_certs: "{{ admin_kubeconfig['users'][0]['user'] }}"
     username: "kubernetes-admin-{{ cluster_name }}"
     context: "kubernetes-admin-{{ cluster_name }}@{{ cluster_name }}"
-    override_cluster_name: "{{ {'clusters': [{'cluster': (cluster_infos | combine({'server': 'https://' + external_apiserver_address + ':' + (external_apiserver_port | string)})), 'name': cluster_name}]} }}"
+    override_cluster_name: "{{ {'clusters': [{'cluster': (cluster_infos | combine({'server': 'https://' + (external_apiserver_address | ansible.utils.ipwrap) + ':' + (external_apiserver_port | string)})), 'name': cluster_name}]} }}"
     override_context: "{{ {'contexts': [{'context': {'user': username, 'cluster': cluster_name}, 'name': context}], 'current-context': context} }}"
     override_user: "{{ {'users': [{'name': username, 'user': user_certs}]} }}"
   when: kubeconfig_localhost

roles/kubernetes/control-plane/defaults/main/kube-scheduler.yml

@@ -4,7 +4,7 @@ kube_kubeadm_scheduler_extra_args: {}
 
 # Associated interface must be reachable by the rest of the cluster, and by
 # CLI/web clients.
-kube_scheduler_bind_address: 0.0.0.0
+kube_scheduler_bind_address: "::"
 
 # ClientConnection options (e.g. Burst, QPS) except from kubeconfig.
 kube_scheduler_client_conn_extra_opts: {}

roles/kubernetes/control-plane/defaults/main/main.yml

@@ -6,7 +6,7 @@ upgrade_cluster_setup: false
 # listen on a specific address/interface.
 # NOTE: If you specific address/interface and use loadbalancer_apiserver_localhost
 # loadbalancer_apiserver_localhost (nginx/haproxy) will deploy on control plane nodes on 127.0.0.1:{{ loadbalancer_apiserver_port | default(kube_apiserver_port) }} too.
-kube_apiserver_bind_address: 0.0.0.0
+kube_apiserver_bind_address: "::"
 
 # A port range to reserve for services with NodePort visibility.
 # Inclusive at both ends of the range.
@@ -29,7 +29,7 @@ kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem
 
 # Associated interfaces must be reachable by the rest of the cluster, and by
 # CLI/web clients.
-kube_controller_manager_bind_address: 0.0.0.0
+kube_controller_manager_bind_address: "::"
 
 # Leader election lease durations and timeouts for controller-manager
 kube_controller_manager_leader_elect_lease_duration: 15s
@@ -242,7 +242,7 @@ kubeadm_upgrade_auto_cert_renewal: true
 
 ## Enable distributed tracing for kube-apiserver
 kube_apiserver_tracing: false
-kube_apiserver_tracing_endpoint: 0.0.0.0:4317
+kube_apiserver_tracing_endpoint: "[::]:4317"
 kube_apiserver_tracing_sampling_rate_per_million: 100
 
 # Enable kubeadm file discovery if anonymous access has been removed

roles/kubernetes/control-plane/handlers/main.yml

@@ -78,7 +78,7 @@
 
 - name: Control plane | wait for kube-scheduler
   vars:
-    endpoint: "{{ kube_scheduler_bind_address if kube_scheduler_bind_address != '0.0.0.0' else 'localhost' }}"
+    endpoint: "{{ kube_scheduler_bind_address if kube_scheduler_bind_address != '::' else 'localhost' }}"
   uri:
     url: https://{{ endpoint }}:10259/healthz
     validate_certs: false
@@ -92,7 +92,7 @@
 
 - name: Control plane | wait for kube-controller-manager
   vars:
-    endpoint: "{{ kube_controller_manager_bind_address if kube_controller_manager_bind_address != '0.0.0.0' else 'localhost' }}"
+    endpoint: "{{ kube_controller_manager_bind_address if kube_controller_manager_bind_address != '::' else 'localhost' }}"
   uri:
     url: https://{{ endpoint }}:10257/healthz
     validate_certs: false

roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml

@@ -4,7 +4,7 @@
     # noqa: jinja[spacing]
     kubeadm_discovery_address: >-
       {%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%}
-      {{ first_kube_control_plane_address }}:{{ kube_apiserver_port }}
+      {{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}
       {%- else -%}
       {{ kube_apiserver_endpoint | regex_replace('https://', '') }}
       {%- endif %}
@@ -43,8 +43,8 @@
 
 - name: Wait for k8s apiserver
   wait_for:
-    host: "{{ kubeadm_discovery_address.split(':')[0] }}"
-    port: "{{ kubeadm_discovery_address.split(':')[1] }}"
+    host: "{{ kubeadm_discovery_address | regex_replace('\\]?:\\d+$', '') | regex_replace('^\\[', '') }}"
+    port: "{{ kubeadm_discovery_address.split(':')[-1] }}"
     timeout: 180
 
 

roles/kubernetes/control-plane/tasks/kubeadm-setup.yml

@@ -35,12 +35,13 @@
       - "{{ kube_apiserver_ip }}"
       - "localhost"
       - "127.0.0.1"
+      - "::1"
     sans_lb: "{{ [apiserver_loadbalancer_domain_name] if apiserver_loadbalancer_domain_name is defined else [] }}"
     sans_lb_ip: "{{ [loadbalancer_apiserver.address] if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined else [] }}"
     sans_supp: "{{ supplementary_addresses_in_ssl_keys if supplementary_addresses_in_ssl_keys is defined else [] }}"
-    sans_access_ip: "{{ groups['kube_control_plane'] | map('extract', hostvars, 'access_ip') | list | select('defined') | list }}"
-    sans_ip: "{{ groups['kube_control_plane'] | map('extract', hostvars, 'ip') | list | select('defined') | list }}"
-    sans_address: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | list | select('defined') | list }}"
+    sans_access_ip: "{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_access_ip') | list | select('defined') | list }}"
+    sans_ip: "{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_ip') | list | select('defined') | list }}"
+    sans_address: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_default_ipv6', 'ansible_default_ipv4', 'address']) | list | select('defined') | list }}"
     sans_override: "{{ [kube_override_hostname] if kube_override_hostname else [] }}"
     sans_hostname: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_hostname']) | list | select('defined') | list }}"
     sans_fqdn: "{{ groups['kube_control_plane'] | map('extract', hostvars, ['ansible_fqdn']) | list | select('defined') | list }}"

roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml

@@ -1,7 +1,7 @@
 ---
 - name: Kubeadm | Check api is up
   uri:
-    url: "https://{{ ip | default(fallback_ip) }}:{{ kube_apiserver_port }}/healthz"
+    url: "https://{{ main_ip | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}/healthz"
     validate_certs: false
   when: ('kube_control_plane' in group_names)
   register: _result

roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2

@@ -7,7 +7,7 @@ bootstrapTokens:
   ttl: "24h"
 {% endif %}
 localAPIEndpoint:
-  advertiseAddress: {{ kube_apiserver_address }}
+  advertiseAddress: "{{ kube_apiserver_address }}"
   bindPort: {{ kube_apiserver_port }}
 {% if kubeadm_certificate_key is defined %}
 certificateKey: {{ kubeadm_certificate_key }}
@@ -41,7 +41,7 @@ etcd:
   external:
       endpoints:
 {% for endpoint in etcd_access_addresses.split(',') %}
-      - {{ endpoint }}
+      - "{{ endpoint }}"
 {% endfor %}
       caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }}
       certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }}
@@ -94,9 +94,9 @@ dns:
   imageTag: {{ coredns_image_tag }}
 networking:
   dnsDomain: {{ dns_domain }}
-  serviceSubnet: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks else '' }}"
+  serviceSubnet: "{{ kube_service_subnets }}"
 {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
-  podSubnet: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks else '' }}"
+  podSubnet: "{{ kube_pods_subnets }}"
 {% endif %}
 {% if kubeadm_feature_gates %}
 featureGates:
@@ -106,9 +106,9 @@ featureGates:
 {% endif %}
 kubernetesVersion: {{ kube_version }}
 {% if kubeadm_config_api_fqdn is defined %}
-controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
+controlPlaneEndpoint: "{{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}"
 {% else %}
-controlPlaneEndpoint: {{ ip | default(fallback_ip) }}:{{ kube_apiserver_port }}
+controlPlaneEndpoint: "{{ main_ip | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}"
 {% endif %}
 certificatesDir: {{ kube_cert_dir }}
 imageRepository: {{ kube_image_repo }}
@@ -131,7 +131,7 @@ apiServer:
 {% else %}
     authorization-mode: {{ authorization_modes | join(',') }}
 {% endif %}
-    bind-address: {{ kube_apiserver_bind_address }}
+    bind-address: "{{ kube_apiserver_bind_address }}"
 {% if kube_apiserver_enable_admission_plugins | length > 0 %}
     enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }}
 {% endif %}
@@ -147,7 +147,7 @@ apiServer:
     etcd-servers-overrides: "/events#{{ etcd_events_access_addresses_semicolon }}"
 {% endif %}
     service-node-port-range: {{ kube_apiserver_node_port_range }}
-    service-cluster-ip-range: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks else '' }}"
+    service-cluster-ip-range: "{{ kube_service_subnets }}"
     kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}"
     profiling: "{{ kube_profiling }}"
     request-timeout: "{{ kube_apiserver_request_timeout }}"
@@ -294,7 +294,7 @@ apiServer:
 {% endif %}
   certSANs:
 {% for san in apiserver_sans %}
-  - "{{ san }}"
+  - {{ san }}
 {% endfor %}
   timeoutForControlPlane: 5m0s
 controllerManager:
@@ -302,22 +302,22 @@ controllerManager:
     node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
     node-monitor-period: {{ kube_controller_node_monitor_period }}
 {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
-    cluster-cidr: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks else '' }}"
+    cluster-cidr: "{{ kube_pods_subnets }}"
 {% endif %}
-    service-cluster-ip-range: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks else '' }}"
+    service-cluster-ip-range: "{{ kube_service_subnets }}"
 {% if kube_network_plugin is defined and kube_network_plugin == "calico" and not calico_ipam_host_local %}
     allocate-node-cidrs: "false"
 {% else %}
-{% if enable_dual_stack_networks %}
+{% if ipv4_stack %}
     node-cidr-mask-size-ipv4: "{{ kube_network_node_prefix }}"
+{% endif %}
+{% if ipv6_stack %}
     node-cidr-mask-size-ipv6: "{{ kube_network_node_prefix_ipv6 }}"
-{% else %}
-    node-cidr-mask-size: "{{ kube_network_node_prefix }}"
 {% endif %}
 {% endif %}
     profiling: "{{ kube_profiling }}"
     terminated-pod-gc-threshold: "{{ kube_controller_terminated_pod_gc_threshold }}"
-    bind-address: {{ kube_controller_manager_bind_address }}
+    bind-address: "{{ kube_controller_manager_bind_address }}"
     leader-elect-lease-duration: {{ kube_controller_manager_leader_elect_lease_duration }}
     leader-elect-renew-deadline: {{ kube_controller_manager_leader_elect_renew_deadline }}
 {% if kube_controller_feature_gates or kube_feature_gates %}
@@ -350,7 +350,7 @@ controllerManager:
 {% endif %}
 scheduler:
   extraArgs:
-    bind-address: {{ kube_scheduler_bind_address }}
+    bind-address: "{{ kube_scheduler_bind_address }}"
     config: {{ kube_config_dir }}/kubescheduler-config.yaml
 {% if kube_scheduler_feature_gates or kube_feature_gates %}
     feature-gates: "{{ kube_scheduler_feature_gates | default(kube_feature_gates, true) | join(',') }}"
@@ -384,7 +384,7 @@ scheduler:
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
-bindAddress: {{ kube_proxy_bind_address }}
+bindAddress: "{{ kube_proxy_bind_address }}"
 clientConnection:
   acceptContentTypes: {{ kube_proxy_client_accept_content_types }}
   burst: {{ kube_proxy_client_burst }}
@@ -392,7 +392,7 @@ clientConnection:
   kubeconfig: {{ kube_proxy_client_kubeconfig }}
   qps: {{ kube_proxy_client_qps }}
 {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
-clusterCIDR: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks else '' }}"
+clusterCIDR: "{{ kube_pods_subnets }}"
 {% endif %}
 configSyncPeriod: {{ kube_proxy_config_sync_period }}
 conntrack:
@@ -401,7 +401,7 @@ conntrack:
   tcpCloseWaitTimeout: {{ kube_proxy_conntrack_tcp_close_wait_timeout }}
   tcpEstablishedTimeout: {{ kube_proxy_conntrack_tcp_established_timeout }}
 enableProfiling: {{ kube_proxy_enable_profiling }}
-healthzBindAddress: {{ kube_proxy_healthz_bind_address }}
+healthzBindAddress: "{{ kube_proxy_healthz_bind_address }}"
 hostnameOverride: "{{ kube_override_hostname }}"
 iptables:
   masqueradeAll: {{ kube_proxy_masquerade_all }}
@@ -417,7 +417,7 @@ ipvs:
   tcpTimeout: {{ kube_proxy_tcp_timeout }}
   tcpFinTimeout: {{ kube_proxy_tcp_fin_timeout }}
   udpTimeout: {{ kube_proxy_udp_timeout }}
-metricsBindAddress: {{ kube_proxy_metrics_bind_address }}
+metricsBindAddress: "{{ kube_proxy_metrics_bind_address }}"
 mode: {{ kube_proxy_mode }}
 nodePortAddresses: {{ kube_proxy_nodeport_addresses }}
 oomScoreAdj: {{ kube_proxy_oom_score_adj }}

roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2

@@ -7,7 +7,7 @@ bootstrapTokens:
   ttl: "24h"
 {% endif %}
 localAPIEndpoint:
-  advertiseAddress: {{ kube_apiserver_address }}
+  advertiseAddress: "{{ kube_apiserver_address }}"
   bindPort: {{ kube_apiserver_port }}
 {% if kubeadm_certificate_key is defined %}
 certificateKey: {{ kubeadm_certificate_key }}
@@ -43,7 +43,7 @@ etcd:
   external:
       endpoints:
 {% for endpoint in etcd_access_addresses.split(',') %}
-      - {{ endpoint }}
+      - "{{ endpoint }}"
 {% endfor %}
       caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }}
       certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }}
@@ -106,9 +106,9 @@ dns:
   imageTag: {{ coredns_image_tag }}
 networking:
   dnsDomain: {{ dns_domain }}
-  serviceSubnet: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks else '' }}"
+  serviceSubnet: "{{ kube_service_subnets }}"
 {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
-  podSubnet: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks else '' }}"
+  podSubnet: "{{ kube_pods_subnets }}"
 {% endif %}
 {% if kubeadm_feature_gates %}
 featureGates:
@@ -118,9 +118,9 @@ featureGates:
 {% endif %}
 kubernetesVersion: {{ kube_version }}
 {% if kubeadm_config_api_fqdn is defined %}
-controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
+controlPlaneEndpoint: "{{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}"
 {% else %}
-controlPlaneEndpoint: {{ ip | default(fallback_ip) }}:{{ kube_apiserver_port }}
+controlPlaneEndpoint: "{{ main_ip | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}"
 {% endif %}
 certificatesDir: {{ kube_cert_dir }}
 imageRepository: {{ kube_image_repo }}
@@ -174,7 +174,7 @@ apiServer:
   - name: service-node-port-range
     value: "{{ kube_apiserver_node_port_range }}"
   - name: service-cluster-ip-range
-    value: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks else '' }}"
+    value: "{{ kube_service_subnets }}"
   - name: kubelet-preferred-address-types
     value: "{{ kubelet_preferred_address_types }}"
   - name: profiling
@@ -351,7 +351,7 @@ apiServer:
 {% endif %}
   certSANs:
 {% for san in apiserver_sans %}
-  - "{{ san }}"
+  - {{ san }}
 {% endfor %}
 controllerManager:
   extraArgs:
@@ -361,22 +361,21 @@ controllerManager:
     value: "{{ kube_controller_node_monitor_period }}"
 {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
   - name: cluster-cidr
-    value: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks else '' }}"
+    value: "{{ kube_pods_subnets }}"
 {% endif %}
   - name: service-cluster-ip-range
-    value: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks else '' }}"
+    value: "{{ kube_service_subnets }}"
 {% if kube_network_plugin is defined and kube_network_plugin == "calico" and not calico_ipam_host_local %}
   - name: allocate-node-cidrs
     value: "false"
 {% else %}
-{% if enable_dual_stack_networks %}
+{% if ipv4_stack %}
   - name: node-cidr-mask-size-ipv4
     value: "{{ kube_network_node_prefix }}"
+{% endif %}
+{% if ipv6_stack %}
   - name: node-cidr-mask-size-ipv6
     value: "{{ kube_network_node_prefix_ipv6 }}"
-{% else %}
-  - name: node-cidr-mask-size
-    value: "{{ kube_network_node_prefix }}"
 {% endif %}
 {% endif %}
   - name: profiling
@@ -480,7 +479,7 @@ scheduler:
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
-bindAddress: {{ kube_proxy_bind_address }}
+bindAddress: "{{ kube_proxy_bind_address }}"
 clientConnection:
   acceptContentTypes: {{ kube_proxy_client_accept_content_types }}
   burst: {{ kube_proxy_client_burst }}
@@ -488,7 +487,7 @@ clientConnection:
   kubeconfig: {{ kube_proxy_client_kubeconfig }}
   qps: {{ kube_proxy_client_qps }}
 {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
-clusterCIDR: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks else '' }}"
+clusterCIDR: "{{ kube_pods_subnets }}"
 {% endif %}
 configSyncPeriod: {{ kube_proxy_config_sync_period }}
 conntrack:
@@ -497,7 +496,7 @@ conntrack:
   tcpCloseWaitTimeout: {{ kube_proxy_conntrack_tcp_close_wait_timeout }}
   tcpEstablishedTimeout: {{ kube_proxy_conntrack_tcp_established_timeout }}
 enableProfiling: {{ kube_proxy_enable_profiling }}
-healthzBindAddress: {{ kube_proxy_healthz_bind_address }}
+healthzBindAddress: "{{ kube_proxy_healthz_bind_address }}"
 hostnameOverride: "{{ kube_override_hostname }}"
 iptables:
   masqueradeAll: {{ kube_proxy_masquerade_all }}
@@ -513,7 +512,7 @@ ipvs:
   tcpTimeout: {{ kube_proxy_tcp_timeout }}
   tcpFinTimeout: {{ kube_proxy_tcp_fin_timeout }}
   udpTimeout: {{ kube_proxy_udp_timeout }}
-metricsBindAddress: {{ kube_proxy_metrics_bind_address }}
+metricsBindAddress: "{{ kube_proxy_metrics_bind_address }}"
 mode: {{ kube_proxy_mode }}
 nodePortAddresses: {{ kube_proxy_nodeport_addresses }}
 oomScoreAdj: {{ kube_proxy_oom_score_adj }}

roles/kubernetes/control-plane/templates/kubeadm-controlplane.yaml.j2

@@ -9,7 +9,7 @@ discovery:
 {% if kubeadm_config_api_fqdn is defined %}
     apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
 {% else %}
-    apiServerEndpoint: {{ kubeadm_discovery_address }}
+    apiServerEndpoint: "{{ kubeadm_discovery_address }}"
 {% endif %}
     token: {{ kubeadm_token }}
     unsafeSkipCAVerification: true
@@ -24,7 +24,7 @@ timeouts:
 {% endif %}
 controlPlane:
   localAPIEndpoint:
-    advertiseAddress: {{ kube_apiserver_address }}
+    advertiseAddress: "{{ kube_apiserver_address }}"
     bindPort: {{ kube_apiserver_port }}
   certificateKey: {{ kubeadm_certificate_key }}
 nodeRegistration:

roles/kubernetes/kubeadm/tasks/main.yml

@@ -4,7 +4,7 @@
     # noqa: jinja[spacing]
     kubeadm_discovery_address: >-
       {%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%}
-      {{ first_kube_control_plane_address }}:{{ kube_apiserver_port }}
+      {{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}
       {%- else -%}
       {{ kube_apiserver_endpoint | replace("https://", "") }}
       {%- endif %}

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2

@@ -8,9 +8,9 @@ discovery:
 {% else %}
   bootstrapToken:
 {% if kubeadm_config_api_fqdn is defined %}
-    apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
+    apiServerEndpoint: "{{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}"
 {% else %}
-    apiServerEndpoint: {{ kubeadm_discovery_address }}
+    apiServerEndpoint: "{{ kubeadm_discovery_address }}"
 {% endif %}
     token: {{ kubeadm_token }}
 {% if ca_cert_content is defined %}
@@ -32,7 +32,7 @@ caCertPath: {{ kube_cert_dir }}/ca.crt
 {% if kubeadm_cert_controlplane is defined and kubeadm_cert_controlplane %}
 controlPlane:
   localAPIEndpoint:
-    advertiseAddress: {{ kube_apiserver_address }}
+    advertiseAddress: "{{ kube_apiserver_address }}"
     bindPort: {{ kube_apiserver_port }}
   certificateKey: {{ kubeadm_certificate_key }}
 {% endif %}

roles/kubernetes/node/defaults/main.yml

@@ -1,9 +1,10 @@
 ---
 # advertised host IP for kubelet. This affects network plugin config. Take caution
-kubelet_address: "{{ ip | default(fallback_ip) }}{{ (',' + ip6) if enable_dual_stack_networks and ip6 is defined else '' }}"
+# add ipv6 manual for dualstack mode because ipv4 priority in main_ip for dualstack
+kubelet_address: "{{ main_ips | join(',') }}"
 
-# bind address for kubelet. Set to 0.0.0.0 to listen on all interfaces
-kubelet_bind_address: "{{ ip | default('0.0.0.0') }}"
+# bind address for kubelet. Set to :: to listen on all interfaces
+kubelet_bind_address: "{{ main_ip | default('::') }}"
 
 # resolv.conf to base dns config
 kube_resolv_conf: "/etc/resolv.conf"
@@ -27,11 +28,12 @@ kubelet_systemd_hardening: false
 kubelet_systemd_wants_dependencies: []
 
 # List of secure IPs for kubelet
+# don't forget ipv6 addresses for dualstack(because "main_ip" prioritizes ipv4)
 kube_node_addresses: >-
   {%- for host in (groups['k8s_cluster'] | union(groups['etcd'])) -%}
-    {{ hostvars[host]['ip'] | default(hostvars[host]['fallback_ip']) }}{{ ' ' if not loop.last else '' }}
+    {{ hostvars[host]['main_ips'] | join(' ') }}{{ ' ' if not loop.last else '' }}
   {%- endfor -%}
-kubelet_secure_addresses: "localhost link-local {{ kube_pods_subnet }} {{ kube_node_addresses }}"
+kubelet_secure_addresses: "localhost link-local {{ kube_pods_subnets | regex_replace(',', ' ') }} {{ kube_node_addresses }}"
 
 # Reserve this space for kube resources
 # Whether to run kubelet and container-engine daemons in a dedicated cgroup. (Not required for resource reservations).
@@ -190,7 +192,7 @@ conntrack_modules:
 
 ## Enable distributed tracing for kubelet
 kubelet_tracing: false
-kubelet_tracing_endpoint: 0.0.0.0:4317
+kubelet_tracing_endpoint: "[::]:4317"
 kubelet_tracing_sampling_rate_per_million: 100
 
 # The maximum number of image pulls in parallel. Set it to a integer great than 1 to enable image pulling in parallel.

roles/kubernetes/node/tasks/main.yml

@@ -27,7 +27,7 @@
 - name: Install nginx-proxy
   import_tasks: loadbalancer/nginx-proxy.yml
   when:
-    - ('kube_control_plane' not in group_names) or (kube_apiserver_bind_address != '0.0.0.0')
+    - ('kube_control_plane' not in group_names) or (kube_apiserver_bind_address != '::')
     - loadbalancer_apiserver_localhost
     - loadbalancer_apiserver_type == 'nginx'
   tags:
@@ -36,7 +36,7 @@
 - name: Install haproxy
   import_tasks: loadbalancer/haproxy.yml
   when:
-    - ('kube_control_plane' not in group_names) or (kube_apiserver_bind_address != '0.0.0.0')
+    - ('kube_control_plane' not in group_names) or (kube_apiserver_bind_address != '::')
     - loadbalancer_apiserver_localhost
     - loadbalancer_apiserver_type == 'haproxy'
   tags:

roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2

@@ -29,10 +29,10 @@ containerLogMaxSize: {{ kubelet_logfiles_max_size }}
 containerRuntimeEndpoint : {{ cri_socket }}
 maxPods: {{ kubelet_max_pods }}
 podPidsLimit: {{ kubelet_pod_pids_limit }}
-address: {{ kubelet_bind_address }}
+address: "{{ kubelet_bind_address }}"
 readOnlyPort: {{ kube_read_only_port }}
 healthzPort: {{ kubelet_healthz_port }}
-healthzBindAddress: {{ kubelet_healthz_bind_address }}
+healthzBindAddress: "{{ kubelet_healthz_bind_address }}"
 kubeletCgroups: {{ kubelet_kubelet_cgroups }}
 clusterDomain: {{ dns_domain }}
 {% if kubelet_protect_kernel_defaults | bool %}
@@ -130,7 +130,7 @@ topologyManagerScope: {{ kubelet_topology_manager_scope }}
 {% endif %}
 {% if kubelet_tracing %}
 tracing:
-  endpoint: {{ kubelet_tracing_endpoint }}
+  endpoint: "{{ kubelet_tracing_endpoint }}"
   samplingRatePerMillion: {{ kubelet_tracing_sampling_rate_per_million }}
 {% endif %}
 maxParallelImagePulls: {{ kubelet_max_parallel_image_pulls }}

roles/kubernetes/node/templates/loadbalancer/haproxy.cfg.j2

@@ -22,7 +22,7 @@ defaults
 {% if loadbalancer_apiserver_healthcheck_port is defined -%}
 frontend healthz
   bind 0.0.0.0:{{ loadbalancer_apiserver_healthcheck_port }}
-  {% if enable_dual_stack_networks -%}
+  {% if ipv6_stack -%}
   bind :::{{ loadbalancer_apiserver_healthcheck_port }}
   {% endif -%}
   mode http
@@ -31,7 +31,7 @@ frontend healthz
 
 frontend kube_api_frontend
   bind 127.0.0.1:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }}
-  {% if enable_dual_stack_networks -%}
+  {% if ipv6_stack -%}
   bind [::1]:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};
   {% endif -%}
   mode tcp
@@ -45,5 +45,5 @@ backend kube_api_backend
   option httpchk GET /healthz
   http-check expect status 200
   {% for host in groups['kube_control_plane'] -%}
-  server {{ host }} {{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['fallback_ip'])) }}:{{ kube_apiserver_port }} check check-ssl verify none
+  server {{ host }} {{ hostvars[host]['main_access_ip'] | ansible.utils.ipwrap }}:{{ kube_apiserver_port }} check check-ssl verify none
   {% endfor -%}

roles/kubernetes/node/templates/loadbalancer/nginx.conf.j2

@@ -14,13 +14,13 @@ stream {
   upstream kube_apiserver {
     least_conn;
     {% for host in groups['kube_control_plane'] -%}
-    server {{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['fallback_ip'])) }}:{{ kube_apiserver_port }};
+    server {{ hostvars[host]['main_access_ip'] | ansible.utils.ipwrap }}:{{ kube_apiserver_port }};
     {% endfor -%}
   }
 
   server {
     listen        127.0.0.1:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};
-    {% if enable_dual_stack_networks -%}
+    {% if ipv6_stack -%}
     listen        [::1]:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};
     {% endif -%}
     proxy_pass    kube_apiserver;
@@ -44,7 +44,7 @@ http {
   {% if loadbalancer_apiserver_healthcheck_port is defined -%}
   server {
     listen {{ loadbalancer_apiserver_healthcheck_port }};
-    {% if enable_dual_stack_networks -%}
+    {% if ipv6_stack -%}
     listen [::]:{{ loadbalancer_apiserver_healthcheck_port }};
     {% endif -%}
     location /healthz {

roles/kubernetes/node/templates/node-kubeconfig.yaml.j2

@@ -5,7 +5,7 @@ clusters:
 - name: local
   cluster:
     certificate-authority: {{ kube_cert_dir }}/ca.pem
-    server: {{ kube_apiserver_endpoint }}
+    server: "{{ kube_apiserver_endpoint }}"
 users:
 - name: kubelet
   user:

roles/kubernetes/preinstall/tasks/0040-verify-settings.yml

@@ -1,7 +1,7 @@
 ---
 - name: Stop if any host not in '--limit' does not have a fact cache
   vars:
-    uncached_hosts: "{{ hostvars | dict2items | selectattr('value.ansible_default_ipv4', 'undefined') | map(attribute='key') }}"
+    uncached_hosts: "{{ hostvars | dict2items | selectattr('value.ansible_default_ipv6', 'value.ansible_default_ipv4', 'undefined') | map(attribute='key') }}"
     excluded_hosts: "{{ groups['k8s_cluster'] | difference(query('inventory_hostnames', ansible_limit)) }}"
   assert:
     that: uncached_hosts | intersect(excluded_hosts) == []
@@ -105,6 +105,7 @@
     - not ignore_assert_errors
     - ('k8s_cluster' in group_names)
     - kube_network_plugin not in ['calico', 'none']
+    - ipv4_stack | bool
 
 - name: Stop if ip var does not match local ips
   assert:
@@ -125,16 +126,16 @@
           {%- endif -%}
     state: present
   when:
-    - access_ip is defined
+    - main_access_ip is defined
     - not ignore_assert_errors
     - ping_access_ip
     - not is_fedora_coreos
     - not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
 
 - name: Stop if access_ip is not pingable
-  command: ping -c1 {{ access_ip }}
+  command: ping -c1 {{ main_access_ip }}
   when:
-    - access_ip is defined
+    - main_access_ip is defined
     - not ignore_assert_errors
     - ping_access_ip
   changed_when: false
@@ -179,12 +180,19 @@
     - cloud-provider
     - facts
 
+- name: Warn if `enable_dual_stack_networks` is set
+  debug:
+    msg: "WARNING! => `enable_dual_stack_networks` deprecation. Please switch to using ipv4_stack and ipv6_stack."
+  when:
+    - enable_dual_stack_networks is defined
+
 - name: "Check that kube_service_addresses is a network range"
   assert:
     that:
       - kube_service_addresses | ansible.utils.ipaddr('net')
     msg: "kube_service_addresses = '{{ kube_service_addresses }}' is not a valid network range"
   run_once: true
+  when: ipv4_stack | bool
 
 - name: "Check that kube_pods_subnet is a network range"
   assert:
@@ -192,6 +200,7 @@
       - kube_pods_subnet | ansible.utils.ipaddr('net')
     msg: "kube_pods_subnet = '{{ kube_pods_subnet }}' is not a valid network range"
   run_once: true
+  when: ipv4_stack | bool
 
 - name: "Check that kube_pods_subnet does not collide with kube_service_addresses"
   assert:
@@ -199,13 +208,50 @@
       - kube_pods_subnet | ansible.utils.ipaddr(kube_service_addresses) | string == 'None'
     msg: "kube_pods_subnet cannot be the same network segment as kube_service_addresses"
   run_once: true
+  when: ipv4_stack | bool
 
-- name: "Check that IP range is enough for the nodes"
+- name: "Check that ipv4 IP range is enough for the nodes"
   assert:
     that:
       - 2 ** (kube_network_node_prefix - kube_pods_subnet | ansible.utils.ipaddr('prefix')) >= groups['k8s_cluster'] | length
-    msg: "Not enough IPs are available for the desired node count."
-  when: kube_network_plugin != 'calico'
+    msg: "Not enough ipv4 IPs are available for the desired node count."
+  when:
+    - ipv4_stack | bool
+    - kube_network_plugin != 'calico'
+  run_once: true
+
+- name: "Check that kube_service_addresses_ipv6 is a network range"
+  assert:
+    that:
+      - kube_service_addresses_ipv6 | ansible.utils.ipaddr('net')
+    msg: "kube_service_addresses_ipv6 = '{{ kube_service_addresses_ipv6 }}' is not a valid network range"
+  run_once: true
+  when: ipv6_stack | bool
+
+- name: "Check that kube_pods_subnet_ipv6 is a network range"
+  assert:
+    that:
+      - kube_pods_subnet_ipv6 | ansible.utils.ipaddr('net')
+    msg: "kube_pods_subnet_ipv6 = '{{ kube_pods_subnet_ipv6 }}' is not a valid network range"
+  run_once: true
+  when: ipv6_stack | bool
+
+- name: "Check that kube_pods_subnet_ipv6 does not collide with kube_service_addresses_ipv6"
+  assert:
+    that:
+      - kube_pods_subnet_ipv6 | ansible.utils.ipaddr(kube_service_addresses_ipv6) | string == 'None'
+    msg: "kube_pods_subnet_ipv6 cannot be the same network segment as kube_service_addresses_ipv6"
+  run_once: true
+  when: ipv6_stack | bool
+
+- name: "Check that ipv6 IP range is enough for the nodes"
+  assert:
+    that:
+      - 2 ** (kube_network_node_prefix_ipv6 - kube_pods_subnet_ipv6 | ansible.utils.ipaddr('prefix')) >= groups['k8s_cluster'] | length
+    msg: "Not enough ipv6 IPs are available for the desired node count."
+  when:
+    - ipv6_stack | bool
+    - kube_network_plugin != 'calico'
   run_once: true
 
 - name: Stop if unsupported options selected

roles/kubernetes/preinstall/tasks/0080-system-configurations.yml

@@ -76,6 +76,7 @@
     value: "1"
     state: present
     reload: true
+  when: ipv4_stack | bool
 
 - name: Enable ipv6 forwarding
   ansible.posix.sysctl:
@@ -84,7 +85,7 @@
     value: "1"
     state: present
     reload: true
-  when: enable_dual_stack_networks | bool
+  when: ipv6_stack | bool
 
 - name: Check if we need to set fs.may_detach_mounts
   stat:

roles/kubernetes/preinstall/tasks/0090-etchosts.yml

@@ -2,11 +2,10 @@
 - name: Hosts | create hosts list from inventory
   set_fact:
     etc_hosts_inventory_block: |-
-      {% for item in (groups['k8s_cluster'] + groups['etcd'] | default([]) + groups['calico_rr'] | default([])) | unique -%}
-      {% if 'access_ip' in hostvars[item] or 'ip' in hostvars[item] or 'ansible_default_ipv4' in hostvars[item] -%}
-      {{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item]['ansible_default_ipv4']['address'])) }}
-      {%- if ('ansible_hostname' in hostvars[item] and item != hostvars[item]['ansible_hostname']) %} {{ hostvars[item]['ansible_hostname'] }}.{{ dns_domain }} {{ hostvars[item]['ansible_hostname'] }} {% else %} {{ item }}.{{ dns_domain }} {{ item }} {% endif %}
-
+      {% for item in (groups['k8s_cluster'] + groups['etcd'] | default([]) + groups['calico_rr'] | default([])) | unique %}
+      {{ hostvars[item]['main_access_ip'] }} {{ hostvars[item]['ansible_hostname'] | default(item) }}.{{ dns_domain }} {{ hostvars[item]['ansible_hostname'] | default(item) }}
+      {% if ipv4_stack and ipv6_stack %}
+      {{ hostvars[item]['access_ip6'] | default(hostvars[item]['ip6'] | default(hostvars[item]['ansible_default_ipv6']['address'])) }} {{ hostvars[item]['ansible_hostname'] | default(item) }}.{{ dns_domain }} {{ hostvars[item]['ansible_hostname'] | default(item) }}
       {% endif %}
       {% endfor %}
   delegate_to: localhost