Changes to support Dual Stack networking

roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2

@@ -88,8 +88,14 @@ dns:
   imageTag: {{ coredns_image_tag }}
 networking:
   dnsDomain: {{ dns_domain }}
-  serviceSubnet: {{ kube_service_addresses }}
-  podSubnet: {{ kube_pods_subnet }}
+  serviceSubnet: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks }}"
+  podSubnet: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks }}"
+{% if kube_feature_gates %}
+featureGates:
+{%   for kube_feature_gate in kube_feature_gates %}
+  {{ kube_feature_gate.split("=")[0] }}: {{ kube_feature_gate.split("=")[1] }}
+{%   endfor %}
+{% endif %}
 kubernetesVersion: {{ kube_version }}
 {% if kubeadm_config_api_fqdn is defined %}
 controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
@@ -127,6 +133,7 @@ apiServer:
     etcd-servers-overrides: "/events#{{ etcd_events_access_addresses_semicolon }}"
 {% endif %}
     service-node-port-range: {{ kube_apiserver_node_port_range }}
+    service-cluster-ip-range: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks }}"
     kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}"
     profiling: "{{ kube_profiling }}"
     request-timeout: "{{ kube_apiserver_request_timeout }}"
@@ -262,7 +269,14 @@ controllerManager:
   extraArgs:
     node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
     node-monitor-period: {{ kube_controller_node_monitor_period }}
+    cluster-cidr: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks }}"
+    service-cluster-ip-range: "{{ kube_service_addresses }}{{ ',' + kube_service_addresses_ipv6 if enable_dual_stack_networks }}"
+{% if enable_dual_stack_networks %}
+    node-cidr-mask-size-ipv4: "{{ kube_network_node_prefix }}"
+    node-cidr-mask-size-ipv6: "{{ kube_network_node_prefix_ipv6 }}"
+{% else %}
     node-cidr-mask-size: "{{ kube_network_node_prefix }}"
+{% endif %}
     profiling: "{{ kube_profiling }}"
     terminated-pod-gc-threshold: "{{ kube_controller_terminated_pod_gc_threshold }}"
     bind-address: {{ kube_controller_manager_bind_address }}
@@ -349,7 +363,7 @@ clientConnection:
   contentType: {{ kube_proxy_client_content_type }}
   kubeconfig: {{ kube_proxy_client_kubeconfig }}
   qps: {{ kube_proxy_client_qps }}
-clusterCIDR: {{ kube_pods_subnet }}
+clusterCIDR: "{{ kube_pods_subnet }}{{ ',' + kube_pods_subnet_ipv6 if enable_dual_stack_networks }}"
 configSyncPeriod: {{ kube_proxy_config_sync_period }}
 conntrack:
   maxPerCore: {{ kube_proxy_conntrack_max_per_core }}
@@ -357,6 +371,12 @@ conntrack:
   tcpCloseWaitTimeout: {{ kube_proxy_conntrack_tcp_close_wait_timeout }}
   tcpEstablishedTimeout: {{ kube_proxy_conntrack_tcp_established_timeout }}
 enableProfiling: {{ kube_proxy_enable_profiling }}
+{% if kube_feature_gates %}
+featureGates:
+{%   for kube_feature_gate in kube_feature_gates %}
+  {{ kube_feature_gate.split("=")[0] }}: {{ kube_feature_gate.split("=")[1] }}
+{%   endfor %}
+{% endif %}
 healthzBindAddress: {{ kube_proxy_healthz_bind_address }}
 hostnameOverride: {{ kube_override_hostname }}
 iptables:
@@ -404,3 +424,9 @@ clusterDNS:
 {% for dns_address in kubelet_cluster_dns %}
 - {{ dns_address }}
 {% endfor %}
+{% if kube_feature_gates %}
+featureGates:
+{%   for kube_feature_gate in kube_feature_gates %}
+  {{ kube_feature_gate.split("=")[0] }}: {{ kube_feature_gate.split("=")[1] }}
+{%   endfor %}
+{% endif %}

roles/kubernetes/preinstall/tasks/0080-system-configurations.yml

@@ -62,6 +62,15 @@
     state: present
     reload: yes
 
+- name: Enable ipv6 forwarding
+  sysctl:
+    sysctl_file: "{{ sysctl_file_path }}"
+    name: net.ipv6.conf.all.forwarding
+    value: 1
+    state: present
+    reload: yes
+  when: enable_dual_stack_networks | bool
+
 - name: Ensure kube-bench parameters are set
   sysctl:
     sysctl_file: /etc/sysctl.d/bridge-nf-call.conf

roles/kubespray-defaults/defaults/main.yaml

@@ -181,6 +181,25 @@ kube_pods_subnet: 10.233.64.0/18
 #  - kubelet_max_pods: 110
 kube_network_node_prefix: 24
 
+# Configure Dual Stack networking (i.e. both IPv4 and IPv6)
+enable_dual_stack_networks: false
+
+# Kubernetes internal network for IPv6 services, unused block of space.
+# This is only used if enable_dual_stack_networks is set to true
+# This provides 4096 IPv6 IPs
+kube_service_addresses_ipv6: fd85:ee78:d8a6:8607::1000/116
+
+# Internal network. When used, it will assign IPv6 addresses from this range to individual pods.
+# This network must not already be in your network infrastructure!
+# This is only used if enable_dual_stack_networks is set to true.
+# This provides room for 256 nodes with 254 pods per node.
+kube_pods_subnet_ipv6: fd85:ee78:d8a6:8607::1:0000/112
+
+# IPv6 subnet size allocated to each for pods.
+# This is only used if enable_dual_stack_networks is set to true
+# This provides room for 254 pods per node.
+kube_network_node_prefix_ipv6: 120
+
 # The virtual cluster IP, real host IPs and ports the API Server will be
 # listening on.
 # NOTE: loadbalancer_apiserver_localhost somewhat alters the final API enpdoint

roles/network_plugin/calico/defaults/main.yml

@@ -12,6 +12,10 @@ ipip_mode: "{{ 'Always' if ipip else 'Never' }}"  # change to "CrossSubnet" if y
 calico_ipip_mode: "{{ ipip_mode }}"
 calico_vxlan_mode: 'Never'
 
+calico_ipip_mode_ipv6: Never
+calico_vxlan_mode_ipv6: Never
+calico_pool_blocksize_ipv6: 116
+
 calico_cert_dir: /etc/calico/certs
 
 # Global as_num (/calico/bgp/v1/global/as_num)

roles/network_plugin/calico/tasks/install.yml

@@ -108,6 +108,31 @@
     - 'calico_conf.stdout == "0"'
     - calico_pool_cidr is defined
 
+- name: Calico | Check if calico IPv6 network pool has already been configured
+  # noqa 306 - grep will exit 1 if no match found
+  shell: >
+    {{ bin_dir }}/calicoctl.sh get ippool | grep -w "{{ calico_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }}" | wc -l
+  args:
+    executable: /bin/bash
+  register: calico_conf_ipv6
+  retries: 4
+  until: calico_conf_ipv6.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  changed_when: false
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - enable_dual_stack_networks
+
+- name: Calico | Ensure that calico_pool_cidr_ipv6 is within kube_pods_subnet_ipv6 when defined
+  assert:
+    that: "[calico_pool_cidr_ipv6] | ipaddr(kube_pods_subnet_ipv6) | length == 1"
+    msg: "{{ calico_pool_cidr_ipv6 }} is not within or equal to {{ kube_pods_subnet_ipv6 }}"
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - calico_conf_ipv6.stdout is defined and calico_conf_ipv6.stdout == "0"
+    - calico_pool_cidr_ipv6 is defined
+    - enable_dual_stack_networks
+
 - name: Calico | Create calico manifests for kdd
   template:
     src: "{{ item.file }}.j2"
@@ -156,6 +181,27 @@
     - inventory_hostname == groups['kube-master'][0]
     - 'calico_conf.stdout == "0"'
 
+- name: Calico | Configure calico ipv6 network pool (version >= v3.3.0)
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    stdin: >
+      { "kind": "IPPool",
+        "apiVersion": "projectcalico.org/v3",
+        "metadata": {
+          "name": "{{ calico_pool_name }}-ipv6",
+        },
+        "spec": {
+          "blockSize": {{ calico_pool_blocksize_ipv6 | default(kube_network_node_prefix_ipv6) }},
+          "cidr": "{{ calico_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }}",
+          "ipipMode": "{{ calico_ipip_mode_ipv6 }}",
+          "vxlanMode": "{{ calico_vxlan_mode_ipv6 }}",
+          "natOutgoing": {{ nat_outgoing_ipv6|default(false) and not peer_with_router_ipv6|default(false) }} }}
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - calico_conf_ipv6.stdout is defined and calico_conf_ipv6.stdout == "0"
+    - calico_version is version("v3.3.0", ">=")
+    - enable_dual_stack_networks | bool
+
 - name: Populate Service External IPs
   set_fact:
     _service_external_ips: "{{ _service_external_ips|default([]) + [ {'cidr': item} ] }}"

roles/network_plugin/calico/templates/calico-node.yml.j2

@@ -200,9 +200,8 @@ spec:
 {% endif %}
             - name: CALICO_IPV4POOL_IPIP
               value: "{{ calico_ipv4pool_ipip }}"
-            # Disable IPv6 on Kubernetes.
             - name: FELIX_IPV6SUPPORT
-              value: "false"
+              value: "{{ enable_dual_stack_networks | default(false) }}"
             # Set Felix logging to "info"
             - name: FELIX_LOGSEVERITYSCREEN
               value: "{{ calico_loglevel }}"
@@ -239,6 +238,10 @@ spec:
             - name: IP
               value: "autodetect"
 {% endif %}
+{% if enable_dual_stack_networks %}
+            - name: IP6
+              value: autodetect
+{% endif %}
 {% if calico_use_default_route_src_ipaddr|default(false) %}
             - name: FELIX_DEVICEROUTESOURCEADDRESS
               valueFrom:

roles/network_plugin/calico/templates/cni-calico.conflist.j2

@@ -30,6 +30,10 @@
 {% else %}
       "ipam": {
         "type": "calico-ipam",
+{%   if enable_dual_stack_networks %}
+        "assign_ipv6": "true",
+        "ipv6_pools": ["{{ calico_pool_cidr_ipv6 | default(kube_pods_subnet_ipv6) }}"],
+{%   endif %}
         "assign_ipv4": "true",
         "ipv4_pools": ["{{ calico_pool_cidr | default(kube_pods_subnet) }}"]
       },