Calico enable support for eBPF (#7618)

* Calico: align manifests with upstream

* allow enabling typha prometheus metrics

* Calico: enable eBPF support

* manage the kubernetes-services-endpoint configmap

* Calico: document the use of eBPF dataplane

* Calico: improve checks before deployment

* enforce disabling kube-proxy when using eBPF dataplane
* ensure calico_version is supported

roles/network_plugin/calico/defaults/main.yml

@@ -32,6 +32,12 @@ calico_advertise_service_external_ips: []
 # Adveritse Service LoadBalancer IPs
 calico_advertise_service_loadbalancer_ips: []
 
+# Calico eBPF support
+calico_bpf_enabled: false
+calico_bpf_log_level: ""
+# Valid option for service mode: Tunnel (default), DSR=Direct Server Return
+calico_bpf_service_mode: Tunnel
+
 # Limits for apps
 calico_node_memory_limit: 500M
 calico_node_cpu_limit: 300m
@@ -91,6 +97,8 @@ kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem
 
 # Use typha (only with kdd)
 typha_enabled: false
+typha_prometheusmetricsenabled: false
+typha_prometheusmetricsport: 9093
 
 # Scaling typha: 1 replica per 100 nodes is adequate
 # Number of typha replicas
@@ -106,3 +114,7 @@ calico_feature_control: {}
 
 # Calico default BGP port
 calico_bgp_listen_port: 179
+
+# Calico FelixConfiguration options
+calico_felix_reporting_interval: 0s
+calico_felix_log_severity_screen: Info

roles/network_plugin/calico/tasks/install.yml

@@ -131,6 +131,27 @@
     - inventory_hostname in groups['kube_control_plane']
     - calico_datastore == "kdd"
 
+- name: Calico | Configure calico FelixConfiguration
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}"
+  vars:
+    stdin: >
+      { "kind": "FelixConfiguration",
+        "apiVersion": "projectcalico.org/v3",
+        "metadata": {
+          "name": "default",
+        },
+        "spec": {
+          "ipipEnabled": {{ calico_ipip_mode != 'Never' | bool }},
+          "reportingInterval": "{{ calico_felix_reporting_interval }}",
+          "bpfLogLevel": "{{ calico_bpf_log_level }}",
+          "bpfEnabled": {{ calico_bpf_enabled | bool }},
+          "bpfExternalServiceMode": "{{ calico_bpf_service_mode }}",
+          "logSeverityScreen": "{{ calico_felix_log_severity_screen }}" }}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
 - name: Calico | Configure calico network pool
   command:
     cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
@@ -302,6 +323,7 @@
     - {name: calico, file: calico-node-sa.yml, type: sa}
     - {name: calico, file: calico-cr.yml, type: clusterrole}
     - {name: calico, file: calico-crb.yml, type: clusterrolebinding}
+    - {name: kubernetes-services-endpoint, file: kubernetes-services-endpoint.yml, type: cm }
   register: calico_node_manifests
   when:
     - inventory_hostname in groups['kube_control_plane']

roles/network_plugin/calico/templates/calico-cr.yml.j2

@@ -28,6 +28,7 @@ rules:
     resources:
       - nodes/status
     verbs:
+      # Needed for clearing NodeNetworkUnavailable flag.
       - patch
 {% if calico_datastore == "etcd" %}
   - apiGroups:

roles/network_plugin/calico/templates/calico-node.yml.j2

@@ -44,6 +44,11 @@ spec:
         - name: upgrade-ipam
           image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
           command: ["/opt/cni/bin/calico-ipam", "-upgrade"]
+          envFrom:
+            - configMapRef:
+                # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
+                name: kubernetes-services-endpoint
+                optional: true
           env:
             - name: KUBERNETES_NODE_NAME
               valueFrom:
@@ -94,12 +99,26 @@ spec:
               name: cni-bin-dir
           securityContext:
             privileged: true
+        # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes
+        # to communicate with Felix over the Policy Sync API.
+        - name: flexvol-driver
+          image: {{ calico_flexvol_image_repo }}:{{ calico_flexvol_image_tag }}
+          volumeMounts:
+            - name: flexvol-driver-host
+              mountPath: /host/driver
+          securityContext:
+            privileged: true
       containers:
         # Runs calico/node container on each Kubernetes node.  This
         # container programs network policy and routes on each
         # host.
         - name: calico-node
           image: {{ calico_node_image_repo }}:{{ calico_node_image_tag }}
+          envFrom:
+            - configMapRef:
+                # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
+                name: kubernetes-services-endpoint
+                optional: true
           env:
             # The location of the Calico etcd cluster.
 {% if calico_datastore == "etcd" %}
@@ -231,8 +250,6 @@ spec:
 {% if calico_ip_auto_method is defined %}
             - name: IP_AUTODETECTION_METHOD
               value: "{{ calico_ip_auto_method }}"
-            - name: IP
-              value: "autodetect"
 {% else %}
             - name: NODEIP
               valueFrom:
@@ -240,9 +257,9 @@ spec:
                   fieldPath: status.hostIP
             - name: IP_AUTODETECTION_METHOD
               value: "can-reach=$(NODEIP)"
+{% endif %}
             - name: IP
               value: "autodetect"
-{% endif %}
 {% if enable_dual_stack_networks %}
             - name: IP6
               value: autodetect
@@ -286,10 +303,10 @@ spec:
 {% if calico_network_backend|default("bird") == "bird" %}
               - -bird-live
 {% endif %}
-            initialDelaySeconds: 5
+            periodSeconds: 10
+            initialDelaySeconds: 10
             failureThreshold: 6
           readinessProbe:
-            failureThreshold: 6
             exec:
               command:
               - /bin/calico-node
@@ -297,18 +314,22 @@ spec:
               - -bird-ready
 {% endif %}
               - -felix-ready
+            periodSeconds: 10
+            failureThreshold: 6
           volumeMounts:
             - mountPath: /lib/modules
               name: lib-modules
               readOnly: true
             - mountPath: /var/run/calico
               name: var-run-calico
+              readOnly: false
             - mountPath: /var/lib/calico
               name: var-lib-calico
               readOnly: false
 {% if calico_datastore == "etcd" %}
             - mountPath: /calico-secrets
               name: etcd-certs
+              readOnly: true
 {% endif %}
             - name: xtables-lock
               mountPath: /run/xtables.lock
@@ -324,7 +345,20 @@ spec:
               mountPath: /etc/typha-ca/ca.crt
               readOnly: true
 {% endif %}
-
+            - name: policysync
+              mountPath: /var/run/nodeagent
+{% if calico_bpf_enabled %}
+            # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the
+            # parent directory.
+            - name: sysfs
+              mountPath: /sys/fs/
+              # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host.
+              # If the host is known to mount that filesystem already then Bidirectional can be omitted.
+              mountPropagation: Bidirectional
+{% endif %}
+            - name: cni-log-dir
+              mountPath: /var/log/calico/cni
+              readOnly: true
       volumes:
         # Used by calico/node.
         - name: lib-modules
@@ -375,6 +409,26 @@ spec:
           hostPath:
             path: "/etc/kubernetes/ssl/"
 {% endif %}
+{% if calico_bpf_enabled %}
+        - name: sysfs
+          hostPath:
+            path: /sys/fs/
+            type: DirectoryOrCreate
+{% endif %}
+        # Used to access CNI logs.
+        - name: cni-log-dir
+          hostPath:
+            path: /var/log/calico/cni
+        # Used to create per-pod Unix Domain Sockets
+        - name: policysync
+          hostPath:
+            type: DirectoryOrCreate
+            path: /var/run/nodeagent
+        # Used to install Flex Volume Driver
+        - name: flexvol-driver-host
+          hostPath:
+            type: DirectoryOrCreate
+            path: "{{ kubelet_flexvolumes_plugins_dir | default('/usr/libexec/kubernetes/kubelet-plugins/volume/exec') }}/nodeagent~uds"
   updateStrategy:
     rollingUpdate:
       maxUnavailable: {{ serial | default('20%') }}

roles/network_plugin/calico/templates/calico-typha.yml.j2

@@ -46,6 +46,10 @@ spec:
         k8s-app: calico-typha
       annotations:
         cluster-autoscaler.kubernetes.io/safe-to-evict: 'true'
+{% if typha_prometheusmetricsenabled %}
+        prometheus.io/scrape: 'true'
+        prometheus.io/port: "{{ typha_prometheusmetricsport }}"
+{% endif %}
     spec:
       nodeSelector:
         kubernetes.io/os: linux
@@ -61,6 +65,9 @@ spec:
       # as a host-networked pod.
       serviceAccountName: calico-node
       priorityClassName: system-cluster-critical
+      # fsGroup allows using projected serviceaccount tokens as described here kubernetes/kubernetes#82573
+      securityContext:
+        fsGroup: 65534
       containers:
       - image: {{ calico_typha_image_repo }}:{{ calico_typha_image_tag }}
         name: calico-typha
@@ -68,6 +75,11 @@ spec:
         - containerPort: 5473
           name: calico-typha
           protocol: TCP
+        envFrom:
+        - configMapRef:
+            # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
+            name: kubernetes-services-endpoint
+            optional: true
         env:
           # Enable "info" logging by default.  Can be set to "debug" to increase verbosity.
           - name: TYPHA_LOGSEVERITYSCREEN
@@ -105,13 +117,14 @@ spec:
             name: cacert
             readOnly: true
 {% endif %}
-          # Uncomment these lines to enable prometheus metrics.  Since Typha is host-networked,
+{% if typha_prometheusmetricsenabled %}
+          # Since Typha is host-networked,
           # this opens a port on the host, which may need to be secured.
-          #- name: TYPHA_PROMETHEUSMETRICSENABLED
-          #  value: "true"
-          #- name: TYPHA_PROMETHEUSMETRICSPORT
-          #  value: "9093"
-
+          - name: TYPHA_PROMETHEUSMETRICSENABLED
+            value: "true"
+          - name: TYPHA_PROMETHEUSMETRICSPORT
+            value: "{{ typha_prometheusmetricsport }}"
+{% endif %}
           # Needed for version >=3.7 when the 'host-local' ipam is used
           # Should never happen given templates/cni-calico.conflist.j2
           # Configure route aggregation based on pod CIDR.

roles/network_plugin/calico/templates/cni-calico.conflist.j2

@@ -64,6 +64,12 @@
       "capabilities": {
         "portMappings": true
       }
+    },
+    {
+      "type":"bandwidth",
+      "capabilities": {
+        "bandwidth": true
+      }
     }
   ]
 }

roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2

@@ -0,0 +1,19 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: kube-system
+  name: kubernetes-services-endpoint
+data:
+{% if calico_bpf_enabled %}
+{% if loadbalancer_apiserver is defined %}
+  KUBERNETES_SERVICE_HOST: "{{ apiserver_loadbalancer_domain_name }}"
+  KUBERNETES_SERVICE_PORT: "{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}"
+{%- elif use_localhost_as_kubeapi_loadbalancer|default(False)|bool %}
+  KUBERNETES_SERVICE_HOST: "127.0.0.1"
+  KUBERNETES_SERVICE_PORT: "{{ kube_apiserver_port }}"
+{%- else %}
+  KUBERNETES_SERVICE_HOST: "{{ first_kube_master }}"
+  KUBERNETES_SERVICE_PORT: "{{ kube_apiserver_port }}"
+{%- endif %}
+{% endif %}