Overhaul Cilium manifests to match the newer versions (#8717)

* [cilium] Separate templates for cilium, cilium-operator, and hubble installations

Signed-off-by: necatican <necaticanyildirim@gmail.com>

* [cilium] Update cilium-operator templates

Signed-off-by: necatican <necaticanyildirim@gmail.com>

* [cilium] Allow using custom args and mounting extra volumes for the Cilium Operator

Signed-off-by: necatican <necaticanyildirim@gmail.com>

* [cilium] Update the cilium configmap to filter out the deprecated variables, and add the new variables

Signed-off-by: necatican <necaticanyildirim@gmail.com>

* [cilium] Add an option to use Wireguard encryption on Cilium 1.10 and up

Signed-off-by: necatican <necaticanyildirim@gmail.com>

* [cilium] Update cilium-agent templates

Signed-off-by: necatican <necaticanyildirim@gmail.com>

* [cilium] Bump Cilium version to 1.11.3

Signed-off-by: necatican <necaticanyildirim@gmail.com>

roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2

@@ -0,0 +1,124 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cilium-operator
+rules:
+- apiGroups:
+  - ""
+  resources:
+  # to automatically delete [core|kube]dns pods so that are starting to being
+  # managed by Cilium
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - delete
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - list
+  - watch
+{% if cilium_version | regex_replace('v') is version('1.10', '>=') %}
+- apiGroups:
+  - ""
+  resources:
+  # to perform LB IP allocation for BGP
+  - services/status
+  verbs:
+  - update
+{% endif %}
+- apiGroups:
+  - ""
+  resources:
+{% if cilium_version | regex_replace('v') is version('1.8', '<') %}
+  # to automatically read from k8s and import the node's pod CIDR to cilium's
+  # etcd so all nodes know how to reach another pod running in in a different
+  # node.
+  - nodes
+{% endif %}
+  # to perform the translation of a CNP that contains `ToGroup` to its endpoints
+  - services
+  - endpoints
+  # to check apiserver connectivity
+  - namespaces
+{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
+  - componentstatuses
+{% endif %}
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumnetworkpolicies
+  - ciliumnetworkpolicies/status
+  - ciliumclusterwidenetworkpolicies
+  - ciliumclusterwidenetworkpolicies/status
+  - ciliumendpoints
+  - ciliumendpoints/status
+{% if cilium_version | regex_replace('v') is version('1.6', '>=') %}
+  - ciliumnodes
+  - ciliumnodes/status
+  - ciliumidentities
+  - ciliumidentities/status
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.9', '>=') %}
+  - ciliumnetworkpolicies/finalizers
+  - ciliumclusterwidenetworkpolicies/finalizers
+  - ciliumendpoints/finalizers
+  - ciliumnodes/finalizers
+  - ciliumidentities/finalizers
+  - ciliumlocalredirectpolicies
+  - ciliumlocalredirectpolicies/status
+  - ciliumlocalredirectpolicies/finalizers
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
+  - ciliumendpointslices
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
+  - ciliumbgploadbalancerippools
+  - ciliumbgppeeringpolicies
+  - ciliumenvoyconfigs
+{% endif %}
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - delete
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - create
+  - get
+  - list
+  - update
+  - watch
+{% if cilium_version | regex_replace('v') is version('1.8', '>=') %}
+  # For cilium-operator running in HA mode.
+  #
+  # Cilium operator running in HA mode requires the use of ResourceLock for Leader Election
+  # between mulitple running instances.
+  # The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less
+  # common and fewer objects in the cluster watch "all Leases".
+  # The support for leases was introduced in coordination.k8s.io/v1 during Kubernetes 1.14 release.
+  # In Cilium we currently don't support HA mode for K8s version < 1.14. This condition make sure
+  # that we only authorize access to leases resources in supported K8s versions.
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - update
+{% endif %}

roles/network_plugin/cilium/templates/cilium-operator/crb.yml.j2

@@ -0,0 +1,13 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cilium-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cilium-operator
+subjects:
+- kind: ServiceAccount
+  name: cilium-operator
+  namespace: kube-system

roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2

@@ -0,0 +1,191 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    io.cilium/app: operator
+    name: cilium-operator
+  name: cilium-operator
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      io.cilium/app: operator
+      name: cilium-operator
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+{% if cilium_enable_prometheus %}
+      annotations:
+        prometheus.io/port: "6942"
+        prometheus.io/scrape: "true"
+{% endif %}
+      labels:
+        io.cilium/app: operator
+        name: cilium-operator
+    spec:
+      # In HA mode, cilium-operator pods must not be scheduled on the same
+      # node as they will clash with each other.
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: io.cilium/app
+                operator: In
+                values:
+                - operator
+            topologyKey: "kubernetes.io/hostname"
+      containers:
+        - args:
+            - --debug=$(CILIUM_DEBUG)
+            - --config-dir=/tmp/cilium/config-map
+{% if cilium_operator_custom_args is string %}
+            - {{ cilium_operator_custom_args }}
+{% else %}
+{% for flag in cilium_operator_custom_args %}
+            - {{ flag }}
+{% endfor %}
+{% endif %}
+          command:
+            - cilium-operator
+          env:
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: K8S_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+            - name: CILIUM_K8S_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: CILIUM_DEBUG
+              valueFrom:
+                configMapKeyRef:
+                  key: debug
+                  name: cilium-config
+                  optional: true
+# We are already mounting the whole ConfigMap as a directory.
+# https://github.com/cilium/cilium/pull/10347
+{% if cilium_version | regex_replace('v') is version('1.8', '<') %}
+            - name: CILIUM_CLUSTER_NAME
+              valueFrom:
+                configMapKeyRef:
+                  key: cluster-name
+                  name: cilium-config
+                  optional: true
+            - name: CILIUM_CLUSTER_ID
+              valueFrom:
+                configMapKeyRef:
+                  key: cluster-id
+                  name: cilium-config
+                  optional: true
+            - name: CILIUM_DISABLE_ENDPOINT_CRD
+              valueFrom:
+                configMapKeyRef:
+                  key: disable-endpoint-crd
+                  name: cilium-config
+                  optional: true
+{% endif %}
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  key: AWS_ACCESS_KEY_ID
+                  name: cilium-aws
+                  optional: true
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  key: AWS_SECRET_ACCESS_KEY
+                  name: cilium-aws
+                  optional: true
+            - name: AWS_DEFAULT_REGION
+              valueFrom:
+                secretKeyRef:
+                  key: AWS_DEFAULT_REGION
+                  name: cilium-aws
+                  optional: true
+{% if cilium_kube_proxy_replacement == 'strict' %}
+            - name: KUBERNETES_SERVICE_HOST
+              value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}"
+            - name: KUBERNETES_SERVICE_PORT
+              value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
+{% endif %}
+          image: "{{ cilium_operator_image_repo }}:{{ cilium_operator_image_tag }}"
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          name: cilium-operator
+{% if cilium_enable_prometheus %}
+          ports:
+            - containerPort: 6942
+              hostPort: 6942
+              name: prometheus
+              protocol: TCP
+{% endif %}
+          livenessProbe:
+            httpGet:
+{% if cilium_enable_ipv4 %}
+              host: 127.0.0.1
+{% else %}
+              host: '::1'
+{% endif %}
+              path: /healthz
+              port: 9234
+              scheme: HTTP
+            initialDelaySeconds: 60
+            periodSeconds: 10
+            timeoutSeconds: 3
+          volumeMounts:
+{% if cilium_identity_allocation_mode == "kvstore" %}
+            - mountPath: /var/lib/etcd-config
+              name: etcd-config-path
+              readOnly: true
+            - mountPath: "{{cilium_cert_dir}}"
+              name: etcd-secrets
+              readOnly: true
+{% endif %}
+            - mountPath: /tmp/cilium/config-map
+              name: cilium-config-path
+              readOnly: true
+{% for volume_mount in cilium_operator_extra_volume_mounts %}
+            - {{ volume_mount | to_nice_yaml(indent=2) | indent(14) }}
+{% endfor %}
+      dnsPolicy: ClusterFirst
+      priorityClassName: system-node-critical
+      restartPolicy: Always
+      serviceAccount: cilium-operator
+      serviceAccountName: cilium-operator
+      hostNetwork: true
+      tolerations:
+        - operator: Exists
+      volumes:
+{% if cilium_identity_allocation_mode == "kvstore" %}
+        # To read the etcd config stored in config maps
+        - configMap:
+            defaultMode: 420
+            items:
+              - key: etcd-config
+                path: etcd.config
+            name: cilium-config
+          name: etcd-config-path
+          # To read the k8s etcd secrets in case the user might want to use TLS
+        - name: etcd-secrets
+          hostPath:
+            path: "{{cilium_cert_dir}}"
+{% endif %}
+        - configMap:
+            name: cilium-config
+          name: cilium-config-path
+{% for volume in cilium_operator_extra_volumes %}
+        - {{ volume | to_nice_yaml(indent=2) | indent(10) }}
+{% endfor %}

roles/network_plugin/cilium/templates/cilium-operator/sa.yml.j2

@@ -0,0 +1,6 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cilium-operator
+  namespace: kube-system