Cilium 1.12 Upgrade (#9225)

* Drop support for Cilium < 1.10 Signed-off-by: necatican <necaticanyildirim@gmail.com> * Synchronize Cilium templates for 1.11.7 Signed-off-by: necatican <contact@necatican.com> * Set Cilium v1.12.1 as the default version Signed-off-by: necatican <contact@necatican.com> Signed-off-by: necatican <necaticanyildirim@gmail.com> Signed-off-by: necatican <contact@necatican.com>
2026-04-11 21:19:28 -02:30 · 2022-09-19 12:14:31 +03:00
parent 680293e79c
commit 7da3dbcb39
12 changed files with 291 additions and 374 deletions
--- a/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2
@@ -39,7 +39,14 @@ rules:
  - get
  - list
  - watch
-{% if cilium_version | regex_replace('v') is version('1.10', '>=') %}
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
  - ""
  resources:
@@ -47,22 +54,14 @@ rules:
  - services/status
  verbs:
  - update
-{% endif %}
 - apiGroups:
  - ""
  resources:
-  # to automatically read from k8s and import the node's pod CIDR to cilium's
-  # etcd so all nodes know how to reach another pod running in in a different
-  # node.
-  - nodes
  # to perform the translation of a CNP that contains `ToGroup` to its endpoints
  - services
  - endpoints
  # to check apiserver connectivity
  - namespaces
-{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
-  - componentstatuses
-{% endif %}
  verbs:
  - get
  - list
@@ -72,26 +71,22 @@ rules:
  resources:
  - ciliumnetworkpolicies
  - ciliumnetworkpolicies/status
+  - ciliumnetworkpolicies/finalizers
  - ciliumclusterwidenetworkpolicies
  - ciliumclusterwidenetworkpolicies/status
+  - ciliumclusterwidenetworkpolicies/finalizers
  - ciliumendpoints
  - ciliumendpoints/status
-{% if cilium_version | regex_replace('v') is version('1.6', '>=') %}
+  - ciliumendpoints/finalizers
  - ciliumnodes
  - ciliumnodes/status
+  - ciliumnodes/finalizers
  - ciliumidentities
  - ciliumidentities/status
-{% endif %}
-{% if cilium_version | regex_replace('v') is version('1.9', '>=') %}
-  - ciliumnetworkpolicies/finalizers
-  - ciliumclusterwidenetworkpolicies/finalizers
-  - ciliumendpoints/finalizers
-  - ciliumnodes/finalizers
  - ciliumidentities/finalizers
  - ciliumlocalredirectpolicies
  - ciliumlocalredirectpolicies/status
  - ciliumlocalredirectpolicies/finalizers
-{% endif %}
 {% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
  - ciliumendpointslices
 {% endif %}
@@ -101,12 +96,7 @@ rules:
  - ciliumenvoyconfigs
 {% endif %}
  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - update
-  - delete
+  - '*'
 - apiGroups:
  - apiextensions.k8s.io
  resources:
@@ -117,16 +107,12 @@ rules:
  - list
  - update
  - watch
-{% if cilium_version | regex_replace('v') is version('1.8', '>=') %}
-  # For cilium-operator running in HA mode.
-  #
-  # Cilium operator running in HA mode requires the use of ResourceLock for Leader Election
-  # between mulitple running instances.
-  # The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less
-  # common and fewer objects in the cluster watch "all Leases".
-  # The support for leases was introduced in coordination.k8s.io/v1 during Kubernetes 1.14 release.
-  # In Cilium we currently don't support HA mode for K8s version < 1.14. This condition make sure
-  # that we only authorize access to leases resources in supported K8s versions.
+# For cilium-operator running in HA mode.
+#
+# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election
+# between multiple running instances.
+# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less
+# common and fewer objects in the cluster watch "all Leases".
 - apiGroups:
  - coordination.k8s.io
  resources:
@@ -135,4 +121,26 @@ rules:
  - create
  - get
  - update
+{% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - update
+  resourceNames:
+  - ciliumbgploadbalancerippools.cilium.io
+  - ciliumbgppeeringpolicies.cilium.io
+  - ciliumclusterwideenvoyconfigs.cilium.io
+  - ciliumclusterwidenetworkpolicies.cilium.io
+  - ciliumegressgatewaypolicies.cilium.io
+  - ciliumegressnatpolicies.cilium.io
+  - ciliumendpoints.cilium.io
+  - ciliumendpointslices.cilium.io
+  - ciliumenvoyconfigs.cilium.io
+  - ciliumexternalworkloads.cilium.io
+  - ciliumidentities.cilium.io
+  - ciliumlocalredirectpolicies.cilium.io
+  - ciliumnetworkpolicies.cilium.io
+  - ciliumnodes.cilium.io
 {% endif %}
--- a/roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2
@@ -2,11 +2,11 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  name: cilium-operator
+  namespace: kube-system
  labels:
    io.cilium/app: operator
    name: cilium-operator
-  name: cilium-operator
-  namespace: kube-system
 spec:
  replicas: {{ cilium_operator_replicas }}
  selector:
@@ -22,29 +22,22 @@ spec:
    metadata:
 {% if cilium_enable_prometheus %}
      annotations:
-        prometheus.io/port: "6942"
+        prometheus.io/port: "{{ cilium_operator_scrape_port }}"
        prometheus.io/scrape: "true"
 {% endif %}
      labels:
        io.cilium/app: operator
        name: cilium-operator
    spec:
-      # In HA mode, cilium-operator pods must not be scheduled on the same
-      # node as they will clash with each other.
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-              - key: io.cilium/app
-                operator: In
-                values:
-                - operator
-            topologyKey: "kubernetes.io/hostname"
      containers:
-        - args:
-            - --debug=$(CILIUM_DEBUG)
+        - name: cilium-operator
+          image: "{{ cilium_operator_image_repo }}:{{ cilium_operator_image_tag }}"
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          command:
+            - cilium-operator
+          args:
            - --config-dir=/tmp/cilium/config-map
+            - --debug=$(CILIUM_DEBUG)
 {% if cilium_operator_custom_args is string %}
            - {{ cilium_operator_custom_args }}
 {% else %}
@@ -52,14 +45,7 @@ spec:
            - {{ flag }}
 {% endfor %}
 {% endif %}
-          command:
-            - cilium-operator
          env:
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  apiVersion: v1
-                  fieldPath: metadata.namespace
            - name: K8S_NODE_NAME
              valueFrom:
                fieldRef:
@@ -76,45 +62,23 @@ spec:
                  key: debug
                  name: cilium-config
                  optional: true
-# We are already mounting the whole ConfigMap as a directory.
-# https://github.com/cilium/cilium/pull/10347
-{% if cilium_version | regex_replace('v') is version('1.8', '<') %}
-            - name: CILIUM_CLUSTER_NAME
-              valueFrom:
-                configMapKeyRef:
-                  key: cluster-name
-                  name: cilium-config
-                  optional: true
-            - name: CILIUM_CLUSTER_ID
-              valueFrom:
-                configMapKeyRef:
-                  key: cluster-id
-                  name: cilium-config
-                  optional: true
-            - name: CILIUM_DISABLE_ENDPOINT_CRD
-              valueFrom:
-                configMapKeyRef:
-                  key: disable-endpoint-crd
-                  name: cilium-config
-                  optional: true
-{% endif %}
            - name: AWS_ACCESS_KEY_ID
              valueFrom:
                secretKeyRef:
-                  key: AWS_ACCESS_KEY_ID
                  name: cilium-aws
+                  key: AWS_ACCESS_KEY_ID
                  optional: true
            - name: AWS_SECRET_ACCESS_KEY
              valueFrom:
                secretKeyRef:
-                  key: AWS_SECRET_ACCESS_KEY
                  name: cilium-aws
+                  key: AWS_SECRET_ACCESS_KEY
                  optional: true
            - name: AWS_DEFAULT_REGION
              valueFrom:
                secretKeyRef:
-                  key: AWS_DEFAULT_REGION
                  name: cilium-aws
+                  key: AWS_DEFAULT_REGION
                  optional: true
 {% if cilium_kube_proxy_replacement == 'strict' %}
            - name: KUBERNETES_SERVICE_HOST
@@ -122,14 +86,16 @@ spec:
            - name: KUBERNETES_SERVICE_PORT
              value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
 {% endif %}
-          image: "{{ cilium_operator_image_repo }}:{{ cilium_operator_image_tag }}"
-          imagePullPolicy: {{ k8s_image_pull_policy }}
-          name: cilium-operator
 {% if cilium_enable_prometheus %}
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
          ports:
-            - containerPort: 6942
-              hostPort: 6942
-              name: prometheus
+            - name: prometheus
+              containerPort: {{ cilium_operator_scrape_port }}
+              hostPort: {{ cilium_operator_scrape_port }}
              protocol: TCP
 {% endif %}
          livenessProbe:
@@ -146,46 +112,55 @@ spec:
            periodSeconds: 10
            timeoutSeconds: 3
          volumeMounts:
-{% if cilium_identity_allocation_mode == "kvstore" %}
-            - mountPath: /var/lib/etcd-config
-              name: etcd-config-path
+            - name: cilium-config-path
+              mountPath: /tmp/cilium/config-map
              readOnly: true
-            - mountPath: "{{cilium_cert_dir}}"
-              name: etcd-secrets
+{% if cilium_identity_allocation_mode == "kvstore" %}
+            - name: etcd-config-path
+              mountPath: /var/lib/etcd-config
+              readOnly: true
+            - name: etcd-secrets
+              mountPath: "{{cilium_cert_dir}}"
              readOnly: true
 {% endif %}
-            - mountPath: /tmp/cilium/config-map
-              name: cilium-config-path
-              readOnly: true
 {% for volume_mount in cilium_operator_extra_volume_mounts %}
            - {{ volume_mount | to_nice_yaml(indent=2) | indent(14) }}
 {% endfor %}
-      dnsPolicy: ClusterFirst
-      priorityClassName: system-node-critical
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
      restartPolicy: Always
+      priorityClassName: system-node-critical
      serviceAccount: cilium-operator
      serviceAccountName: cilium-operator
-      hostNetwork: true
+      # In HA mode, cilium-operator pods must not be scheduled on the same
+      # node as they will clash with each other.
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - topologyKey: kubernetes.io/hostname
+            labelSelector:
+              matchLabels:
+                io.cilium/app: operator
      tolerations:
        - operator: Exists
      volumes:
+        - name: cilium-config-path
+          configMap:
+            name: cilium-config
 {% if cilium_identity_allocation_mode == "kvstore" %}
        # To read the etcd config stored in config maps
-        - configMap:
+        - name: etcd-config-path
+          configMap:
+            name: cilium-config
            defaultMode: 420
            items:
              - key: etcd-config
                path: etcd.config
-            name: cilium-config
-          name: etcd-config-path
          # To read the k8s etcd secrets in case the user might want to use TLS
        - name: etcd-secrets
          hostPath:
            path: "{{cilium_cert_dir}}"
 {% endif %}
-        - configMap:
-            name: cilium-config
-          name: cilium-config-path
 {% for volume in cilium_operator_extra_volumes %}
        - {{ volume | to_nice_yaml(indent=2) | indent(10) }}
 {% endfor %}