Overhaul Cilium manifests to match the newer versions (#8717)

* [cilium] Separate templates for cilium, cilium-operator, and hubble installations

Signed-off-by: necatican <necaticanyildirim@gmail.com>

* [cilium] Update cilium-operator templates

Signed-off-by: necatican <necaticanyildirim@gmail.com>

* [cilium] Allow using custom args and mounting extra volumes for the Cilium Operator

Signed-off-by: necatican <necaticanyildirim@gmail.com>

* [cilium] Update the cilium configmap to filter out the deprecated variables, and add the new variables

Signed-off-by: necatican <necaticanyildirim@gmail.com>

* [cilium] Add an option to use Wireguard encryption on Cilium 1.10 and up

Signed-off-by: necatican <necaticanyildirim@gmail.com>

* [cilium] Update cilium-agent templates

Signed-off-by: necatican <necaticanyildirim@gmail.com>

* [cilium] Bump Cilium version to 1.11.3

Signed-off-by: necatican <necaticanyildirim@gmail.com>

roles/network_plugin/cilium/templates/cilium/config.yml.j2

@@ -0,0 +1,230 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cilium-config
+  namespace: kube-system
+data:
+  identity-allocation-mode: {{ cilium_identity_allocation_mode }}
+
+{% if cilium_identity_allocation_mode == "kvstore" %}
+  # This etcd-config contains the etcd endpoints of your cluster. If you use
+  # TLS please make sure you follow the tutorial in https://cilium.link/etcd-config
+  etcd-config: |-
+    ---
+    endpoints:
+{% for ip_addr in etcd_access_addresses.split(',') %}
+      - {{ ip_addr }}
+{% endfor %}
+
+    # In case you want to use TLS in etcd, uncomment the 'ca-file' line
+    # and create a kubernetes secret by following the tutorial in
+    # https://cilium.link/etcd-config
+    ca-file: "{{ cilium_cert_dir }}/ca_cert.crt"
+
+    # In case you want client to server authentication, uncomment the following
+    # lines and create a kubernetes secret by following the tutorial in
+    # https://cilium.link/etcd-config
+    key-file: "{{ cilium_cert_dir }}/key.pem"
+    cert-file: "{{ cilium_cert_dir }}/cert.crt"
+
+  # kvstore
+  # https://docs.cilium.io/en/latest/cmdref/kvstore/
+  kvstore: etcd
+  kvstore-opt: '{"etcd.config": "/var/lib/etcd-config/etcd.config"}'
+{% endif %}
+
+  # If you want metrics enabled in all of your Cilium agents, set the port for
+  # which the Cilium agents will have their metrics exposed.
+  # This option deprecates the "prometheus-serve-addr" in the
+  # "cilium-metrics-config" ConfigMap
+  # NOTE that this will open the port on ALL nodes where Cilium pods are
+  # scheduled.
+{% if cilium_enable_prometheus %}
+  prometheus-serve-addr: ":9090"
+  operator-prometheus-serve-addr: ":6942"
+  enable-metrics: "true"
+{% endif %}
+
+  # If you want to run cilium in debug mode change this value to true
+  debug: "{{ cilium_debug }}"
+  enable-ipv4: "{{ cilium_enable_ipv4 }}"
+  enable-ipv6: "{{ cilium_enable_ipv6 }}"
+  # If a serious issue occurs during Cilium startup, this
+  # invasive option may be set to true to remove all persistent
+  # state. Endpoints will not be restored using knowledge from a
+  # prior Cilium run, so they may receive new IP addresses upon
+  # restart. This also triggers clean-cilium-bpf-state.
+  clean-cilium-state: "false"
+  # If you want to clean cilium BPF state, set this to true;
+  # Removes all BPF maps from the filesystem. Upon restart,
+  # endpoints are restored with the same IP addresses, however
+  # any ongoing connections may be disrupted briefly.
+  # Loadbalancing decisions will be reset, so any ongoing
+  # connections via a service may be loadbalanced to a different
+  # backend after restart.
+  clean-cilium-bpf-state: "false"
+
+  # Users who wish to specify their own custom CNI configuration file must set
+  # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration.
+  custom-cni-conf: "false"
+
+  # If you want cilium monitor to aggregate tracing for packets, set this level
+  # to "low", "medium", or "maximum". The higher the level, the less packets
+  # that will be seen in monitor output.
+  monitor-aggregation: "{{ cilium_monitor_aggregation }}"
+
+  # ct-global-max-entries-* specifies the maximum number of connections
+  # supported across all endpoints, split by protocol: tcp or other. One pair
+  # of maps uses these values for IPv4 connections, and another pair of maps
+  # use these values for IPv6 connections.
+  #
+  # If these values are modified, then during the next Cilium startup the
+  # tracking of ongoing connections may be disrupted. This may lead to brief
+  # policy drops or a change in loadbalancing decisions for a connection.
+  #
+  # For users upgrading from Cilium 1.2 or earlier, to minimize disruption
+  # during the upgrade process, comment out these options.
+  bpf-ct-global-tcp-max: "524288"
+  bpf-ct-global-any-max: "262144"
+
+  # Pre-allocation of map entries allows per-packet latency to be reduced, at
+  # the expense of up-front memory allocation for the entries in the maps. The
+  # default value below will minimize memory usage in the default installation;
+  # users who are sensitive to latency may consider setting this to "true".
+  #
+  # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore
+  # this option and behave as though it is set to "true".
+  #
+  # If this value is modified, then during the next Cilium startup the restore
+  # of existing endpoints and tracking of ongoing connections may be disrupted.
+  # This may lead to policy drops or a change in loadbalancing decisions for a
+  # connection for some time. Endpoints may need to be recreated to restore
+  # connectivity.
+  #
+  # If this option is set to "false" during an upgrade from 1.3 or earlier to
+  # 1.4 or later, then it may cause one-time disruptions during the upgrade.
+  preallocate-bpf-maps: "{{cilium_preallocate_bpf_maps}}"
+
+  # Regular expression matching compatible Istio sidecar istio-proxy
+  # container image names
+  sidecar-istio-proxy-image: "cilium/istio_proxy"
+
+  # Encapsulation mode for communication between nodes
+  # Possible values:
+  #   - disabled
+  #   - vxlan (default)
+  #   - geneve
+  tunnel: "{{ cilium_tunnel_mode }}"
+
+  # Name of the cluster. Only relevant when building a mesh of clusters.
+  cluster-name: "{{ cilium_cluster_name }}"
+
+  # Unique ID of the cluster. Must be unique across all conneted clusters and
+  # in the range of 1 and 255. Only relevant when building a mesh of clusters.
+  #cluster-id: 1
+{% if cilium_cluster_id is defined %}
+  cluster-id: "{{ cilium_cluster_id }}"
+{% endif %}
+
+# `tofqdns-enable-poller` is deprecated in 1.8, removed in 1.9
+# https://github.com/cilium/cilium/issues/8604
+{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
+  # DNS Polling periodically issues a DNS lookup for each `matchName` from
+  # cilium-agent. The result is used to regenerate endpoint policy.
+  # DNS lookups are repeated with an interval of 5 seconds, and are made for
+  # A(IPv4) and AAAA(IPv6) addresses. Should a lookup fail, the most recent IP
+  # data is used instead. An IP change will trigger a regeneration of the Cilium
+  # policy for each endpoint and increment the per cilium-agent policy
+  # repository revision.
+  #
+  # This option is disabled by default starting from version 1.4.x in favor
+  # of a more powerful DNS proxy-based implementation, see [0] for details.
+  # Enable this option if you want to use FQDN policies but do not want to use
+  # the DNS proxy.
+  #
+  # To ease upgrade, users may opt to set this option to "true".
+  # Otherwise please refer to the Upgrade Guide [1] which explains how to
+  # prepare policy rules for upgrade.
+  #
+  # [0] http://docs.cilium.io/en/stable/policy/language/#dns-based
+  # [1] http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action
+  tofqdns-enable-poller: "{{cilium_tofqdns_enable_poller}}"
+{% endif %}
+
+# `wait-bpf-mount` is removed after v1.10.4
+# https://github.com/cilium/cilium/commit/d2217045cb3726a7f823174e086913b69b8090da
+{% if cilium_version | regex_replace('v') is version('1.10.4', '<') %}
+  # wait-bpf-mount makes init container wait until bpf filesystem is mounted
+  wait-bpf-mount: "false"
+{% endif %}
+
+# `enable-legacy-services` is deprecated in 1.6, removed in 1.9
+# https://github.com/cilium/cilium/pull/10255
+{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
+  # Enable legacy services (prior v1.5) to prevent from terminating existing
+  # connections with services when upgrading Cilium from < v1.5 to v1.5.
+  enable-legacy-services: "{{cilium_enable_legacy_services}}"
+{% endif %}
+
+  kube-proxy-replacement: "{{ cilium_kube_proxy_replacement }}"
+
+# `native-routing-cidr` is deprecated in 1.10, removed in 1.12.
+# Replaced by `ipv4-native-routing-cidr`
+# https://github.com/cilium/cilium/pull/16695
+{% if cilium_version | regex_replace('v') is version('1.12', '<') %}
+  native-routing-cidr: "{{ cilium_native_routing_cidr }}"
+{% else %}
+  {% if cilium_native_routing_cidr | length %}
+  ipv4-native-routing-cidr: "{{ cilium_native_routing_cidr }}"
+  {% endif %}
+  {% if cilium_native_routing_cidr_ipv6 | length %}
+  ipv6-native-routing-cidr: "{{ cilium_native_routing_cidr_ipv6 }}"
+  {% endif %}
+{% endif %}
+
+  auto-direct-node-routes: "{{ cilium_auto_direct_node_routes }}"
+
+  operator-api-serve-addr: "{{ cilium_operator_api_serve_addr }}"
+
+  # Hubble settings
+{% if cilium_enable_hubble %}
+  enable-hubble: "true"
+{% if cilium_enable_hubble_metrics %}
+  hubble-metrics-server: ":9091"
+  hubble-metrics:
+{% for hubble_metrics_cycle in cilium_hubble_metrics %}
+    {{ hubble_metrics_cycle }}
+{% endfor %}
+{% endif %}
+  hubble-listen-address: ":4244"
+{% if cilium_enable_hubble and cilium_hubble_install %}
+  hubble-disable-tls: "false"
+  hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt
+  hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key
+  hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt
+{% endif %}
+{% endif %}
+
+{% for key, value in cilium_config_extra_vars.items() %}
+  {{ key }}: "{{ value }}"
+{% endfor %}
+
+  # Enable transparent network encryption
+{% if cilium_encryption_enabled %}
+  {% if cilium_encryption_type == "ipsec" %}
+  enable-ipsec: "true"
+  ipsec-key-file: /etc/ipsec/keys
+  encrypt-node: "{{ cilium_ipsec_node_encryption }}"
+  {% endif %}
+
+  {% if cilium_encryption_type == "wireguard" %}
+  enable-wireguard: "true"
+  enable-wireguard-userspace-fallback: "{{ cilium_wireguard_userspace_fallback }}"
+  {% endif %}
+{% endif %}
+
+  # IPAM settings
+{% if cilium_version | regex_replace('v') is version('1.9', '>=') %}
+  ipam: "{{ cilium_ipam_mode }}"
+{% endif %}

roles/network_plugin/cilium/templates/cilium/cr.yml.j2

@@ -0,0 +1,134 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cilium
+rules:
+- apiGroups:
+  - networking.k8s.io
+  resources:
+{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
+  - ingresses
+{% endif %}
+  - networkpolicies
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  - services
+  - nodes
+  - endpoints
+{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
+  - componentstatuses
+{% endif %}
+  verbs:
+  - get
+  - list
+  - watch
+{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
+- apiGroups:
+  - extensions
+  resources:
+  - ingresses
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.7', '>') %}
+- apiGroups:
+  - ""
+  resources:
+  - pods/finalizers
+  verbs:
+  - update
+{% endif %}
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumnetworkpolicies
+  - ciliumnetworkpolicies/status
+{% if cilium_version | regex_replace('v') is version('1.7', '>=') %}
+  - ciliumclusterwidenetworkpolicies
+  - ciliumclusterwidenetworkpolicies/status
+{% endif %}
+  - ciliumendpoints
+  - ciliumendpoints/status
+{% if cilium_version | regex_replace('v') is version('1.6', '>=') %}
+  - ciliumnodes
+  - ciliumnodes/status
+  - ciliumidentities
+  - ciliumidentities/status
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.9', '>=') %}
+  - ciliumnetworkpolicies/finalizers
+  - ciliumclusterwidenetworkpolicies/finalizers
+  - ciliumendpoints/finalizers
+  - ciliumnodes/finalizers
+  - ciliumidentities/finalizers
+  - ciliumlocalredirectpolicies
+  - ciliumlocalredirectpolicies/status
+  - ciliumlocalredirectpolicies/finalizers
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.10', '>=') %}
+  - ciliumegressnatpolicies
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
+  - ciliumendpointslices
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
+  - ciliumbgploadbalancerippools
+  - ciliumbgppeeringpolicies
+{% endif %}
+  verbs:
+  - '*'
+{% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumclusterwideenvoyconfigs
+  - ciliumenvoyconfigs
+  verbs:
+  - list
+  - watch
+{% endif %}

roles/network_plugin/cilium/templates/cilium/crb.yml.j2

@@ -0,0 +1,18 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cilium
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cilium
+subjects:
+- kind: ServiceAccount
+  name: cilium
+  namespace: kube-system
+{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:nodes
+{% endif %}

roles/network_plugin/cilium/templates/cilium/ds.yml.j2

@@ -0,0 +1,398 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    k8s-app: cilium
+  name: cilium
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: cilium
+  template:
+    metadata:
+      annotations:
+{% if cilium_enable_prometheus %}
+        prometheus.io/port: "9090"
+        prometheus.io/scrape: "true"
+{% endif %}
+        scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated","operator":"Equal","value":"master","effect":"NoSchedule"}]'
+      labels:
+        k8s-app: cilium
+    spec:
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: k8s-app
+                operator: In
+                values:
+                - cilium
+            topologyKey: kubernetes.io/hostname
+      containers:
+      - args:
+        - --config-dir=/tmp/cilium/config-map
+{% if cilium_mtu != "" %}
+        - --mtu={{ cilium_mtu }}
+{% endif %}
+{% if cilium_agent_custom_args is string %}
+        - {{ cilium_agent_custom_args }}
+{% else %}
+{% for flag in cilium_agent_custom_args %}
+        - {{ flag }}
+{% endfor %}
+{% endif %}
+        command:
+        - cilium-agent
+        env:
+        - name: K8S_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: CILIUM_K8S_NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        - name: CILIUM_CLUSTERMESH_CONFIG
+          value: /var/lib/cilium/clustermesh/
+{% if cilium_kube_proxy_replacement == 'strict' %}
+        - name: KUBERNETES_SERVICE_HOST
+          value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}"
+        - name: KUBERNETES_SERVICE_PORT
+          value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
+{% endif %}
+{% for env_var in cilium_agent_extra_env_vars %}
+        - {{ env_var | to_nice_yaml(indent=2) | indent(10) }}
+{% endfor %}
+        image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        resources:
+          limits:
+            cpu: {{ cilium_cpu_limit }}
+            memory: {{ cilium_memory_limit }}
+          requests:
+            cpu: {{ cilium_cpu_requests }}
+            memory: {{ cilium_memory_requests }}
+        lifecycle:
+          postStart:
+            exec:
+              command:
+              - "/cni-install.sh"
+{% if cilium_version | regex_replace('v') is version('1.10', '>=') %}
+              - "--cni-exclusive={{ cilium_cni_exclusive }}"
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
+              - "--log-file={{ cilium_cni_log_file }}"
+{% endif %}
+          preStop:
+            exec:
+              command:
+              - /cni-uninstall.sh
+        startupProbe:
+          httpGet:
+            host: '127.0.0.1'
+            path: /healthz
+            port: 9876
+            scheme: HTTP
+            httpHeaders:
+            - name: "brief"
+              value: "true"
+          failureThreshold: 105
+          periodSeconds: 2
+          successThreshold: 1
+        livenessProbe:
+          httpGet:
+            host: '127.0.0.1'
+            path: /healthz
+            port: 9876
+            scheme: HTTP
+            httpHeaders:
+            - name: "brief"
+              value: "true"
+          failureThreshold: 10
+          periodSeconds: 30
+          successThreshold: 1
+          timeoutSeconds: 5
+        readinessProbe:
+          httpGet:
+            host: 127.0.0.1
+            path: /healthz
+            port: 9876
+            scheme: HTTP
+            httpHeaders:
+            - name: "brief"
+              value: "true"
+          initialDelaySeconds: 5
+          periodSeconds: 30
+          successThreshold: 1
+          failureThreshold: 3
+          timeoutSeconds: 5
+        name: cilium-agent
+{% if cilium_enable_prometheus or cilium_enable_hubble_metrics %}
+        ports:
+{% endif %}
+{% if cilium_enable_prometheus %}
+        - containerPort: 9090
+          hostPort: 9090
+          name: prometheus
+          protocol: TCP
+{% endif %}
+{% if cilium_enable_hubble_metrics %}
+        - containerPort: 9091
+          hostPort: 9091
+          name: hubble-metrics
+          protocol: TCP
+{% endif %}
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /sys/fs/bpf
+          name: bpf-maps
+          mountPropagation: Bidirectional
+        - mountPath: /var/run/cilium
+          name: cilium-run
+        - mountPath: /host/opt/cni/bin
+          name: cni-path
+        - mountPath: /host/etc/cni/net.d
+          name: etc-cni-netd
+# pkg/workloads was depreca, removed in 1.7
+# https://github.com/cilium/cilium/pull/9447
+{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
+{% if container_manager == 'docker' %}
+        - mountPath: /var/run/docker.sock
+          name: docker-socket
+          readOnly: true
+{% else %}
+        - name: "{{ container_manager }}-socket"
+          mountPath: {{ cri_socket }}
+          readOnly: true
+{% endif %}
+{% endif %}
+{% if cilium_identity_allocation_mode == "kvstore" %}
+        - mountPath: /var/lib/etcd-config
+          name: etcd-config-path
+          readOnly: true
+        - mountPath: "{{cilium_cert_dir}}"
+          name: etcd-secrets
+          readOnly: true
+{% endif %}
+        - mountPath: /var/lib/cilium/clustermesh
+          name: clustermesh-secrets
+          readOnly: true
+        - mountPath: /tmp/cilium/config-map
+          name: cilium-config-path
+          readOnly: true
+          # Needed to be able to load kernel modules
+        - mountPath: /lib/modules
+          name: lib-modules
+          readOnly: true
+        - mountPath: /run/xtables.lock
+          name: xtables-lock
+{% if cilium_encryption_enabled and cilium_encryption_type == "ipsec" %}
+        - mountPath: /etc/ipsec
+          name: cilium-ipsec-secrets
+          readOnly: true
+{% endif %}
+{% if cilium_hubble_install %}
+        - mountPath: /var/lib/cilium/tls/hubble
+          name: hubble-tls
+          readOnly: true
+{% endif %}
+{% for volume_mount in cilium_agent_extra_volume_mounts %}
+        - {{ volume_mount | to_nice_yaml(indent=2) | indent(10) }}
+{% endfor %}
+# In managed etcd mode, Cilium must be able to resolve the DNS name of the etcd service
+{% if cilium_identity_allocation_mode == "kvstore" %}
+      dnsPolicy: ClusterFirstWithHostNet
+{% endif %}
+      hostNetwork: true
+      initContainers:
+      - command:
+        - /init-container.sh
+        env:
+        - name: CILIUM_ALL_STATE
+          valueFrom:
+            configMapKeyRef:
+              key: clean-cilium-state
+              name: cilium-config
+              optional: true
+        # CLEAN_CILIUM_BPF_STATE is deprecated in 1.6.
+        # https://github.com/cilium/cilium/pull/7478
+        - name: "{{ cilium_version | regex_replace('v') is version('1.6', '<')| ternary('CLEAN_CILIUM_BPF_STATE','CILIUM_BPF_STATE') }}"
+          valueFrom:
+            configMapKeyRef:
+              key: clean-cilium-bpf-state
+              name: cilium-config
+              optional: true
+# Removed in 1.11 and up.
+# https://github.com/cilium/cilium/commit/f7a3f59fd74983c600bfce9cac364b76d20849d9
+{% if cilium_version | regex_replace('v') is version('1.11', '<') %}
+        - name: CILIUM_WAIT_BPF_MOUNT
+          valueFrom:
+            configMapKeyRef:
+              key: wait-bpf-mount
+              name: cilium-config
+              optional: true
+{% endif %}
+{% if cilium_kube_proxy_replacement == 'strict' %}
+        - name: KUBERNETES_SERVICE_HOST
+          value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}"
+        - name: KUBERNETES_SERVICE_PORT
+          value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
+        image: "{{cilium_init_image_repo}}:{{cilium_init_image_tag}}"
+{% else %}
+        image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
+{% endif %}
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        name: clean-cilium-state
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /sys/fs/bpf
+          name: bpf-maps
+{% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
+          # Required to mount cgroup filesystem from the host to cilium agent pod
+        - name: cilium-cgroup
+          mountPath: /run/cilium/cgroupv2
+          mountPropagation: HostToContainer
+{% endif %}
+        - mountPath: /var/run/cilium
+          name: cilium-run
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+      priorityClassName: system-node-critical
+      restartPolicy: Always
+      serviceAccount: cilium
+      serviceAccountName: cilium
+      terminationGracePeriodSeconds: 1
+      hostNetwork: true
+# In managed etcd mode, Cilium must be able to resolve the DNS name of the etcd service
+{% if cilium_identity_allocation_mode == "kvstore" %}
+      dnsPolicy: ClusterFirstWithHostNet
+{% endif %}
+      tolerations:
+      - operator: Exists
+      volumes:
+        # To keep state between restarts / upgrades
+      - hostPath:
+          path: /var/run/cilium
+          type: DirectoryOrCreate
+        name: cilium-run
+        # To keep state between restarts / upgrades for bpf maps
+      - hostPath:
+          path: /sys/fs/bpf
+          type: DirectoryOrCreate
+        name: bpf-maps
+# pkg/workloads was deprecated in 1.6, removed in 1.7
+# https://github.com/cilium/cilium/pull/9447
+{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
+{% if container_manager == 'docker' %}
+        # To read docker events from the node
+      - hostPath:
+          path: /var/run/docker.sock
+          type: Socket
+        name: docker-socket
+{% else %}
+        # To read crio events from the node
+      - hostPath:
+          path: {{ cri_socket }}
+          type: Socket
+        name: {{ container_manager }}-socket
+{% endif %}
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
+      # To mount cgroup2 filesystem on the host
+      - name: hostproc
+        hostPath:
+          path: /proc
+          type: Directory
+      # To keep state between restarts / upgrades for cgroup2 filesystem
+      - name: cilium-cgroup
+        hostPath:
+          path: /run/cilium/cgroupv2
+          type: DirectoryOrCreate
+{% endif %}
+        # To install cilium cni plugin in the host
+      - hostPath:
+          path: /opt/cni/bin
+          type: DirectoryOrCreate
+        name: cni-path
+        # To install cilium cni configuration in the host
+      - hostPath:
+          path: /etc/cni/net.d
+          type: DirectoryOrCreate
+        name: etc-cni-netd
+        # To be able to load kernel modules
+      - hostPath:
+          path: /lib/modules
+        name: lib-modules
+        # To access iptables concurrently with other processes (e.g. kube-proxy)
+      - hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+        name: xtables-lock
+{% if cilium_identity_allocation_mode == "kvstore" %}
+        # To read the etcd config stored in config maps
+      - configMap:
+          # note: the leading zero means this number is in octal representation: do not remove it
+          defaultMode: 0400
+          items:
+          - key: etcd-config
+            path: etcd.config
+          name: cilium-config
+        name: etcd-config-path
+        # To read the k8s etcd secrets in case the user might want to use TLS
+      - name: etcd-secrets
+        hostPath:
+          path: "{{cilium_cert_dir}}"
+{% endif %}
+        # To read the clustermesh configuration
+      - name: clustermesh-secrets
+        secret:
+          # note: the leading zero means this number is in octal representation: do not remove it
+          defaultMode: 0400
+          optional: true
+          secretName: cilium-clustermesh
+        # To read the configuration from the config map
+      - configMap:
+          name: cilium-config
+        name: cilium-config-path
+{% if cilium_encryption_enabled and cilium_encryption_type == "ipsec" %}
+      - name: cilium-ipsec-secrets
+        secret:
+          secretName: cilium-ipsec-keys
+{% endif %}
+{% if cilium_hubble_install %}
+      - name: hubble-tls
+        projected:
+          # note: the leading zero means this number is in octal representation: do not remove it
+          defaultMode: 0400
+          sources:
+          - secret:
+              name: hubble-server-certs
+              items:
+                - key: tls.crt
+                  path: server.crt
+                - key: tls.key
+                  path: server.key
+              optional: true
+          - configMap:
+              name: hubble-ca-cert
+              items:
+                - key: ca.crt
+                  path: client-ca.crt
+              optional: true
+{% endif %}
+  updateStrategy:
+    rollingUpdate:
+      # Specifies the maximum number of Pods that can be unavailable during the update process.
+      maxUnavailable: 2
+    type: RollingUpdate

roles/network_plugin/cilium/templates/cilium/sa.yml.j2

@@ -0,0 +1,6 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cilium
+  namespace: kube-system

roles/network_plugin/cilium/templates/cilium/secret.yml.j2

@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+data:
+  keys: {{ cilium_ipsec_key }}
+kind: Secret
+metadata:
+  name: cilium-ipsec-keys
+  namespace: kube-system
+type: Opaque