Fix containerd tracing with containerd 2.x (#13630)

* fix(containerd): pass tracing settings as OpenTelemetry environment variables

* fix(containerd): address review on tracing

* fix(containerd): drop tracing comments

roles/container-engine/containerd/defaults/main.yml

@@ -131,7 +131,7 @@ enable_cdi: false
 # For containerd tracing configuration please check out the official documentation:
 # https://github.com/containerd/containerd/blob/main/docs/tracing.md
 containerd_tracing_enabled: false
-containerd_tracing_endpoint: "[::]:4317"
+containerd_tracing_endpoint: "https://localhost:4317"
 containerd_tracing_protocol: "grpc"
 containerd_tracing_sampling_ratio: 1.0
 containerd_tracing_service_name: "containerd"

roles/container-engine/containerd/molecule/default/converge.yml

@@ -4,6 +4,7 @@
   become: true
   vars:
     container_manager: containerd
+    containerd_tracing_enabled: true
   roles:
     - role: kubespray_defaults
     - role: container-engine/containerd

roles/container-engine/containerd/molecule/default/verify.yml

@@ -37,3 +37,24 @@
     assert:
       that:
       - ('Hello from Docker' in nerdctl.results[3].stdout)
+
+- name: Test containerd tracing
+  hosts: all
+  gather_facts: false
+  become: true
+  tasks:
+  - name: Get kubespray defaults
+    import_role:
+      name: ../../../../../kubespray_defaults
+  - name: List containerd plugins and deprecations
+    command: "{{ bin_dir }}/ctr {{ item }}"
+    loop:
+    - plugins ls
+    - deprecations list
+    register: ctr_tracing
+    changed_when: false
+  - name: Check the OTLP exporter is loaded and no tracing config is deprecated
+    assert:
+      that:
+      - ctr_tracing.results[0].stdout_lines | select('search', '^io\.containerd\.tracing\.processor\.v1\s+otlp\s.*\sok\s*$') | list | length == 1
+      - ('tracing' not in ctr_tracing.results[1].stdout)

roles/container-engine/containerd/templates/config.toml.j2

@@ -91,15 +91,3 @@ oom_score = {{ containerd_oom_score }}
 
   [plugins."io.containerd.nri.v1.nri"]
     disable = {{ 'false' if nri_enabled else 'true' }}
-
-{% if containerd_tracing_enabled %}
-  [plugins."io.containerd.tracing.processor.v1.otlp"]
-    endpoint = "{{ containerd_tracing_endpoint }}"
-    protocol = "{{ containerd_tracing_protocol }}"
-{% if containerd_tracing_protocol == "grpc" %}
-    insecure = false
-{% endif %}
-  [plugins."io.containerd.internal.v1.tracing"]
-    sampling_ratio = {{ containerd_tracing_sampling_ratio }}
-    service_name = "{{ containerd_tracing_service_name }}"
-{% endif %}

roles/container-engine/containerd/templates/containerd.service.j2

@@ -40,6 +40,13 @@ OOMScoreAdjust=-999
 {% if kube_reserved is defined and kube_reserved|bool %}
 Slice={{ kube_reserved_cgroups_for_service_slice }}
 {% endif %}
+{% if containerd_tracing_enabled and containerd_version is version('2.0.0', '>=') %}
+Environment="OTEL_EXPORTER_OTLP_ENDPOINT={{ containerd_tracing_endpoint }}"
+Environment="OTEL_EXPORTER_OTLP_PROTOCOL={{ containerd_tracing_protocol }}"
+Environment="OTEL_SERVICE_NAME={{ containerd_tracing_service_name }}"
+Environment="OTEL_TRACES_SAMPLER=parentbased_traceidratio"
+Environment="OTEL_TRACES_SAMPLER_ARG={{ containerd_tracing_sampling_ratio }}"
+{% endif %}
 
 [Install]
 WantedBy=multi-user.target