Fix containerd config_path mirrors and remove nerdctl insecure_registry (#10196)

* Fix containerd_registries in config_path for mirrors and remove nerdctl global insecure_registry setting

* Make containerd hosts.toml mode 0640

* Add containerd_registries_mirrors and keep containerd_registries to pass packet_debian11-calico-upgrade

roles/container-engine/containerd/defaults/main.yml

@@ -50,6 +50,13 @@ containerd_metrics_grpc_histogram: false
 containerd_registries:
   "docker.io": "https://registry-1.docker.io"
 
+containerd_registries_mirrors:
+  - prefix: docker.io
+    mirrors:
+      - host: https://registry-1.docker.io
+        capabilities: ["pull", "resolve"]
+        skip_verify: false
+
 containerd_max_container_log_line_size: -1
 
 # If enabled it will allow non root users to use port numbers <1024
@@ -74,7 +81,7 @@ containerd_limit_core: "infinity"
 containerd_limit_open_file_num: "infinity"
 containerd_limit_mem_lock: "infinity"
 
-# If enabled it will use config_path and disable use mirrors config
+# If enabled it will use config_path and config to be put in {{ containerd_cfg_dir }}/certs.d/
 containerd_use_config_path: false
 
 # OS distributions that already support containerd

roles/container-engine/containerd/tasks/main.yml

@@ -112,26 +112,20 @@
   notify: Restart containerd
 
 - name: Containerd | Configure containerd registries
-  when: containerd_use_config_path is defined and containerd_use_config_path | bool and containerd_insecure_registries is defined
+  when: containerd_registries_mirrors is defined
   block:
-    - name: Containerd ｜ Create registry directories
+    - name: Containerd | Create registry directories
       file:
-        path: "{{ containerd_cfg_dir }}/certs.d/{{ item.key }}"
+        path: "{{ containerd_cfg_dir }}/certs.d/{{ item.prefix }}"
         state: directory
         mode: 0755
-        recurse: true
-      with_dict: "{{ containerd_insecure_registries }}"
-    - name: Containerd ｜ Write hosts.toml file
-      blockinfile:
-        path: "{{ containerd_cfg_dir }}/certs.d/{{ item.key }}/hosts.toml"
+      loop: "{{ containerd_registries_mirrors }}"
+    - name: Containerd | Write hosts.toml file
+      template:
+        src: hosts.toml.j2
+        dest: "{{ containerd_cfg_dir }}/certs.d/{{ item.prefix }}/hosts.toml"
         mode: 0640
-        create: true
-        block: |
-          server = "{{ item.value }}"
-          [host."{{ item.value }}"]
-            capabilities = ["pull", "resolve", "push"]
-            skip_verify = true
-      with_dict: "{{ containerd_insecure_registries }}"
+      loop: "{{ containerd_registries_mirrors }}"
 
 # you can sometimes end up in a state where everything is installed
 # but containerd was not started / enabled

roles/container-engine/containerd/templates/config.toml.j2

@@ -51,18 +51,18 @@ oom_score = {{ containerd_oom_score }}
       config_path = "{{ containerd_cfg_dir }}/certs.d"
 {% else %}
       [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
-{% for registry, addr in containerd_registries.items() %}
-        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry }}"]
-          endpoint = ["{{ ([ addr ] | flatten ) | join('","') }}"]
+{% set insecure_registries_addr = [] %}
+{% for registry in containerd_registries_mirrors %}
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry.prefix }}"]
+{% set endpoint = [] %}
+{% for mirror in registry.mirrors %}
+{% if endpoint.append(mirror.host) %}{% endif %}
+{% if mirror.skip_verify is defined and mirror.skip_verify|bool %}{% if insecure_registries_addr.append(mirror.host | urlsplit('netloc')) %}{% endif %}{% endif %}
 {% endfor %}
-{% if containerd_insecure_registries is defined and containerd_insecure_registries|length>0 %}
-{% for registry, addr in containerd_insecure_registries.items() %}
-        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry }}"]
-          endpoint = ["{{ ([ addr ] | flatten ) | join('","') }}"]
+          endpoint = ["{{ ( endpoint | unique ) | join('","') }}"]
 {% endfor %}
-{% endif %}
-{% for addr in containerd_insecure_registries.values() | flatten | unique %}
-        [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ addr | urlsplit('netloc') }}".tls]
+{% for addr in insecure_registries_addr | unique %}
+        [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ addr }}".tls]
           insecure_skip_verify = true
 {% endfor %}
 {% endif %}

roles/container-engine/containerd/templates/hosts.toml.j2

@@ -0,0 +1,8 @@
+server = "https://{{ item.prefix }}"
+{% for mirror in item.mirrors %}
+[host."{{ mirror.host }}"]
+  capabilities = ["{{ ([ mirror.capabilities ] | flatten ) | join('","') }}"]
+{% if mirror.skip_verify is defined %}
+  skip_verify = {{ mirror.skip_verify | default('false') | string | lower }}
+{% endif %}
+{% endfor %}

roles/container-engine/nerdctl/templates/nerdctl.toml.j2

@@ -6,5 +6,4 @@ snapshotter       = "{{ nerdctl_snapshotter | default('overlayfs') }}"
 cni_path          = "/opt/cni/bin"
 cni_netconfpath   = "/etc/cni/net.d"
 cgroup_manager    = "{{ kubelet_cgroup_driver | default('systemd') }}"
-insecure_registry = {{ (containerd_insecure_registries is defined and containerd_insecure_registries|length>0) | bool | lower }}
-hosts_dir         = ["/etc/containerd/certs.d"]
+hosts_dir         = ["{{ containerd_cfg_dir }}/certs.d"]