gVisor: initial support for gVisor container runtime (#7661)

* Docker/Containerd: move downloads urls to containerd-common

* gVisor: initial support for gVisor container runtime

roles/container-engine/containerd-common/defaults/main.yml

@@ -1,2 +1,17 @@
 ---
 containerd_package: 'containerd.io'
+
+# Fedora docker-ce repo
+docker_fedora_repo_base_url: 'https://download.docker.com/linux/fedora/{{ ansible_distribution_major_version }}/$basearch/stable'
+docker_fedora_repo_gpgkey: 'https://download.docker.com/linux/fedora/gpg'
+# CentOS/RedHat docker-ce repo
+docker_rh_repo_base_url: 'https://download.docker.com/linux/centos/{{ ansible_distribution_major_version }}/$basearch/stable'
+docker_rh_repo_gpgkey: 'https://download.docker.com/linux/centos/gpg'
+# Ubuntu docker-ce repo
+docker_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu"
+docker_ubuntu_repo_gpgkey: 'https://download.docker.com/linux/ubuntu/gpg'
+docker_ubuntu_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'
+# Debian docker-ce repo
+docker_debian_repo_base_url: "https://download.docker.com/linux/debian"
+docker_debian_repo_gpgkey: 'https://download.docker.com/linux/debian/gpg'
+docker_debian_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'

roles/container-engine/containerd/templates/config.toml.j2

@@ -35,6 +35,10 @@ oom_score = {{ containerd_oom_score }}
 {% if kata_containers_enabled %}
         [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu]
           runtime_type = "io.containerd.kata-qemu.v2"
+{% endif %}
+{% if gvisor_enabled %}
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc]
+          runtime_type = "io.containerd.runsc.v1"
 {% endif %}
     [plugins."io.containerd.grpc.v1.cri".registry]
       [plugins."io.containerd.grpc.v1.cri".registry.mirrors]

roles/container-engine/docker/defaults/main.yml

@@ -15,20 +15,6 @@ docker_cgroup_driver: systemd
 
 yum_repo_dir: /etc/yum.repos.d
 
-# Fedora docker-ce repo
-docker_fedora_repo_base_url: 'https://download.docker.com/linux/fedora/{{ ansible_distribution_major_version }}/$basearch/stable'
-docker_fedora_repo_gpgkey: 'https://download.docker.com/linux/fedora/gpg'
-# CentOS/RedHat docker-ce repo
-docker_rh_repo_base_url: 'https://download.docker.com/linux/centos/{{ ansible_distribution_major_version }}/$basearch/stable'
-docker_rh_repo_gpgkey: 'https://download.docker.com/linux/centos/gpg'
-# Ubuntu docker-ce repo
-docker_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu"
-docker_ubuntu_repo_gpgkey: 'https://download.docker.com/linux/ubuntu/gpg'
-docker_ubuntu_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'
-# Debian docker-ce repo
-docker_debian_repo_base_url: "https://download.docker.com/linux/debian"
-docker_debian_repo_gpgkey: 'https://download.docker.com/linux/debian/gpg'
-docker_debian_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'
 docker_bin_dir: "/usr/bin"
 
 # flag to enable/disable docker cleanup

roles/container-engine/gvisor/molecule/default/converge.yml

@@ -0,0 +1,11 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+  vars:
+    gvisor_enabled: true
+    container_manager: containerd
+  roles:
+    - role: kubespray-defaults
+    - role: containerd
+    - role: gvisor

roles/container-engine/gvisor/molecule/default/files/10-mynet.conf

@@ -0,0 +1,17 @@
+{
+  "cniVersion": "0.2.0",
+  "name": "mynet",
+  "type": "bridge",
+  "bridge": "cni0",
+  "isGateway": true,
+  "ipMasq": true,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "172.19.0.0/24",
+    "routes": [
+      {
+        "dst": "0.0.0.0/0"
+      }
+    ]
+  }
+}

roles/container-engine/gvisor/molecule/default/files/container.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "gvisor1"
+  },
+  "image": {
+    "image": "docker.io/library/hello-world:latest"
+  },
+  "log_path": "gvisor1.0.log",
+  "linux": {}
+}

roles/container-engine/gvisor/molecule/default/files/sandbox.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "gvisor1",
+    "namespace": "default",
+    "attempt": 1,
+    "uid": "hdishd83djaidwnduwk28bcsb"
+  },
+  "linux": {},
+  "log_directory": "/tmp"
+}

roles/container-engine/gvisor/molecule/default/molecule.yml

@@ -0,0 +1,44 @@
+---
+driver:
+  name: vagrant
+  provider:
+    name: libvirt
+    options:
+      driver: kvm
+lint: |
+  set -e
+  yamllint -c ../../../.yamllint .
+platforms:
+  - name: ubuntu20
+    box: generic/ubuntu2004
+    cpus: 1
+    memory: 1024
+    nested: true
+    groups:
+      - kube_control_plane
+  - name: centos8
+    box: generic/centos8
+    cpus: 1
+    memory: 1024
+    nested: true
+    groups:
+      - kube_control_plane
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_ROLES_PATH: ../../../../
+  config_options:
+    defaults:
+      callback_whitelist: profile_tasks
+  lint:
+    name: ansible-lint
+    options:
+      c: ../../../.ansible-lint
+  inventory:
+    group_vars:
+      all:
+        become: true
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

roles/container-engine/gvisor/molecule/default/prepare.yml

@@ -0,0 +1,48 @@
+---
+- name: Prepare generic
+  hosts: all
+  become: true
+  roles:
+    - role: kubespray-defaults
+    - role: bootstrap-os
+    - role: ../adduser
+      user: "{{ addusers.kube }}"
+  tasks:
+    - include_tasks: "../../../../download/tasks/download_file.yml"
+      vars:
+        download: "{{ download_defaults | combine(downloads.cni) }}"
+
+- name: Prepare container runtime
+  hosts: all
+  become: true
+  vars:
+    container_manager: containerd
+    kube_network_plugin: cni
+  roles:
+    - role: kubespray-defaults
+    - role: ../network_plugin/cni
+    - role: crictl
+  tasks:
+    - name: Copy test container files
+      copy:
+        src: "{{ item }}"
+        dest: "/tmp/{{ item }}"
+        owner: root
+        mode: 0644
+      with_items:
+        - container.json
+        - sandbox.json
+    - name: Create /etc/cni/net.d directory
+      file:
+        path: /etc/cni/net.d
+        state: directory
+        owner: kube
+        mode: 0755
+    - name: Setup CNI
+      copy:
+        src: "{{ item }}"
+        dest: "/etc/cni/net.d/{{ item }}"
+        owner: root
+        mode: 0644
+      with_items:
+        - 10-mynet.conf

roles/container-engine/gvisor/molecule/default/tests/test_default.py

@@ -0,0 +1,29 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_run(host):
+    gvisorruntime = "/usr/local/bin/runsc"
+    with host.sudo():
+        cmd = host.command(gvisorruntime + " --version")
+    assert cmd.rc == 0
+    assert "runsc version" in cmd.stdout
+
+
+def test_run_pod(host):
+    runtime = "runsc"
+
+    run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime)
+    with host.sudo():
+        cmd = host.command(run_command)
+    assert cmd.rc == 0
+
+    with host.sudo():
+      log_f = host.file("/tmp/gvisor1.0.log")
+
+      assert log_f.exists
+      assert b"Hello from Docker!" in log_f.content

roles/container-engine/gvisor/tasks/main.yml

@@ -0,0 +1,20 @@
+---
+- name: gVisor | Download runsc binary
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.gvisor_runsc) }}"
+
+- name: gVisor | Download containerd-shim-runsc-v1 binary
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.gvisor_containerd_shim) }}"
+
+- name: gVisor | Copy binaries
+  copy:
+    src: "{{ local_release_dir }}/gvisor-{{ item }}"
+    dest: "{{ bin_dir }}/{{ item }}"
+    mode: 0755
+    remote_src: yes
+  with_items:
+    - runsc
+    - containerd-shim-runsc-v1

roles/container-engine/meta/main.yml

@@ -7,6 +7,14 @@ dependencies:
       - container-engine
       - kata-containers
 
+  - role: container-engine/gvisor
+    when:
+      - gvisor_enabled
+      - container_manager in ['docker', 'containerd']
+    tags:
+      - container-engine
+      - gvisor
+
   - role: container-engine/crun
     when:
       - crun_enabled

roles/download/defaults/main.yml

@@ -53,6 +53,7 @@ kubeadm_version: "{{ kube_version }}"
 etcd_version: v3.4.13
 crun_version: 0.19
 kata_containers_version: 2.1.0
+gvisor_version: 20210518.0
 
 # gcr and kubernetes image repo define
 gcr_image_repo: "gcr.io"
@@ -108,6 +109,8 @@ crictl_download_url: "https://github.com/kubernetes-sigs/cri-tools/releases/down
 helm_download_url: "https://get.helm.sh/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz"
 crun_download_url: "https://github.com/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}"
 kata_containers_download_url: "https://github.com/kata-containers/kata-containers/releases/download/{{ kata_containers_version }}/kata-static-{{ kata_containers_version }}-{{ ansible_architecture }}.tar.xz"
+# gVisor only supports amd64 and uses x86_64 to in the download link
+gvisor_download_url: "https://storage.googleapis.com/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}"
 nerdctl_download_url: "https://github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
 krew_download_url: "https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew.tar.gz"
 
@@ -401,6 +404,22 @@ kata_containers_binary_checksums:
     2.0.4: 0
     2.1.0: 0
 
+gvisor_runsc_binary_checksums:
+  arm:
+    20210518.0: 0
+  arm64:
+    20210518.0: 0
+  amd64:
+    20210518.0: 907811e10576d31b6f63d2f91f1c3be2b2df94e222fb48eb509fce48cd6ae9e0
+
+gvisor_containerd_shim_binary_checksums:
+  arm:
+    20210518.0: 0
+  arm64:
+    20210518.0: 0
+  amd64:
+    20210518.0: d6deda79562cadf4fc98ccf1d6d9fd1d2dff3890b184c7b546167bd6eff241ec
+
 nerdctl_archive_checksums:
   arm:
     0.8.1: 27bdad3f9e2667620f70617c48d595c5c4e24a10fbcd00d31202cd8d571c9233
@@ -420,6 +439,8 @@ crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}"
 helm_archive_checksum: "{{ helm_archive_checksums[image_arch][helm_version] }}"
 crun_binary_checksum: "{{ crun_checksums[image_arch][crun_version] }}"
 kata_containers_binary_checksum: "{{ kata_containers_binary_checksums[image_arch][kata_containers_version] }}"
+gvisor_runsc_binary_checksum: "{{ gvisor_runsc_binary_checksums[image_arch][gvisor_version] }}"
+gvisor_containerd_shim_binary_checksum: "{{ gvisor_containerd_shim_binary_checksums[image_arch][gvisor_version] }}"
 nerdctl_archive_checksum: "{{ nerdctl_archive_checksums[image_arch][nerdctl_version] }}"
 krew_archive_checksum: "{{ krew_archive_checksums[krew_version] }}"
 
@@ -715,6 +736,32 @@ downloads:
     groups:
     - k8s_cluster
 
+  gvisor_runsc:
+    enabled: "{{ gvisor_enabled }}"
+    file: true
+    version: "{{ gvisor_version }}"
+    dest: "{{ local_release_dir }}/gvisor-runsc"
+    sha256: "{{ gvisor_runsc_binary_checksum }}"
+    url: "{{ gvisor_download_url }}/runsc"
+    unarchive: false
+    owner: "root"
+    mode: 755
+    groups:
+    - k8s_cluster
+
+  gvisor_containerd_shim:
+    enabled: "{{ gvisor_enabled }}"
+    file: true
+    version: "{{ gvisor_version }}"
+    dest: "{{ local_release_dir }}/gvisor-containerd-shim-runsc-v1"
+    sha256: "{{ gvisor_containerd_shim_binary_checksum }}"
+    url: "{{ gvisor_download_url }}/containerd-shim-runsc-v1"
+    unarchive: false
+    owner: "root"
+    mode: 755
+    groups:
+    - k8s_cluster
+
   nerdctl:
     file: true
     enabled: "{{ nerdctl_enabled }}"

roles/kubernetes-apps/container_runtimes/gvisor/tasks/main.yaml

@@ -0,0 +1,33 @@
+---
+- name: gVisor | Create addon dir
+  file:
+    path: "{{ kube_config_dir}}/addons/gvisor"
+    owner: root
+    group: root
+    mode: 0755
+    recurse: true
+
+- name: gVisor | Templates List
+  set_fact:
+    gvisor_templates:
+      - { name: runtimeclass-gvisor, file: runtimeclass-gvisor.yml, type: runtimeclass }
+
+- name: gVisort | Create manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir}}/addons/gvisor/{{ item.file }}"
+  with_items: "{{ gvisor_templates }}"
+  register: gvisor_manifests
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: gVisor | Apply manifests
+  kube:
+    name: "{{ item.item.name }}"
+    kubectl: "{{ bin_dir}}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/addons/gvisor/{{ item.item.file }}"
+    state: "latest"
+  with_items: "{{ gvisor_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]

roles/kubernetes-apps/container_runtimes/gvisor/templates/runtimeclass-gvisor.yml.j2

@@ -0,0 +1,6 @@
+---
+kind: RuntimeClass
+apiVersion: node.k8s.io/v1{{ 'beta1' if kube_version is version('v1.20.0', '<') else '' }}
+metadata:
+  name: gvisor
+handler: runsc

roles/kubernetes-apps/container_runtimes/meta/main.yml

@@ -7,6 +7,13 @@ dependencies:
       - kata-containers
       - container-runtimes
 
+  - role: kubernetes-apps/container_runtimes/gvisor
+    when: gvisor_enabled
+    tags:
+      - apps
+      - gvisor
+      - container-runtimes
+
   - role: kubernetes-apps/container_runtimes/crun
     when: crun_enabled
     tags:

roles/kubernetes/preinstall/tasks/0020-verify-settings.yml

@@ -312,6 +312,12 @@
     msg: "kata_containers_enabled support only for containerd and crio-o. See https://github.com/kata-containers/documentation/blob/1.11.4/how-to/run-kata-with-k8s.md#install-a-cri-implementation for details"
   when: kata_containers_enabled
 
+- name: Stop if gvisor_enabled is enabled when container_manager is not containerd
+  assert:
+    that: container_manager == 'containerd'
+    msg: "gvisor_enabled support only compatible with containerd. See https://github.com/kubernetes-sigs/kubespray/issues/7650 for details"
+  when: gvisor_enabled
+
 - name: Stop if download_localhost is enabled for Flatcar Container Linux
   assert:
     that: ansible_os_family not in ["Flatcar Container Linux by Kinvolk"]

roles/kubespray-defaults/defaults/main.yaml

@@ -237,6 +237,10 @@ container_manager: docker
 # When enabled, it requires `container_manager` different than Docker
 kata_containers_enabled: false
 
+# Enable gVisor as an additional container runtime
+# gVisor is only supported with container_manager Docker or containerd
+gvisor_enabled: false
+
 # Enable crun as additional container runtime
 # When enabled, it requires container_manager=crio
 crun_enabled: false