From b18ed5922b9b30a77fa83df06c16392e993db0df Mon Sep 17 00:00:00 2001
From: Takashi Okamoto <toraneko@gmail.com>
Date: Wed, 8 Aug 2018 15:25:23 +0000
Subject: [PATCH 01/15] Add etcd default value in kubespray-default.

---
 roles/kubespray-defaults/defaults/main.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml
index 626c797bc..1a75178a2 100644
--- a/roles/kubespray-defaults/defaults/main.yaml
+++ b/roles/kubespray-defaults/defaults/main.yaml
@@ -366,3 +366,6 @@ etcd_events_peer_addresses: |-
   {%- endfor %}
 
 podsecuritypolicy_enabled: false
+etcd_heartbeat_interval: "250"
+etcd_election_timeout: "5000"
+etcd_snapshot_count: "10000"

From ac639b2a172a7914e917b103ee9ca21a8cecf7f5 Mon Sep 17 00:00:00 2001
From: Takashi Okamoto <toraneko@gmail.com>
Date: Fri, 10 Aug 2018 04:25:10 -0400
Subject: [PATCH 02/15] Change kubeadm config to run etcd by kubeadm.

---
 .../templates/kubeadm-config.v1alpha1.yaml.j2       | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2
index fd569b887..14c4c445c 100644
--- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2
+++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2
@@ -1,4 +1,4 @@
-apiVersion: kubeadm.k8s.io/v1alpha1
+apiVersion: kubeadm.k8s.io/v1alpha2
 kind: MasterConfiguration
 api:
   advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }}
@@ -7,13 +7,14 @@ api:
   controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}
 {% endif %}
 etcd:
-  endpoints:
+  external:
+      endpoints:
 {% for endpoint in etcd_access_addresses.split(',') %}
-  - {{ endpoint }}
+      - {{ endpoint }}
 {% endfor %}
-  caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem
-  certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem
-  keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem
+      caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem
+      certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem
+      keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem
 networking:
   dnsDomain: {{ dns_domain }}
   serviceSubnet: {{ kube_service_addresses }}

From 6849788ebc3debb5a8a64e9df5542e9403b4149b Mon Sep 17 00:00:00 2001
From: Takashi Okamoto <toraneko@gmail.com>
Date: Wed, 8 Aug 2018 00:49:18 -0400
Subject: [PATCH 03/15] Fix copy ca cert and ca key for kubeadm.

---
 roles/kubernetes/master/tasks/kubeadm-setup.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml
index 69ad06e4f..e02c885e9 100644
--- a/roles/kubernetes/master/tasks/kubeadm-setup.yml
+++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml
@@ -97,6 +97,14 @@
     kubeadm_config_api_fqdn: "{{ apiserver_loadbalancer_domain_name|default('lb-apiserver.kubernetes.local') }}"
   when: loadbalancer_apiserver is defined
 
+- name: kubeadm | Copy etcd ca file as k8s ca
+  command: "cp -T {{ etcd_cert_dir }}/ca.pem {{ kube_config_dir }}/ssl/etcd/ca.crt"
+  changed_when: false
+
+- name: kubeadm | Copy etcd cakey as k8s cakey
+  command: "cp -T {{ etcd_cert_dir }}/ca-key.pem {{ kube_config_dir }}/ssl/etcd/ca.key"
+  changed_when: false
+
 - name: kubeadm | Create kubeadm config
   template:
     src: "kubeadm-config.{{ kubeadmConfig_api_version }}.yaml.j2"

From bdbfa4d40310549c63c5b287d21e40cc20c422f4 Mon Sep 17 00:00:00 2001
From: Takashi Okamoto <toraneko@gmail.com>
Date: Sat, 4 Aug 2018 04:08:01 +0000
Subject: [PATCH 04/15] Add ipvs support for kubeadm 1.10 or later.

---
 .../master/templates/kubeadm-config.v1alpha1.yaml.j2        | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2
index 14c4c445c..f9fb621b5 100644
--- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2
+++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2
@@ -28,6 +28,12 @@ kubeProxy:
 {% if kube_proxy_mode == 'ipvs' and kube_version | version_compare('v1.10', '<') %}
     featureGates: SupportIPVSProxyMode=true
     mode: ipvs
+{% elif kube_proxy_mode == 'ipvs' %}
+kubeProxy:
+  config:
+    featureGates:
+      SupportIPVSProxyMode: true
+    mode: ipvs
 {% endif %}
 {% if kube_proxy_nodeport_addresses %}
     nodePortAddresses: [{{ kube_proxy_nodeport_addresses_cidr }}]

From 359009bb05fe8fd04ea302331fe227216e451ece Mon Sep 17 00:00:00 2001
From: Takashi Okamoto <toraneko@gmail.com>
Date: Mon, 30 Jul 2018 18:55:25 +0900
Subject: [PATCH 05/15] Download etcd and hyperkube binary.

---
 roles/download/defaults/main.yml             | 30 ++++++++++++++++++++
 roles/download/tasks/main.yml                |  1 +
 roles/etcd/tasks/install_host.yml            | 28 +++++++++++-------
 roles/kubernetes/master/tasks/main.yml       | 24 ++++++----------
 roles/kubernetes/node/tasks/install_host.yml | 24 +++++++---------
 5 files changed, 67 insertions(+), 40 deletions(-)

diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
index 97c8d8562..546cdbc63 100644
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@@ -48,8 +48,12 @@ cilium_version: "v1.2.0"
 # Download URLs
 kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/{{ image_arch }}/kubeadm"
 vault_download_url: "https://releases.hashicorp.com/vault/{{ vault_version }}/vault_{{ vault_version }}_linux_{{ image_arch }}.zip"
+etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz"
+hyperkube_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/amd64/hyperkube"
 
 # Checksums
+etcd_checksum: b729db0732448064271ea6fdcb901773c4fe917763ca07776f22d0e5e0bd4097
+hyperkube_checksum: d727f8cae3fc26b1add9b4ff0d4d9b99605544ff7fb3baeecdca394362adbfb8
 kubeadm_checksum: 6b17720a65b8ff46efe92a5544f149c39a221910d89939838d75581d4e6924c0
 vault_binary_checksum: 3c4d70ba71619a43229e65c67830e30e050eab7a81ac6b28325ff707e5914188
 
@@ -173,6 +177,19 @@ downloads:
     sha256: "{{ etcd_digest_checksum|default(None) }}"
     groups:
       - etcd
+  etcd_file:
+    enabled: true
+    file: true
+    version: "{{ etcd_version }}"
+    dest: "etcd-{{ etcd_version }}-linux-amd64.tar.gz"
+    sha256: "{{ etcd_checksum }}"
+    source_url: "{{ etcd_download_url }}"
+    url: "{{ etcd_download_url }}"
+    unarchive: true
+    owner: "root"
+    mode: "0755"
+    groups:
+      - etcd
   kubeadm:
     enabled: "{{ kubeadm_enabled }}"
     file: true
@@ -194,6 +211,19 @@ downloads:
     sha256: "{{ hyperkube_digest_checksum|default(None) }}"
     groups:
       - k8s-cluster
+  hyperkube_file:
+    enabled: true
+    file: true
+    version: "{{ kube_version }}"
+    dest: "hyperkube"
+    sha256: "{{ hyperkube_checksum }}"
+    source_url: "{{ hyperkube_download_url }}"
+    url: "{{ hyperkube_download_url }}"
+    unarchive: false
+    owner: "root"
+    mode: "0755"
+    groups:
+      - k8s-cluster
   cilium:
     enabled: "{{ kube_network_plugin == 'cilium' }}"
     container: true
diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml
index 6a317fd89..190fb737d 100644
--- a/roles/download/tasks/main.yml
+++ b/roles/download/tasks/main.yml
@@ -11,6 +11,7 @@
   when:
     - not skip_downloads|default(false)
     - item.value.enabled
+    - not item.value.container | item.value.container and download_container
 
 - name: "Sync container"
   include_tasks: sync_container.yml
diff --git a/roles/etcd/tasks/install_host.yml b/roles/etcd/tasks/install_host.yml
index 1d06a7d5a..0dc226e66 100644
--- a/roles/etcd/tasks/install_host.yml
+++ b/roles/etcd/tasks/install_host.yml
@@ -1,13 +1,21 @@
 ---
-- name: Install | Copy etcdctl and etcd binary from docker container
-  command: sh -c "{{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy;
-           {{ docker_bin_dir }}/docker create --name etcdctl-binarycopy {{ etcd_image_repo }}:{{ etcd_image_tag }} &&
-           {{ docker_bin_dir }}/docker cp etcdctl-binarycopy:/usr/local/bin/etcdctl {{ bin_dir }}/etcdctl &&
-           {{ docker_bin_dir }}/docker cp etcdctl-binarycopy:/usr/local/bin/etcd {{ bin_dir }}/etcd &&
-           {{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy"
-  register: etcd_task_result
-  until: etcd_task_result.rc == 0
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
+- name: install | Copy etcd binary from download dir
+  shell: |
+    rsync -piu "{{ local_release_dir }}/etcd-{{ etcd_version }}-linux-amd64/etcd" "{{ bin_dir }}/etcd"
+    rsync -piu "{{ local_release_dir }}/etcd-{{ etcd_version }}-linux-amd64/etcdctl" "{{ bin_dir }}/etcdctl"
   changed_when: false
   when: etcd_cluster_setup
+
+- name: install | Set etcd binary permissions
+  file:
+    path: "{{ bin_dir }}/etcd"
+    mode: "0755"
+    state: file
+  when: etcd_cluster_setup
+
+- name: install | Set etcdctl binary permissions
+  file:
+    path: "{{ bin_dir }}/etcdctl"
+    mode: "0755"
+    state: file
+  when: etcd_cluster_setup
\ No newline at end of file
diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml
index be2044e31..93da9760b 100644
--- a/roles/kubernetes/master/tasks/main.yml
+++ b/roles/kubernetes/master/tasks/main.yml
@@ -9,27 +9,19 @@
 - import_tasks: encrypt-at-rest.yml
   when: kube_encrypt_secret_data
 
-- name: Compare host kubectl with hyperkube container
-  command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/cmp /hyperkube /systembindir/kubectl"
-  register: kubectl_task_compare_result
-  until: kubectl_task_compare_result.rc in [0,1,2]
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
+- name: install | Copy kubectl binary from download dir
+  command: rsync -piu "{{ local_release_dir }}/hyperkube" "{{ bin_dir }}/kubectl"
   changed_when: false
-  failed_when: "kubectl_task_compare_result.rc not in [0,1,2]"
   tags:
     - hyperkube
     - kubectl
     - upgrade
 
-- name: Copy kubectl from hyperkube container
-  command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -f /hyperkube /systembindir/kubectl"
-  when: kubectl_task_compare_result.rc != 0
-  register: kubectl_task_result
-  until: kubectl_task_result.rc == 0
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
-  changed_when: false
+- name: install | Set kubectl binary permissions
+  file:
+    path: "{{ bin_dir }}/kubectl"
+    mode: "0755"
+    state: file
   tags:
     - hyperkube
     - kubectl
@@ -37,7 +29,7 @@
 
 - name: Install kubectl bash completion
   shell: "{{ bin_dir }}/kubectl completion bash >/etc/bash_completion.d/kubectl.sh"
-  when: kubectl_task_compare_result.rc != 0 and ansible_os_family in ["Debian","RedHat"]
+  when: ansible_os_family in ["Debian","RedHat"]
   tags:
     - kubectl
 
diff --git a/roles/kubernetes/node/tasks/install_host.yml b/roles/kubernetes/node/tasks/install_host.yml
index 7fcb4a01d..47a53b281 100644
--- a/roles/kubernetes/node/tasks/install_host.yml
+++ b/roles/kubernetes/node/tasks/install_host.yml
@@ -1,23 +1,19 @@
 ---
-- name: install | Compare host kubelet with hyperkube container
-  command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/cmp /hyperkube /systembindir/kubelet"
-  register: kubelet_task_compare_result
-  until: kubelet_task_compare_result.rc in [0,1,2]
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
+
+- name: install | Copy kubelet binary from download dir
+  command: rsync -piu "{{ local_release_dir }}/hyperkube" "{{ bin_dir }}/kubelet"
   changed_when: false
-  failed_when: "kubelet_task_compare_result.rc not in [0,1,2]"
+  when: hyperkube_enabled
   tags:
     - hyperkube
     - upgrade
 
-- name: install | Copy kubelet from hyperkube container
-  command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -f /hyperkube /systembindir/kubelet"
-  when: kubelet_task_compare_result.rc != 0
-  register: kubelet_task_result
-  until: kubelet_task_result.rc == 0
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
+- name: install | Set kubelet binary permissions
+  file:
+    path: "{{ bin_dir }}/kubelet"
+    mode: "0755"
+    state: file
+  when: hyperkube_enabled
   tags:
     - hyperkube
     - upgrade

From 6090af29e7ea23a9614e1da8376a5dc30d91f404 Mon Sep 17 00:00:00 2001
From: Takashi Okamoto <toraneko@gmail.com>
Date: Thu, 9 Aug 2018 13:53:11 -0400
Subject: [PATCH 06/15] Add cri-o role.

---
 cluster.yml                                   |   1 +
 roles/cri-o/defaults/main.yml                 |   2 +
 roles/cri-o/tasks/main.yaml                   |  40 +++
 roles/cri-o/templates/crio.conf.j2            | 234 ++++++++++++++++++
 roles/kubernetes/node/defaults/main.yml       |   5 +
 .../node/templates/kubelet.kubeadm.env.j2     |   1 +
 scale.yml                                     |   1 +
 upgrade-cluster.yml                           |   1 +
 8 files changed, 285 insertions(+)
 create mode 100644 roles/cri-o/defaults/main.yml
 create mode 100644 roles/cri-o/tasks/main.yaml
 create mode 100644 roles/cri-o/templates/crio.conf.j2

diff --git a/cluster.yml b/cluster.yml
index 8462ea894..14a4a6d37 100644
--- a/cluster.yml
+++ b/cluster.yml
@@ -34,6 +34,7 @@
     - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: docker, tags: docker, when: manage_docker|default(true) }
+    - { role: cri-o, tags: crio, when: manage_crio }
     - role: rkt
       tags: rkt
       when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"
diff --git a/roles/cri-o/defaults/main.yml b/roles/cri-o/defaults/main.yml
new file mode 100644
index 000000000..3ae39da22
--- /dev/null
+++ b/roles/cri-o/defaults/main.yml
@@ -0,0 +1,2 @@
+---
+crio_rhel_repo_base_url: 'https://cbs.centos.org/repos/paas7-openshift-origin311-candidate/x86_64/os/'
diff --git a/roles/cri-o/tasks/main.yaml b/roles/cri-o/tasks/main.yaml
new file mode 100644
index 000000000..7edfd035e
--- /dev/null
+++ b/roles/cri-o/tasks/main.yaml
@@ -0,0 +1,40 @@
+---
+- name: Add OpenShift Origin repository
+  yum_repository:
+    name: origin
+    description: OpenShift Origin Repo
+    baseurl: "{{ crio_rhel_repo_base_url }}"
+    gpgcheck: no
+  when:
+    - ansible_os_family == 'RedHat'
+
+- name: Install cri-o
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - cri-o
+    - cri-tools
+    - oci-systemd-hook
+
+- name: Install cri-o config
+  template:
+    src: crio.conf.j2
+    dest: /etc/crio/crio.conf
+
+- name: Copy mounts.conf
+  shell: |
+    cp -T /usr/share/containers/mounts.conf /etc/containers/mounts.conf
+
+- name: Create directory for oci hooks
+  file:
+    path: /etc/containers/oci/hooks.d
+    state: directory
+    owner: root
+    mode: 0755
+
+- name: Install cri-o service
+  service:
+    name: crio
+    enabled: yes
+    state: restarted
diff --git a/roles/cri-o/templates/crio.conf.j2 b/roles/cri-o/templates/crio.conf.j2
new file mode 100644
index 000000000..b20a50c70
--- /dev/null
+++ b/roles/cri-o/templates/crio.conf.j2
@@ -0,0 +1,234 @@
+
+# The "crio" table contains all of the server options.
+[crio]
+
+# CRI-O reads its storage defaults from the containers/storage configuration
+# file, /etc/containers/storage.conf. Modify storage.conf if you want to
+# change default storage for all tools that use containers/storage.  If you
+# want to modify just crio, you can change the storage configuration in this
+# file.
+
+# root is a path to the "root directory". CRIO stores all of its data,
+# including container images, in this directory.
+#root = "/var/lib/containers/storage"
+
+# run is a path to the "run directory". CRIO stores all of its state
+# in this directory.
+#runroot = "/var/run/containers/storage"
+
+# storage_driver select which storage driver is used to manage storage
+# of images and containers.
+storage_driver = "overlay2"
+
+# storage_option is used to pass an option to the storage driver.
+#storage_option = [
+#]
+
+# The "crio.api" table contains settings for the kubelet/gRPC interface.
+[crio.api]
+
+# listen is the path to the AF_LOCAL socket on which crio will listen.
+listen = "/var/run/crio/crio.sock"
+
+# stream_address is the IP address on which the stream server will listen
+stream_address = ""
+
+# stream_port is the port on which the stream server will listen
+stream_port = "10010"
+
+# stream_enable_tls enables encrypted tls transport of the stream server
+stream_enable_tls = false
+
+# stream_tls_cert is the x509 certificate file path used to serve the encrypted stream.
+# This file can change, and CRIO will automatically pick up the changes within 5 minutes.
+stream_tls_cert = ""
+
+# stream_tls_key is the key file path used to serve the encrypted stream.
+# This file can change, and CRIO will automatically pick up the changes within 5 minutes.
+stream_tls_key = ""
+
+# stream_tls_ca is the x509 CA(s) file used to verify and authenticate client
+# communication with the tls encrypted stream.
+# This file can change, and CRIO will automatically pick up the changes within 5 minutes.
+stream_tls_ca = ""
+
+# file_locking is whether file-based locking will be used instead of
+# in-memory locking
+file_locking = true
+
+# The "crio.runtime" table contains settings pertaining to the OCI
+# runtime used and options for how to set up and manage the OCI runtime.
+[crio.runtime]
+
+# runtime is the OCI compatible runtime used for trusted container workloads.
+# This is a mandatory setting as this runtime will be the default one
+# and will also be used for untrusted container workloads if
+# runtime_untrusted_workload is not set.
+runtime = "/usr/bin/runc"
+
+# runtime_untrusted_workload is the OCI compatible runtime used for untrusted
+# container workloads. This is an optional setting, except if
+# default_container_trust is set to "untrusted".
+runtime_untrusted_workload = ""
+
+# default_workload_trust is the default level of trust crio puts in container
+# workloads. It can either be "trusted" or "untrusted", and the default
+# is "trusted".
+# Containers can be run through different container runtimes, depending on
+# the trust hints we receive from kubelet:
+# - If kubelet tags a container workload as untrusted, crio will try first to
+# run it through the untrusted container workload runtime. If it is not set,
+# crio will use the trusted runtime.
+# - If kubelet does not provide any information about the container workload trust
+# level, the selected runtime will depend on the default_container_trust setting.
+# If it is set to "untrusted", then all containers except for the host privileged
+# ones, will be run by the runtime_untrusted_workload runtime. Host privileged
+# containers are by definition trusted and will always use the trusted container
+# runtime. If default_container_trust is set to "trusted", crio will use the trusted
+# container runtime for all containers.
+default_workload_trust = "trusted"
+
+# no_pivot instructs the runtime to not use pivot_root, but instead use MS_MOVE
+no_pivot = false
+
+# conmon is the path to conmon binary, used for managing the runtime.
+conmon = "/usr/libexec/crio/conmon"
+
+# conmon_env is the environment variable list for conmon process,
+# used for passing necessary environment variable to conmon or runtime.
+conmon_env = [
+	"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+]
+
+# selinux indicates whether or not SELinux will be used for pod
+# separation on the host. If you enable this flag, SELinux must be running
+# on the host.
+selinux = {{ (preinstall_selinux_state == 'enforcing')|lower }}
+
+# seccomp_profile is the seccomp json profile path which is used as the
+# default for the runtime.
+seccomp_profile = "/etc/crio/seccomp.json"
+
+# apparmor_profile is the apparmor profile name which is used as the
+# default for the runtime.
+apparmor_profile = "crio-default"
+
+# cgroup_manager is the cgroup management implementation to be used
+# for the runtime.
+cgroup_manager = "cgroupfs"
+
+# default_capabilities is the list of capabilities to add and can be modified here.
+# If capabilities below is commented out, the default list of capabilities defined in the
+# spec will be added.
+# If capabilities is empty below, only the capabilities defined in the container json
+# file by the user/kube will be added.
+default_capabilities = [
+	"CHOWN", 
+	"DAC_OVERRIDE", 
+	"FSETID", 
+	"FOWNER", 
+	"NET_RAW", 
+	"SETGID", 
+	"SETUID", 
+	"SETPCAP", 
+	"NET_BIND_SERVICE", 
+	"SYS_CHROOT", 
+	"KILL", 
+]
+
+# hooks_dir_path is the oci hooks directory for automatically executed hooks
+hooks_dir_path = "/usr/share/containers/oci/hooks.d"
+
+# default_mounts is the mounts list to be mounted for the container when created
+# deprecated, will be taken out in future versions, add default mounts to either
+# /usr/share/containers/mounts.conf or /etc/containers/mounts.conf
+default_mounts = [
+]
+
+# CRI-O reads its default mounts from the following two files:
+# 1) /etc/containers/mounts.conf - this is the override file, where users can
+# either add in their own default mounts, or override the default mounts shipped
+# with the package.
+# 2) /usr/share/containers/mounts.conf - this is the default file read for mounts.
+# If you want CRI-O to read from a different, specific mounts file, you can change
+# the default_mounts_file path right below. Note, if this is done, CRI-O will only add
+# mounts it finds in this file.
+
+# default_mounts_file is the file path holding the default mounts to be mounted for the
+# container when created.
+# default_mounts_file = ""
+
+# pids_limit is the number of processes allowed in a container
+pids_limit = 1024
+
+# log_size_max is the max limit for the container log size in bytes.
+# Negative values indicate that no limit is imposed.
+log_size_max = -1
+
+# read-only indicates whether all containers will run in read-only mode
+read_only = false
+
+# The "crio.image" table contains settings pertaining to the
+# management of OCI images.
+
+# uid_mappings specifies the UID mappings to have in the user namespace.
+# A range is specified in the form containerUID:HostUID:Size.  Multiple
+# ranges are separed by comma.
+uid_mappings = ""
+
+# gid_mappings specifies the GID mappings to have in the user namespace.
+# A range is specified in the form containerGID:HostGID:Size.  Multiple
+# ranges are separed by comma.
+gid_mappings = ""
+
+[crio.image]
+
+# default_transport is the prefix we try prepending to an image name if the
+# image name as we receive it can't be parsed as a valid source reference
+default_transport = "docker://"
+
+# pause_image is the image which we use to instantiate infra containers.
+pause_image = "docker://k8s.gcr.io/pause:3.1"
+
+# pause_command is the command to run in a pause_image to have a container just
+# sit there.  If the image contains the necessary information, this value need
+# not be specified.
+pause_command = "/pause"
+
+# signature_policy is the name of the file which decides what sort of policy we
+# use when deciding whether or not to trust an image that we've pulled.
+# Outside of testing situations, it is strongly advised that this be left
+# unspecified so that the default system-wide policy will be used.
+signature_policy = ""
+
+# image_volumes controls how image volumes are handled.
+# The valid values are mkdir and ignore.
+image_volumes = "mkdir"
+
+# CRI-O reads its configured registries defaults from the containers/image configuration
+# file, /etc/containers/registries.conf. Modify registries.conf if you want to
+# change default registries for all tools that use containers/image.  If you
+# want to modify just crio, you can change the registies configuration in this
+# file.
+
+# insecure_registries is used to skip TLS verification when pulling images.
+insecure_registries = [
+  "{{ kube_service_addresses }}"
+]
+
+# registries is used to specify a comma separated list of registries to be used
+# when pulling an unqualified image (e.g. fedora:rawhide).
+registries = [
+  "docker.io"
+]
+
+# The "crio.network" table contains settings pertaining to the
+# management of CNI plugins.
+[crio.network]
+
+# network_dir is is where CNI network configuration
+# files are stored.
+network_dir = "/etc/cni/net.d/"
+
+# plugin_dir is is where CNI plugin binaries are stored.
+plugin_dir = "/opt/cni/bin/"
diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml
index 2c541c112..0e73d7932 100644
--- a/roles/kubernetes/node/defaults/main.yml
+++ b/roles/kubernetes/node/defaults/main.yml
@@ -31,6 +31,11 @@ kubelet_cgroups_per_qos: true
 # Set to empty to avoid cgroup creation
 kubelet_enforce_node_allocatable: "\"\""
 
+# Set runtime cgroups
+kubelet_runtime_cgroups: "/systemd/system.slice"
+# Set kubelet cgroups
+kubelet_kubelet_cgroups: "/systemd/system.slice"
+
 # Set false to enable sharing a pid namespace between containers in a pod.
 # Note that PID namespace sharing requires docker >= 1.13.1.
 kubelet_disable_shared_pid: true
diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2
index aca97ae12..46b5e40d7 100644
--- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2
+++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2
@@ -42,6 +42,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% else %}
 --fail-swap-on={{ kubelet_fail_swap_on|default(true)}} \
 {% endif %}
+--runtime-cgroups={{ kubelet_runtime_cgroups }} --kubelet-cgroups={{ kubelet_kubelet_cgroups }} \
 {% endset %}
 
 {# Node reserved CPU/memory #}
diff --git a/scale.yml b/scale.yml
index 676fba610..ff027d6c0 100644
--- a/scale.yml
+++ b/scale.yml
@@ -36,6 +36,7 @@
     - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: docker, tags: docker, when: manage_docker|default(true) }
+    - { role: cri-o, tags: crio, when: manage_crio }
     - role: rkt
       tags: rkt
       when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"
diff --git a/upgrade-cluster.yml b/upgrade-cluster.yml
index abc89e18f..4e4cfb654 100644
--- a/upgrade-cluster.yml
+++ b/upgrade-cluster.yml
@@ -35,6 +35,7 @@
     - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: docker, tags: docker, when: manage_docker|default(true) }
+    - { role: cri-o, tags: crio, when: manage_crio }
     - role: rkt
       tags: rkt
       when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"

From cf7b9cfeefe5d963591572e88d3e666e9eaeddf8 Mon Sep 17 00:00:00 2001
From: Takashi Okamoto <toraneko@gmail.com>
Date: Mon, 30 Jul 2018 19:37:36 +0900
Subject: [PATCH 07/15] Support crio in kubelet service.

---
 roles/kubernetes/node/templates/kubelet.kubeadm.env.j2  | 6 ++++++
 roles/kubernetes/node/templates/kubelet.standard.env.j2 | 6 ++++++
 2 files changed, 12 insertions(+)

diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2
index 46b5e40d7..b2a27d58c 100644
--- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2
+++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2
@@ -34,7 +34,13 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 --node-status-update-frequency={{ kubelet_status_update_frequency }} \
 --cgroup-driver={{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }} \
 --max-pods={{ kubelet_max_pods }} \
+{% if manage_docker %}
 --docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \
+{% endif %}
+{% if manage_crio %}
+--container-runtime=remote \
+--container-runtime-endpoint=/var/run/crio/crio.sock \
+{% endif %}
 --anonymous-auth=false \
 --read-only-port={{ kube_read_only_port }} \
 {% if kube_version | version_compare('v1.8', '<') %}
diff --git a/roles/kubernetes/node/templates/kubelet.standard.env.j2 b/roles/kubernetes/node/templates/kubelet.standard.env.j2
index c99194ba9..0c5dd0122 100644
--- a/roles/kubernetes/node/templates/kubelet.standard.env.j2
+++ b/roles/kubernetes/node/templates/kubelet.standard.env.j2
@@ -15,7 +15,9 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 --cadvisor-port={{ kube_cadvisor_port }} \
 --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \
 --node-status-update-frequency={{ kubelet_status_update_frequency }} \
+{% if manage_docker %}
 --docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \
+{% endif %}
 --client-ca-file={{ kube_cert_dir }}/ca.pem \
 --tls-cert-file={{ kube_cert_dir }}/node-{{ inventory_hostname }}.pem \
 --tls-private-key-file={{ kube_cert_dir }}/node-{{ inventory_hostname }}-key.pem \
@@ -26,6 +28,10 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% if kube_version | version_compare('v1.7', '<') %}
 --enable-cri={{ kubelet_enable_cri }} \
 {% endif %}
+{% if manage_crio %}
+--container-runtime=remote \
+--container-runtime-endpoint=/var/run/crio/crio.sock \
+{% endif %}
 --cgroup-driver={{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }} \
 --cgroups-per-qos={{ kubelet_cgroups_per_qos }} \
 --max-pods={{ kubelet_max_pods }} \

From 5ab8a712d99f458ceb7c40460e949117fe4e6e37 Mon Sep 17 00:00:00 2001
From: Takashi Okamoto <toraneko@gmail.com>
Date: Sat, 4 Aug 2018 10:01:35 +0000
Subject: [PATCH 08/15] Add download_container flag to avoid docker pull when
 use cri-o.

---
 roles/download/defaults/main.yml             | 3 +++
 roles/download/tasks/download_prep.yml       | 3 +++
 roles/download/tasks/main.yml                | 2 +-
 roles/kubernetes/node/tasks/install_host.yml | 2 --
 roles/kubespray-defaults/defaults/main.yaml  | 4 ++++
 5 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
index 546cdbc63..d673c72b5 100644
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@@ -10,6 +10,9 @@ skip_downloads: false
 download_run_once: False
 download_compress: 1
 
+# if this is set to true will download container
+download_container: True
+
 # if this is set to true, uses the localhost for download_run_once mode
 # (requires docker and sudo to access docker). You may want this option for
 # local caching of docker images or for Container Linux by CoreOS cluster nodes.
diff --git a/roles/download/tasks/download_prep.yml b/roles/download/tasks/download_prep.yml
index 1fd7abf2f..b44da45da 100644
--- a/roles/download/tasks/download_prep.yml
+++ b/roles/download/tasks/download_prep.yml
@@ -7,6 +7,7 @@
   failed_when: false
   changed_when: false
   check_mode: no
+  when: download_container
 
 - name: container_download | Create dest directory for saved/loaded container images
   file:
@@ -15,6 +16,7 @@
     recurse: yes
     mode: 0755
     owner: "{{ansible_ssh_user|default(ansible_user_id)}}"
+  when: download_container
 
 - name: container_download | create local directory for saved/loaded container images
   file:
@@ -28,5 +30,6 @@
   when:
     - download_run_once
     - download_delegate == 'localhost'
+    - download_container
   tags:
     - localhost
diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml
index 190fb737d..1984f626d 100644
--- a/roles/download/tasks/main.yml
+++ b/roles/download/tasks/main.yml
@@ -11,7 +11,7 @@
   when:
     - not skip_downloads|default(false)
     - item.value.enabled
-    - not item.value.container | item.value.container and download_container
+    - (not (item.value.container|default(False))) or (item.value.container and download_container)
 
 - name: "Sync container"
   include_tasks: sync_container.yml
diff --git a/roles/kubernetes/node/tasks/install_host.yml b/roles/kubernetes/node/tasks/install_host.yml
index 47a53b281..3ca923848 100644
--- a/roles/kubernetes/node/tasks/install_host.yml
+++ b/roles/kubernetes/node/tasks/install_host.yml
@@ -3,7 +3,6 @@
 - name: install | Copy kubelet binary from download dir
   command: rsync -piu "{{ local_release_dir }}/hyperkube" "{{ bin_dir }}/kubelet"
   changed_when: false
-  when: hyperkube_enabled
   tags:
     - hyperkube
     - upgrade
@@ -13,7 +12,6 @@
     path: "{{ bin_dir }}/kubelet"
     mode: "0755"
     state: file
-  when: hyperkube_enabled
   tags:
     - hyperkube
     - upgrade
diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml
index 1a75178a2..c676598bc 100644
--- a/roles/kubespray-defaults/defaults/main.yaml
+++ b/roles/kubespray-defaults/defaults/main.yaml
@@ -135,6 +135,10 @@ kube_api_aggregator_routing: false
 # Optionally do not run docker role
 manage_docker: true
 
+# cri-o options
+# Optionally run cri-o role
+manage_crio: false
+
 # Path used to store Docker data
 docker_daemon_graph: "/var/lib/docker"
 

From 236f0666356fc2cb0a6f0e9ecb2cf29a42664bd2 Mon Sep 17 00:00:00 2001
From: Takashi Okamoto <toraneko@gmail.com>
Date: Tue, 28 Aug 2018 02:24:45 +0000
Subject: [PATCH 09/15] kubeadm cri-o support.

---
 .../kubeadm/templates/kubeadm-client.conf.v1alpha2.j2          | 3 +++
 .../master/templates/kubeadm-config.v1alpha2.yaml.j2           | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2
index 38ac215a2..1b0ab9c40 100644
--- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2
+++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2
@@ -15,3 +15,6 @@ discoveryTokenAPIServers:
 discoveryTokenUnsafeSkipCAVerification: true
 nodeRegistration:
   name: {{ inventory_hostname  }}
+{% if manage_crio %}
+  criSocket: /var/run/crio/crio.sock
+{% endif %}
diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2
index 31c499e0f..4ee15a181 100644
--- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2
+++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2
@@ -138,3 +138,6 @@ nodeRegistration:
   taints:
   - effect: NoSchedule
     key: node-role.kubernetes.io/master
+{% if manage_crio %}
+  criSocket: /var/run/crio/crio.sock
+{% endif %}

From f47c31dce5ee29a955a1e9969162015fd7492f1e Mon Sep 17 00:00:00 2001
From: Takashi Okamoto <toraneko@gmail.com>
Date: Mon, 30 Jul 2018 15:47:13 +0000
Subject: [PATCH 10/15] Add cri-o document.

---
 docs/cri-o.md | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 docs/cri-o.md

diff --git a/docs/cri-o.md b/docs/cri-o.md
new file mode 100644
index 000000000..ef37d7a7f
--- /dev/null
+++ b/docs/cri-o.md
@@ -0,0 +1,32 @@
+cri-o
+===============
+
+cri-o is container developed by kubernetes project.
+Currently, only basic function is supported for cri-o.
+
+* cri-o is supported kubernetes 1.11.1 or later.
+* helm and other feature may not be supported due to docker dependency.
+* scale.yml and upgrade-cluster.yml are not supported.
+
+helm and other feature may not be supported due to docker dependency.
+
+Use cri-o instead of docker, set following variable:
+
+#### all.yml
+
+```
+kubeadm_enable: true
+...
+download_container: false
+skip_downloads: false
+```
+
+#### k8s-cluster.yml
+
+```
+etcd_deployment_type: host
+kubelet_deployment_type: host
+manage_docker: false
+manage_crio: true
+```
+

From 659cccc5079a3ffebd73a1e56aba02d72674adf5 Mon Sep 17 00:00:00 2001
From: Takashi Okamoto <toraneko@gmail.com>
Date: Mon, 30 Jul 2018 22:56:21 +0000
Subject: [PATCH 11/15] Update sample.

---
 inventory/sample/group_vars/all.yml         | 4 ++++
 inventory/sample/group_vars/k8s-cluster.yml | 8 ++++++++
 2 files changed, 12 insertions(+)

diff --git a/inventory/sample/group_vars/all.yml b/inventory/sample/group_vars/all.yml
index e2829cb98..e347f4f17 100644
--- a/inventory/sample/group_vars/all.yml
+++ b/inventory/sample/group_vars/all.yml
@@ -155,3 +155,7 @@ bin_dir: /usr/local/bin
 
 # Does coreos need auto upgrade, default is true
 #coreos_auto_upgrade: true
+
+# Set true to download and cache container
+#download_container: true
+
diff --git a/inventory/sample/group_vars/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster.yml
index aa0210ebd..627b92e72 100644
--- a/inventory/sample/group_vars/k8s-cluster.yml
+++ b/inventory/sample/group_vars/k8s-cluster.yml
@@ -135,6 +135,14 @@ skydns_server_secondary: "{{ kube_service_addresses|ipaddr('net')|ipaddr(4)|ipad
 dnsmasq_dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}"
 dns_domain: "{{ cluster_name }}"
 
+# Use docker as container runtime
+# If you set manage_crio true, set manage_docker false.
+manage_docker: true
+
+# Use cri-o as container runtime
+# If you set manage_docker true, set manage_crio false.
+manage_crio: false
+
 # Path used to store Docker data
 docker_daemon_graph: "/var/lib/docker"
 

From dfdcb567849e85b17cdc51436decc1b84f50b2e5 Mon Sep 17 00:00:00 2001
From: Takashi Okamoto <toraneko@gmail.com>
Date: Wed, 8 Aug 2018 00:50:15 -0400
Subject: [PATCH 12/15] Delete all cri-o containers when execute reset.yml.

---
 roles/reset/tasks/main.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml
index 47b51546f..9c0d994f5 100644
--- a/roles/reset/tasks/main.yml
+++ b/roles/reset/tasks/main.yml
@@ -60,6 +60,16 @@
   tags:
     - docker
 
+- name: reset | remove all cri-o containers
+  shell: "crictl ps -aq | xargs -r crictl rm"
+  register: remove_all_crio_containers
+  retries: 4
+  until: remove_all_crio_containers.rc == 0
+  delay: 5
+  tags:
+    - crio
+  when: manage_crio
+
 - name: reset | gather mounted kubelet dirs
   shell: mount | grep /var/lib/kubelet/ | awk '{print $3}' | tac
   check_mode: no

From 5eb805f0989a4245d157bbad85e59855c77946b1 Mon Sep 17 00:00:00 2001
From: Takashi Okamoto <toraneko@gmail.com>
Date: Tue, 28 Aug 2018 04:51:38 +0000
Subject: [PATCH 13/15] Change timeout for kubeadm 600s.

* kubeadm timeout is too short and it may interrupt by timeout.
---
 roles/kubernetes/master/tasks/kubeadm-setup.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml
index e02c885e9..5b33e199e 100644
--- a/roles/kubernetes/master/tasks/kubeadm-setup.yml
+++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml
@@ -112,7 +112,7 @@
   register: kubeadm_config
 
 - name: kubeadm | Initialize first master
-  command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml --ignore-preflight-errors=all
+  command: timeout -k 600s 600s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml --ignore-preflight-errors=all
   register: kubeadm_init
   # Retry is because upload config sometimes fails
   retries: 3
@@ -122,7 +122,7 @@
 
 - name: kubeadm | Upgrade first master
   command: >-
-    timeout -k 240s 240s
+    timeout -k 600s 600s
     {{ bin_dir }}/kubeadm
     upgrade apply -y {{ kube_version }}
     --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml
@@ -175,7 +175,7 @@
   when: inventory_hostname != groups['kube-master']|first
 
 - name: kubeadm | Init other uninitialized masters
-  command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml --ignore-preflight-errors=all
+  command: timeout -k 600s 600s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml --ignore-preflight-errors=all
   register: kubeadm_init
   when: inventory_hostname != groups['kube-master']|first and not kubeadm_ca.stat.exists
   failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
@@ -183,7 +183,7 @@
 
 - name: kubeadm | Upgrade other masters
   command: >-
-    timeout -k 240s 240s
+    timeout -k 600s 600s
     {{ bin_dir }}/kubeadm
     upgrade apply -y {{ kube_version }}
     --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml

From d407a590a62bd65650a682ed87c9bc477d46efd3 Mon Sep 17 00:00:00 2001
From: Takashi Okamoto <toraneko@gmail.com>
Date: Tue, 28 Aug 2018 06:23:38 +0000
Subject: [PATCH 14/15] container_manager variable to specify runtime.

---
 cluster.yml                                            |  4 ++--
 docs/cri-o.md                                          |  3 +--
 inventory/sample/group_vars/k8s-cluster.yml            | 10 +++-------
 .../kubeadm/templates/kubeadm-client.conf.v1alpha2.j2  |  2 +-
 .../master/templates/kubeadm-config.v1alpha2.yaml.j2   |  2 +-
 roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 |  4 ++--
 .../kubernetes/node/templates/kubelet.standard.env.j2  |  4 ++--
 roles/kubespray-defaults/defaults/main.yaml            |  9 ++-------
 roles/reset/tasks/main.yml                             |  2 +-
 scale.yml                                              |  5 +++--
 upgrade-cluster.yml                                    |  4 ++--
 11 files changed, 20 insertions(+), 29 deletions(-)

diff --git a/cluster.yml b/cluster.yml
index 14a4a6d37..4fc852d97 100644
--- a/cluster.yml
+++ b/cluster.yml
@@ -33,8 +33,8 @@
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
-    - { role: docker, tags: docker, when: manage_docker|default(true) }
-    - { role: cri-o, tags: crio, when: manage_crio }
+    - { role: docker, tags: docker, when: container_manager == 'docker' }
+    - { role: cri-o, tags: crio, when: container_manager == 'crio' }
     - role: rkt
       tags: rkt
       when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"
diff --git a/docs/cri-o.md b/docs/cri-o.md
index ef37d7a7f..796b7513a 100644
--- a/docs/cri-o.md
+++ b/docs/cri-o.md
@@ -26,7 +26,6 @@ skip_downloads: false
 ```
 etcd_deployment_type: host
 kubelet_deployment_type: host
-manage_docker: false
-manage_crio: true
+container_manager: crio
 ```
 
diff --git a/inventory/sample/group_vars/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster.yml
index 627b92e72..eb1d01cb9 100644
--- a/inventory/sample/group_vars/k8s-cluster.yml
+++ b/inventory/sample/group_vars/k8s-cluster.yml
@@ -135,13 +135,9 @@ skydns_server_secondary: "{{ kube_service_addresses|ipaddr('net')|ipaddr(4)|ipad
 dnsmasq_dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}"
 dns_domain: "{{ cluster_name }}"
 
-# Use docker as container runtime
-# If you set manage_crio true, set manage_docker false.
-manage_docker: true
-
-# Use cri-o as container runtime
-# If you set manage_docker true, set manage_crio false.
-manage_crio: false
+# Container runtime
+# docker for docker and crio for cri-o.
+container_manager: docker
 
 # Path used to store Docker data
 docker_daemon_graph: "/var/lib/docker"
diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2
index 1b0ab9c40..35ed7a3e6 100644
--- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2
+++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2
@@ -15,6 +15,6 @@ discoveryTokenAPIServers:
 discoveryTokenUnsafeSkipCAVerification: true
 nodeRegistration:
   name: {{ inventory_hostname  }}
-{% if manage_crio %}
+{% if container_manager == 'crio' %}
   criSocket: /var/run/crio/crio.sock
 {% endif %}
diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2
index 4ee15a181..3f123b24d 100644
--- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2
+++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2
@@ -138,6 +138,6 @@ nodeRegistration:
   taints:
   - effect: NoSchedule
     key: node-role.kubernetes.io/master
-{% if manage_crio %}
+{% if container_manager == 'crio' %}
   criSocket: /var/run/crio/crio.sock
 {% endif %}
diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2
index b2a27d58c..7597fd9ae 100644
--- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2
+++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2
@@ -34,10 +34,10 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 --node-status-update-frequency={{ kubelet_status_update_frequency }} \
 --cgroup-driver={{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }} \
 --max-pods={{ kubelet_max_pods }} \
-{% if manage_docker %}
+{% if container_manager == 'docker' %}
 --docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \
 {% endif %}
-{% if manage_crio %}
+{% if container_manager == 'crio' %}
 --container-runtime=remote \
 --container-runtime-endpoint=/var/run/crio/crio.sock \
 {% endif %}
diff --git a/roles/kubernetes/node/templates/kubelet.standard.env.j2 b/roles/kubernetes/node/templates/kubelet.standard.env.j2
index 0c5dd0122..ae4654424 100644
--- a/roles/kubernetes/node/templates/kubelet.standard.env.j2
+++ b/roles/kubernetes/node/templates/kubelet.standard.env.j2
@@ -15,7 +15,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 --cadvisor-port={{ kube_cadvisor_port }} \
 --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \
 --node-status-update-frequency={{ kubelet_status_update_frequency }} \
-{% if manage_docker %}
+{% if container_manager == 'docker' %}
 --docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \
 {% endif %}
 --client-ca-file={{ kube_cert_dir }}/ca.pem \
@@ -28,7 +28,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% if kube_version | version_compare('v1.7', '<') %}
 --enable-cri={{ kubelet_enable_cri }} \
 {% endif %}
-{% if manage_crio %}
+{% if container_manager == 'crio' %}
 --container-runtime=remote \
 --container-runtime-endpoint=/var/run/crio/crio.sock \
 {% endif %}
diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml
index c676598bc..54986fe25 100644
--- a/roles/kubespray-defaults/defaults/main.yaml
+++ b/roles/kubespray-defaults/defaults/main.yaml
@@ -131,13 +131,8 @@ kube_apiserver_insecure_port: 8080
 # Aggregator
 kube_api_aggregator_routing: false
 
-# Docker options
-# Optionally do not run docker role
-manage_docker: true
-
-# cri-o options
-# Optionally run cri-o role
-manage_crio: false
+# Container for runtime
+container_manager: docker
 
 # Path used to store Docker data
 docker_daemon_graph: "/var/lib/docker"
diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml
index 9c0d994f5..88dec8d7a 100644
--- a/roles/reset/tasks/main.yml
+++ b/roles/reset/tasks/main.yml
@@ -68,7 +68,7 @@
   delay: 5
   tags:
     - crio
-  when: manage_crio
+  when: container_manager == 'crio'
 
 - name: reset | gather mounted kubelet dirs
   shell: mount | grep /var/lib/kubelet/ | awk '{print $3}' | tac
diff --git a/scale.yml b/scale.yml
index ff027d6c0..c4cd117f0 100644
--- a/scale.yml
+++ b/scale.yml
@@ -35,8 +35,9 @@
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
-    - { role: docker, tags: docker, when: manage_docker|default(true) }
-    - { role: cri-o, tags: crio, when: manage_crio }
+
+    - { role: docker, tags: docker, when: container_manager == 'docker' }
+    - { role: cri-o, tags: crio, when: container_manager == 'crio' }
     - role: rkt
       tags: rkt
       when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"
diff --git a/upgrade-cluster.yml b/upgrade-cluster.yml
index 4e4cfb654..7d8534d78 100644
--- a/upgrade-cluster.yml
+++ b/upgrade-cluster.yml
@@ -34,8 +34,8 @@
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
-    - { role: docker, tags: docker, when: manage_docker|default(true) }
-    - { role: cri-o, tags: crio, when: manage_crio }
+    - { role: docker, tags: docker, when: container_manager == 'docker' }
+    - { role: cri-o, tags: crio, when: container_manager == 'crio' }
     - role: rkt
       tags: rkt
       when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"

From c0dfa72707cb97ba5f02a07cc81857eea521bf34 Mon Sep 17 00:00:00 2001
From: Takashi Okamoto <toraneko@gmail.com>
Date: Tue, 28 Aug 2018 11:48:37 +0000
Subject: [PATCH 15/15] Separate RedHat specific vars for cri-o.

---
 roles/cri-o/files/mounts.conf |  1 +
 roles/cri-o/tasks/main.yaml   | 36 +++++++++++++++++++++++++----------
 roles/cri-o/vars/redhat.yml   |  7 +++++++
 3 files changed, 34 insertions(+), 10 deletions(-)
 create mode 100644 roles/cri-o/files/mounts.conf
 create mode 100644 roles/cri-o/vars/redhat.yml

diff --git a/roles/cri-o/files/mounts.conf b/roles/cri-o/files/mounts.conf
new file mode 100644
index 000000000..b7cde9d8a
--- /dev/null
+++ b/roles/cri-o/files/mounts.conf
@@ -0,0 +1 @@
+/usr/share/rhel/secrets:/run/secrets
diff --git a/roles/cri-o/tasks/main.yaml b/roles/cri-o/tasks/main.yaml
index 7edfd035e..3d9e67c86 100644
--- a/roles/cri-o/tasks/main.yaml
+++ b/roles/cri-o/tasks/main.yaml
@@ -1,21 +1,34 @@
 ---
+- name: gather os specific variables
+  include_vars: "{{ item }}"
+  with_first_found:
+    - files:
+        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_version|lower|replace('/', '_') }}.yml"
+        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_release }}.yml"
+        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml"
+        - "{{ ansible_distribution|lower }}.yml"
+        - "{{ ansible_os_family|lower }}-{{ ansible_architecture }}.yml"
+        - "{{ ansible_os_family|lower }}.yml"
+        - defaults.yml
+      paths:
+        - ../vars
+      skip: true
+  tags:
+    - facts
+
 - name: Add OpenShift Origin repository
   yum_repository:
     name: origin
     description: OpenShift Origin Repo
     baseurl: "{{ crio_rhel_repo_base_url }}"
     gpgcheck: no
-  when:
-    - ansible_os_family == 'RedHat'
+  when: ansible_distribution in ["CentOS","RedHat"] and not is_atomic
 
-- name: Install cri-o
+- name: Install cri-o packages
   package:
     name: "{{ item }}"
     state: present
-  with_items:
-    - cri-o
-    - cri-tools
-    - oci-systemd-hook
+  with_items: "{{ crio_packages }}"
 
 - name: Install cri-o config
   template:
@@ -23,8 +36,11 @@
     dest: /etc/crio/crio.conf
 
 - name: Copy mounts.conf
-  shell: |
-    cp -T /usr/share/containers/mounts.conf /etc/containers/mounts.conf
+  copy:
+    src: mounts.conf
+    dest: /etc/containers/mounts.conf
+  when:
+    - ansible_os_family == 'RedHat'
 
 - name: Create directory for oci hooks
   file:
@@ -35,6 +51,6 @@
 
 - name: Install cri-o service
   service:
-    name: crio
+    name: "{{ crio_service }}"
     enabled: yes
     state: restarted
diff --git a/roles/cri-o/vars/redhat.yml b/roles/cri-o/vars/redhat.yml
new file mode 100644
index 000000000..962dc9a0a
--- /dev/null
+++ b/roles/cri-o/vars/redhat.yml
@@ -0,0 +1,7 @@
+---
+crio_packages:
+  - cri-o
+  - cri-tools
+  - oci-systemd-hook
+
+crio_service: crio
\ No newline at end of file