Merge pull request #3061 from okamototk/crio

cri-o support

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2

@@ -15,3 +15,6 @@ discoveryTokenAPIServers:
 discoveryTokenUnsafeSkipCAVerification: true
 nodeRegistration:
   name: {{ inventory_hostname  }}
+{% if container_manager == 'crio' %}
+  criSocket: /var/run/crio/crio.sock
+{% endif %}

roles/kubernetes/master/tasks/kubeadm-setup.yml

@@ -97,6 +97,14 @@
     kubeadm_config_api_fqdn: "{{ apiserver_loadbalancer_domain_name|default('lb-apiserver.kubernetes.local') }}"
   when: loadbalancer_apiserver is defined
 
+- name: kubeadm | Copy etcd ca file as k8s ca
+  command: "cp -T {{ etcd_cert_dir }}/ca.pem {{ kube_config_dir }}/ssl/etcd/ca.crt"
+  changed_when: false
+
+- name: kubeadm | Copy etcd cakey as k8s cakey
+  command: "cp -T {{ etcd_cert_dir }}/ca-key.pem {{ kube_config_dir }}/ssl/etcd/ca.key"
+  changed_when: false
+
 - name: kubeadm | Create kubeadm config
   template:
     src: "kubeadm-config.{{ kubeadmConfig_api_version }}.yaml.j2"
@@ -104,7 +112,7 @@
   register: kubeadm_config
 
 - name: kubeadm | Initialize first master
-  command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml --ignore-preflight-errors=all
+  command: timeout -k 600s 600s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml --ignore-preflight-errors=all
   register: kubeadm_init
   # Retry is because upload config sometimes fails
   retries: 3
@@ -114,7 +122,7 @@
 
 - name: kubeadm | Upgrade first master
   command: >-
-    timeout -k 240s 240s
+    timeout -k 600s 600s
     {{ bin_dir }}/kubeadm
     upgrade apply -y {{ kube_version }}
     --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml
@@ -167,7 +175,7 @@
   when: inventory_hostname != groups['kube-master']|first
 
 - name: kubeadm | Init other uninitialized masters
-  command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml --ignore-preflight-errors=all
+  command: timeout -k 600s 600s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml --ignore-preflight-errors=all
   register: kubeadm_init
   when: inventory_hostname != groups['kube-master']|first and not kubeadm_ca.stat.exists
   failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
@@ -175,7 +183,7 @@
 
 - name: kubeadm | Upgrade other masters
   command: >-
-    timeout -k 240s 240s
+    timeout -k 600s 600s
     {{ bin_dir }}/kubeadm
     upgrade apply -y {{ kube_version }}
     --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml

roles/kubernetes/master/tasks/main.yml

@@ -9,27 +9,19 @@
 - import_tasks: encrypt-at-rest.yml
   when: kube_encrypt_secret_data
 
-- name: Compare host kubectl with hyperkube container
-  command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/cmp /hyperkube /systembindir/kubectl"
-  register: kubectl_task_compare_result
-  until: kubectl_task_compare_result.rc in [0,1,2]
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
+- name: install | Copy kubectl binary from download dir
+  command: rsync -piu "{{ local_release_dir }}/hyperkube" "{{ bin_dir }}/kubectl"
   changed_when: false
-  failed_when: "kubectl_task_compare_result.rc not in [0,1,2]"
   tags:
     - hyperkube
     - kubectl
     - upgrade
 
-- name: Copy kubectl from hyperkube container
-  command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -f /hyperkube /systembindir/kubectl"
-  when: kubectl_task_compare_result.rc != 0
-  register: kubectl_task_result
-  until: kubectl_task_result.rc == 0
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
-  changed_when: false
+- name: install | Set kubectl binary permissions
+  file:
+    path: "{{ bin_dir }}/kubectl"
+    mode: "0755"
+    state: file
   tags:
     - hyperkube
     - kubectl
@@ -37,7 +29,7 @@
 
 - name: Install kubectl bash completion
   shell: "{{ bin_dir }}/kubectl completion bash >/etc/bash_completion.d/kubectl.sh"
-  when: kubectl_task_compare_result.rc != 0 and ansible_os_family in ["Debian","RedHat"]
+  when: ansible_os_family in ["Debian","RedHat"]
   tags:
     - kubectl
 

roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2

@@ -1,4 +1,4 @@
-apiVersion: kubeadm.k8s.io/v1alpha1
+apiVersion: kubeadm.k8s.io/v1alpha2
 kind: MasterConfiguration
 api:
   advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }}
@@ -7,13 +7,14 @@ api:
   controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}
 {% endif %}
 etcd:
-  endpoints:
+  external:
+      endpoints:
 {% for endpoint in etcd_access_addresses.split(',') %}
-  - {{ endpoint }}
+      - {{ endpoint }}
 {% endfor %}
-  caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem
-  certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem
-  keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem
+      caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem
+      certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem
+      keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem
 networking:
   dnsDomain: {{ dns_domain }}
   serviceSubnet: {{ kube_service_addresses }}
@@ -27,6 +28,12 @@ kubeProxy:
 {% if kube_proxy_mode == 'ipvs' and kube_version | version_compare('v1.10', '<') %}
     featureGates: SupportIPVSProxyMode=true
     mode: ipvs
+{% elif kube_proxy_mode == 'ipvs' %}
+kubeProxy:
+  config:
+    featureGates:
+      SupportIPVSProxyMode: true
+    mode: ipvs
 {% endif %}
 {% if kube_proxy_nodeport_addresses %}
     nodePortAddresses: [{{ kube_proxy_nodeport_addresses_cidr }}]

roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2

@@ -138,3 +138,6 @@ nodeRegistration:
   taints:
   - effect: NoSchedule
     key: node-role.kubernetes.io/master
+{% if container_manager == 'crio' %}
+  criSocket: /var/run/crio/crio.sock
+{% endif %}

roles/kubernetes/node/defaults/main.yml

@@ -31,6 +31,11 @@ kubelet_cgroups_per_qos: true
 # Set to empty to avoid cgroup creation
 kubelet_enforce_node_allocatable: "\"\""
 
+# Set runtime cgroups
+kubelet_runtime_cgroups: "/systemd/system.slice"
+# Set kubelet cgroups
+kubelet_kubelet_cgroups: "/systemd/system.slice"
+
 # Set false to enable sharing a pid namespace between containers in a pod.
 # Note that PID namespace sharing requires docker >= 1.13.1.
 kubelet_disable_shared_pid: true

roles/kubernetes/node/tasks/install_host.yml

@@ -1,23 +1,17 @@
 ---
-- name: install | Compare host kubelet with hyperkube container
-  command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/cmp /hyperkube /systembindir/kubelet"
-  register: kubelet_task_compare_result
-  until: kubelet_task_compare_result.rc in [0,1,2]
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
+
+- name: install | Copy kubelet binary from download dir
+  command: rsync -piu "{{ local_release_dir }}/hyperkube" "{{ bin_dir }}/kubelet"
   changed_when: false
-  failed_when: "kubelet_task_compare_result.rc not in [0,1,2]"
   tags:
     - hyperkube
     - upgrade
 
-- name: install | Copy kubelet from hyperkube container
-  command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -f /hyperkube /systembindir/kubelet"
-  when: kubelet_task_compare_result.rc != 0
-  register: kubelet_task_result
-  until: kubelet_task_result.rc == 0
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
+- name: install | Set kubelet binary permissions
+  file:
+    path: "{{ bin_dir }}/kubelet"
+    mode: "0755"
+    state: file
   tags:
     - hyperkube
     - upgrade

roles/kubernetes/node/templates/kubelet.kubeadm.env.j2

@@ -34,7 +34,13 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 --node-status-update-frequency={{ kubelet_status_update_frequency }} \
 --cgroup-driver={{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }} \
 --max-pods={{ kubelet_max_pods }} \
+{% if container_manager == 'docker' %}
 --docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \
+{% endif %}
+{% if container_manager == 'crio' %}
+--container-runtime=remote \
+--container-runtime-endpoint=/var/run/crio/crio.sock \
+{% endif %}
 --anonymous-auth=false \
 --read-only-port={{ kube_read_only_port }} \
 {% if kube_version | version_compare('v1.8', '<') %}
@@ -42,6 +48,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% else %}
 --fail-swap-on={{ kubelet_fail_swap_on|default(true)}} \
 {% endif %}
+--runtime-cgroups={{ kubelet_runtime_cgroups }} --kubelet-cgroups={{ kubelet_kubelet_cgroups }} \
 {% endset %}
 
 {# Node reserved CPU/memory #}

roles/kubernetes/node/templates/kubelet.standard.env.j2

@@ -15,7 +15,9 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 --cadvisor-port={{ kube_cadvisor_port }} \
 --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \
 --node-status-update-frequency={{ kubelet_status_update_frequency }} \
+{% if container_manager == 'docker' %}
 --docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \
+{% endif %}
 --client-ca-file={{ kube_cert_dir }}/ca.pem \
 --tls-cert-file={{ kube_cert_dir }}/node-{{ inventory_hostname }}.pem \
 --tls-private-key-file={{ kube_cert_dir }}/node-{{ inventory_hostname }}-key.pem \
@@ -26,6 +28,10 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% if kube_version | version_compare('v1.7', '<') %}
 --enable-cri={{ kubelet_enable_cri }} \
 {% endif %}
+{% if container_manager == 'crio' %}
+--container-runtime=remote \
+--container-runtime-endpoint=/var/run/crio/crio.sock \
+{% endif %}
 --cgroup-driver={{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }} \
 --cgroups-per-qos={{ kubelet_cgroups_per_qos }} \
 --max-pods={{ kubelet_max_pods }} \