mirror of
https://github.com/kubernetes-sigs/kubespray.git
synced 2026-10-29 22:03:22 -02:30
* fix(container-engine/cri-o): flush Restart crio handler inside the role During a Kubernetes upgrade (e.g. 1.32 -> 1.33) the cri-o systemd service is not restarted even though the new cri-o binaries are deployed. This leaves the node running the old cri-o version (reported as crio://1.32.x after kubelet is upgraded to 1.33.x); a manual 'systemctl restart crio' fixes it. The cri-o role notifies 'Restart crio' from several tasks (binary copy, registries, proxy drop-in, service file, etc.) but never flushes the handler itself. The handler only fired when the kubernetes/node role ran its own 'meta: flush_handlers' later in the same play. That cross-role dependency is fragile during upgrades: if kubernetes/node is skipped, tagged out, or the flush is reordered, the restart is silently dropped. Route every restart through the single handler: crio.conf and config.json now notify 'Restart crio' and the ad-hoc 'trigger service restart only when needed' task is dropped (its restart duplicated what the handler already does, and a restarted unit also starts a stopped one). Then add 'meta: flush_handlers' right after the service start task so the pending handlers run as part of this role regardless of any downstream role's flush timing. Signed-off-by: Mehrdad Biukian Naeini <mehrdadbiukian@gmail.com> * test(youki): run youki before cri-o in molecule converge The converge ran container-engine/cri-o before container-engine/youki, the reverse of the order used by the container-engine role, which installs the runtime roles (kata-containers, gvisor, crun, youki) before the container manager role. With cri-o flushing its restart handler inside the role, cri-o is now restarted while /usr/local/bin/youki does not exist yet: CRI-O skips the runtime whose binary is missing, so the youki runtime handler is never registered and the verify step fails with 'failed to find runtime handler youki from runtime list map[crun:...]'. Running the roles in the playbook order puts the youki binary in place before cri-o starts. Signed-off-by: Mehrdad Biukian Naeini <mehrdadbiukian@gmail.com> --------- Signed-off-by: Mehrdad Biukian Naeini <mehrdadbiukian@gmail.com>
308 lines
8.9 KiB
YAML
308 lines
8.9 KiB
YAML
---
|
|
- name: Cri-o | load vars
|
|
import_tasks: load_vars.yml
|
|
|
|
- name: Cri-o | check if fedora coreos
|
|
stat:
|
|
path: /run/ostree-booted
|
|
get_attributes: false
|
|
get_checksum: false
|
|
get_mime: false
|
|
register: ostree
|
|
|
|
- name: Cri-o | set is_ostree
|
|
set_fact:
|
|
is_ostree: "{{ ostree.stat.exists }}"
|
|
|
|
- name: Cri-o | get ostree version
|
|
shell: "set -o pipefail && rpm-ostree --version | awk -F\\' '/Version/{print $2}'"
|
|
args:
|
|
executable: /bin/bash
|
|
register: ostree_version
|
|
when: is_ostree
|
|
|
|
- name: Cri-o | Download cri-o
|
|
include_tasks: "../../../download/tasks/download_file.yml"
|
|
vars:
|
|
download: "{{ download_defaults | combine(downloads.crio) }}"
|
|
|
|
- name: Cri-o | special handling for amazon linux
|
|
import_tasks: "setup-amazon.yaml"
|
|
when: ansible_facts['distribution'] in ["Amazon"]
|
|
|
|
- name: Cri-o | build a list of crio runtimes with Katacontainers runtimes
|
|
set_fact:
|
|
crio_runtimes: "{{ crio_runtimes + kata_runtimes }}"
|
|
when:
|
|
- kata_containers_enabled
|
|
|
|
## After CRI-O v1.31, crun is default runtime.
|
|
# - name: Cri-o | build a list of crio runtimes with crun runtime
|
|
# set_fact:
|
|
# crio_runtimes: "{{ crio_runtimes + [crun_runtime] }}"
|
|
# when:
|
|
# - crun_enabled
|
|
|
|
- name: Cri-o | build a list of crio runtimes with runc runtime
|
|
set_fact:
|
|
crio_runtimes: "{{ crio_runtimes + [runc_runtime] }}"
|
|
when:
|
|
- runc_enabled
|
|
|
|
- name: Cri-o | build a list of crio runtimes with youki runtime
|
|
set_fact:
|
|
crio_runtimes: "{{ crio_runtimes + [youki_runtime] }}"
|
|
when:
|
|
- youki_enabled
|
|
|
|
- name: Cri-o | Stop kubelet service if running
|
|
service:
|
|
name: kubelet
|
|
state: stopped
|
|
when:
|
|
- crio_runtime_switch
|
|
- ansible_facts.services['kubelet.service'] is defined and ansible_facts.services['kubelet.service'].state == 'running'
|
|
|
|
- name: Cri-o | Get all pods
|
|
ansible.builtin.command: "{{ bin_dir }}/crictl pods -o json"
|
|
changed_when: false
|
|
register: crio_pods
|
|
when:
|
|
- crio_runtime_switch
|
|
- ansible_facts.services['crio.service'] is defined
|
|
|
|
- name: Cri-o | Stop and remove pods not on host network
|
|
ansible.builtin.command: "{{ bin_dir }}/crictl rmp -f {{ item.id }}"
|
|
loop: "{{ (crio_pods.stdout | from_json).items | default([]) | selectattr('metadata.namespace', 'ne', 'NODE') }}"
|
|
changed_when: true
|
|
when:
|
|
- crio_runtime_switch
|
|
- ansible_facts.services['crio.service'] is defined
|
|
- crio_pods.stdout is defined
|
|
|
|
- name: Cri-o | Stop and remove all remaining pods
|
|
ansible.builtin.command: "{{ bin_dir }}/crictl rmp -fa"
|
|
changed_when: true
|
|
when:
|
|
- crio_runtime_switch
|
|
- ansible_facts.services['crio.service'] is defined
|
|
|
|
- name: Cri-o | stop crio service if running
|
|
service:
|
|
name: crio
|
|
state: stopped
|
|
when:
|
|
- crio_runtime_switch
|
|
- ansible_facts.services['crio.service'] is defined and ansible_facts.services['crio.service'].state == 'running'
|
|
|
|
- name: Cri-o | make sure needed folders exist in the system
|
|
with_items:
|
|
- /etc/crio
|
|
- /etc/containers
|
|
- /etc/systemd/system/crio.service.d
|
|
file:
|
|
path: "{{ item }}"
|
|
state: directory
|
|
mode: "0755"
|
|
|
|
- name: Cri-o | install cri-o config
|
|
template:
|
|
src: crio.conf.j2
|
|
dest: /etc/crio/crio.conf
|
|
mode: "0644"
|
|
notify: Restart crio
|
|
|
|
- name: Cri-o | install config.json
|
|
template:
|
|
src: config.json.j2
|
|
dest: /etc/crio/config.json
|
|
mode: "0644"
|
|
notify: Restart crio
|
|
|
|
- name: Cri-o | copy binaries
|
|
copy:
|
|
src: "{{ local_release_dir }}/cri-o/bin/{{ item }}"
|
|
dest: "{{ bin_dir }}/{{ item }}"
|
|
mode: "0755"
|
|
remote_src: true
|
|
with_items:
|
|
- "{{ crio_bin_files }}"
|
|
notify: Restart crio
|
|
|
|
- name: Cri-o | create directory for libexec
|
|
file:
|
|
path: "{{ crio_libexec_dir }}"
|
|
state: directory
|
|
owner: root
|
|
mode: "0755"
|
|
|
|
- name: Cri-o | copy libexec
|
|
copy:
|
|
src: "{{ local_release_dir }}/cri-o/bin/{{ item }}"
|
|
dest: "{{ crio_libexec_dir }}/{{ item }}"
|
|
mode: "0755"
|
|
remote_src: true
|
|
with_items:
|
|
- "{{ crio_libexec_files }}"
|
|
notify: Restart crio
|
|
|
|
- name: Cri-o | copy service file
|
|
copy:
|
|
src: "{{ local_release_dir }}/cri-o/contrib/crio.service"
|
|
dest: /etc/systemd/system/crio.service
|
|
mode: "0644"
|
|
remote_src: true
|
|
notify: Restart crio
|
|
|
|
- name: Cri-o | configure crio to use kube reserved cgroups
|
|
ansible.builtin.copy:
|
|
dest: /etc/systemd/system/crio.service.d/00-slice.conf
|
|
owner: root
|
|
group: root
|
|
mode: '0644'
|
|
content: |
|
|
[Service]
|
|
Slice={{ kube_reserved_cgroups_for_service_slice }}
|
|
notify: Restart crio
|
|
when:
|
|
- kube_reserved is defined and kube_reserved is true
|
|
- kube_reserved_cgroups_for_service_slice is defined
|
|
|
|
- name: Cri-o | update the bin dir for crio.service file
|
|
replace:
|
|
dest: /etc/systemd/system/crio.service
|
|
regexp: "/usr/local/bin/crio"
|
|
replace: "{{ bin_dir }}/crio"
|
|
notify: Restart crio
|
|
|
|
- name: Cri-o | copy default policy
|
|
copy:
|
|
src: "{{ local_release_dir }}/cri-o/contrib/policy.json"
|
|
dest: /etc/containers/policy.json
|
|
mode: "0755"
|
|
remote_src: true
|
|
notify: Restart crio
|
|
|
|
- name: Cri-o | copy mounts.conf
|
|
template:
|
|
src: mounts.conf.j2
|
|
dest: /etc/containers/mounts.conf
|
|
mode: "0644"
|
|
when:
|
|
- ansible_facts['os_family'] == 'RedHat'
|
|
notify: Restart crio
|
|
|
|
- name: Cri-o | create directory for oci hooks
|
|
file:
|
|
path: /etc/containers/oci/hooks.d
|
|
state: directory
|
|
owner: root
|
|
mode: "0755"
|
|
|
|
- name: Cri-o | set overlay driver
|
|
community.general.ini_file:
|
|
dest: /etc/containers/storage.conf
|
|
section: storage
|
|
option: "{{ item.option }}"
|
|
value: "{{ item.value }}"
|
|
mode: "0644"
|
|
with_items:
|
|
- option: driver
|
|
value: '"overlay"'
|
|
- option: graphroot
|
|
value: '"/var/lib/containers/storage"'
|
|
- option: runroot
|
|
value: '"/var/run/containers/storage"'
|
|
|
|
# metacopy=on is available since 4.19 and was backported to RHEL 4.18 kernel
|
|
- name: Cri-o | set metacopy mount options correctly
|
|
community.general.ini_file:
|
|
dest: /etc/containers/storage.conf
|
|
section: storage.options.overlay
|
|
option: mountopt
|
|
value: >-
|
|
{{ '"nodev"' if ansible_facts['kernel'] is version(('4.18' if ansible_facts['os_family'] == 'RedHat' else '4.19'), '<')
|
|
else '"nodev,metacopy=on"' }}
|
|
mode: "0644"
|
|
|
|
- name: Cri-o | create directory registries configs
|
|
file:
|
|
path: /etc/containers/registries.conf.d
|
|
state: directory
|
|
owner: root
|
|
mode: "0755"
|
|
|
|
- name: Cri-o | write registries configs
|
|
template:
|
|
src: registry.conf.j2
|
|
dest: "/etc/containers/registries.conf.d/10-{{ item.prefix | default(item.location) | regex_replace(':|/', '_') }}.conf"
|
|
mode: "0644"
|
|
loop: "{{ crio_registries }}"
|
|
notify: Restart crio
|
|
|
|
- name: Cri-o | configure unqualified registry settings
|
|
template:
|
|
src: unqualified.conf.j2
|
|
dest: "/etc/containers/registries.conf.d/01-unqualified.conf"
|
|
mode: "0644"
|
|
notify: Restart crio
|
|
|
|
- name: Cri-o | write cri-o proxy drop-in
|
|
template:
|
|
src: http-proxy.conf.j2
|
|
dest: /etc/systemd/system/crio.service.d/http-proxy.conf
|
|
mode: "0644"
|
|
notify: Restart crio
|
|
when: http_proxy is defined or https_proxy is defined
|
|
|
|
- name: Cri-o | configure the uid/gid space for user namespaces
|
|
lineinfile:
|
|
path: '{{ item.path }}'
|
|
line: '{{ item.entry }}'
|
|
regex: '^\s*{{ crio_remap_user }}:'
|
|
state: '{{ "present" if crio_remap_enable | bool else "absent" }}'
|
|
loop:
|
|
- path: /etc/subuid
|
|
entry: '{{ crio_remap_user }}:{{ crio_subuid_start }}:{{ crio_subuid_length }}'
|
|
- path: /etc/subgid
|
|
entry: '{{ crio_remap_user }}:{{ crio_subgid_start }}:{{ crio_subgid_length }}'
|
|
loop_control:
|
|
label: '{{ item.path }}'
|
|
|
|
- name: Cri-o | ensure crio service is started and enabled
|
|
service:
|
|
name: crio
|
|
daemon_reload: true
|
|
enabled: true
|
|
state: started
|
|
|
|
# Every config/binary change above notifies "Restart crio", so this flush runs
|
|
# the pending handlers right away instead of leaving the cri-o restart to the
|
|
# end of the play (where it is lost if kubernetes/node is skipped, tagged out,
|
|
# or runs in a separate play, as happens during cluster upgrades).
|
|
# Note: flush_handlers flushes *all* notified handlers in the play, not only
|
|
# "Restart crio". See kubernetes-sigs/kubespray#13031.
|
|
- name: Cri-o | flush handlers to apply cri-o restart
|
|
meta: flush_handlers
|
|
|
|
- name: Cri-o | verify that crio is running
|
|
command: "{{ bin_dir }}/{{ crio_status_command }} info"
|
|
register: get_crio_info
|
|
until: get_crio_info is succeeded
|
|
changed_when: false
|
|
retries: 5
|
|
delay: "{{ retry_stagger | random + 3 }}"
|
|
|
|
# The kubelet service status can be 'not-found' if something depends on it.
|
|
# This check prevents failures when the service is in this indeterminate state,
|
|
# which can occur when adding new nodes to a cluster.
|
|
# See: https://superuser.com/questions/1755211/cleaning-debugging-services/1755215#1755215
|
|
|
|
- name: Cri-o | ensure kubelet service is started if present and stopped
|
|
service:
|
|
name: kubelet
|
|
state: started
|
|
when:
|
|
- crio_runtime_switch
|
|
- ansible_facts.services['kubelet.service'] is defined and ansible_facts.services['kubelet.service']['status'] != 'not-found'
|