From 0bfd43499ada7aedfb685a4fd28570188d8daf23 Mon Sep 17 00:00:00 2001 From: Takuya Murakami Date: Mon, 16 Feb 2026 22:46:01 +0900 Subject: [PATCH] Update etcd to 3.6 (#12634) * [etcd] Update etcd 3.5.x to 3.6.5 - Add hashes for etcd 3.6.5 - Remove etcd v2 backup task for etcd 3.6 The etcd 3.6 removes 'etcdctl backup' command with ETCDCTL_API=2 - Downgrade etcd to 3.5 in netchecker The netchecker does not work with etcd 3.6 becaust it removes v2 API support (--enable-v2). And netchekcer does not support v3 API. * Fix: Change etcd config to clean up v2 store before upgrading etcd to 3.6 * Bump etcd to 3.6.8 --- README.md | 2 +- roles/etcd/handlers/backup.yml | 1 + roles/etcd/tasks/clean_v2_store.yml | 43 +++++++++++++++++++ roles/etcd/tasks/install_docker.yml | 8 ++++ roles/etcd/tasks/install_host.yml | 8 ++++ roles/etcd/tasks/main.yml | 12 +++--- roles/etcd/templates/etcd.env.j2 | 2 - .../defaults/main/download.yml | 3 +- .../vars/main/checksums.yml | 27 ++++++++++++ roles/kubespray_defaults/vars/main/main.yml | 2 +- 10 files changed, 97 insertions(+), 11 deletions(-) create mode 100644 roles/etcd/tasks/clean_v2_store.yml diff --git a/README.md b/README.md index 831adb686..d8bf0836e 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ Note: - Core - [kubernetes](https://github.com/kubernetes/kubernetes) 1.35.1 - - [etcd](https://github.com/etcd-io/etcd) 3.5.27 + - [etcd](https://github.com/etcd-io/etcd) 3.6.8 - [docker](https://www.docker.com/) 28.3 - [containerd](https://containerd.io/) 2.2.1 - [cri-o](http://cri-o.io/) 1.35.0 (experimental: see [CRI-O Note](docs/CRI/cri-o.md). Only on fedora, ubuntu and centos based OS) diff --git a/roles/etcd/handlers/backup.yml b/roles/etcd/handlers/backup.yml index 84e03accd..b2e02fb91 100644 --- a/roles/etcd/handlers/backup.yml +++ b/roles/etcd/handlers/backup.yml @@ -34,6 +34,7 @@ when: - etcd_data_dir_member.stat.exists - etcd_cluster_is_healthy.rc == 0 + - etcd_version is version('3.6.0', '<') command: >- {{ bin_dir }}/etcdctl backup --data-dir {{ etcd_data_dir }} diff --git a/roles/etcd/tasks/clean_v2_store.yml b/roles/etcd/tasks/clean_v2_store.yml new file mode 100644 index 000000000..126efa2fb --- /dev/null +++ b/roles/etcd/tasks/clean_v2_store.yml @@ -0,0 +1,43 @@ +--- +# When upgrading from etcd 3.5 to 3.6, need to clean up v2 store before upgrading. +# Without this, etcd 3.6 will crash with following error: +# "panic: detected disallowed v2 WAL for stage --v2-deprecation=write-only [recovered]" +- name: Cleanup v2 store when upgrade etcd from <3.6 to >=3.6 + when: + - etcd_cluster_setup + - etcd_current_version != '' + - etcd_current_version is version('3.6.0', '<') + - etcd_version is version('3.6.0', '>=') + block: + - name: Ensure etcd version is >=3.5.26 + when: + - etcd_current_version is version('3.5.26', '<') + fail: + msg: "You need to upgrade etcd to 3.5.26 or later before upgrade to 3.6. Current version is {{ etcd_current_version }}." + + # Workarounds: + # Disable --enable-v2 (recommended in 20289) and do workaround of 20231 (MAX_WALS=1 and SNAPSHOT_COUNT=1) + # - https://github.com/etcd-io/etcd/issues/20809 + # - https://github.com/etcd-io/etcd/discussions/20231#discussioncomment-13958051 + - name: Change etcd configuration temporally to limit number of WALs and snapshots to clean up v2 store + ansible.builtin.lineinfile: + path: /etc/etcd.env + regexp: "{{ item.regexp }}" + line: "{{ item.line }}" + loop: + - { regexp: '^ETCD_SNAPSHOT_COUNT=', line: 'ETCD_SNAPSHOT_COUNT=1' } + - { regexp: '^ETCD_MAX_WALS=', line: 'ETCD_MAX_WALS=1' } + - { regexp: '^ETCD_MAX_SNAPSHOTS=', line: 'ETCD_MAX_SNAPSHOTS=1' } + - { regexp: '^ETCD_ENABLE_V2=', line: 'ETCD_ENABLE_V2=false' } + + # Restart etcd to apply temporal configuration and prevent some upgrade failures + # See also: https://etcd.io/blog/2025/upgrade_from_3.5_to_3.6_issue_followup/ + - name: Stop etcd + service: + name: etcd + state: stopped + + - name: Start etcd + service: + name: etcd + state: started diff --git a/roles/etcd/tasks/install_docker.yml b/roles/etcd/tasks/install_docker.yml index f393bd4eb..f6e9067b7 100644 --- a/roles/etcd/tasks/install_docker.yml +++ b/roles/etcd/tasks/install_docker.yml @@ -23,6 +23,14 @@ - etcd_events_cluster_setup - etcd_image_tag not in etcd_events_current_docker_image.stdout | default('') +- name: Get currently-deployed etcd version as x.y.z format + set_fact: + etcd_current_version: "{{ (etcd_current_docker_image.stdout | regex_search('.*:v([0-9]+\\.[0-9]+\\.[0-9]+)', '\\1'))[0] | default('') }}" + when: etcd_cluster_setup + +- name: Cleanup v2 store data + import_tasks: clean_v2_store.yml + - name: Install etcd launch script template: src: etcd.j2 diff --git a/roles/etcd/tasks/install_host.yml b/roles/etcd/tasks/install_host.yml index 8dca96546..f76dd2df6 100644 --- a/roles/etcd/tasks/install_host.yml +++ b/roles/etcd/tasks/install_host.yml @@ -21,6 +21,14 @@ - etcd_events_cluster_setup - etcd_version not in etcd_current_host_version.stdout | default('') +- name: Get currently-deployed etcd version as x.y.z format + set_fact: + etcd_current_version: "{{ (etcd_current_host_version.stdout | regex_search('etcd Version: ([0-9]+\\.[0-9]+\\.[0-9]+)', '\\1'))[0] | default('') }}" + when: etcd_cluster_setup + +- name: Cleanup v2 store data + import_tasks: clean_v2_store.yml + - name: Install | Copy etcd binary from download dir copy: src: "{{ local_release_dir }}/etcd-v{{ etcd_version }}-linux-{{ host_architecture }}/{{ item }}" diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml index 543eefb3c..5f77892b1 100644 --- a/roles/etcd/tasks/main.yml +++ b/roles/etcd/tasks/main.yml @@ -53,6 +53,12 @@ - control-plane - network +- name: Install etcd + include_tasks: "install_{{ etcd_deployment_type }}.yml" + when: ('etcd' in group_names) + tags: + - upgrade + - name: Install etcdctl and etcdutl binary import_role: name: etcdctl_etcdutl @@ -64,12 +70,6 @@ - ('etcd' in group_names) - etcd_cluster_setup -- name: Install etcd - include_tasks: "install_{{ etcd_deployment_type }}.yml" - when: ('etcd' in group_names) - tags: - - upgrade - - name: Configure etcd include_tasks: configure.yml when: ('etcd' in group_names) diff --git a/roles/etcd/templates/etcd.env.j2 b/roles/etcd/templates/etcd.env.j2 index 33794412c..36bf59c5c 100644 --- a/roles/etcd/templates/etcd.env.j2 +++ b/roles/etcd/templates/etcd.env.j2 @@ -25,8 +25,6 @@ ETCD_MAX_REQUEST_BYTES={{ etcd_max_request_bytes }} ETCD_LOG_LEVEL={{ etcd_log_level }} ETCD_MAX_SNAPSHOTS={{ etcd_max_snapshots }} ETCD_MAX_WALS={{ etcd_max_wals }} -# Flannel need etcd v2 API -ETCD_ENABLE_V2=true # TLS settings ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem diff --git a/roles/kubespray_defaults/defaults/main/download.yml b/roles/kubespray_defaults/defaults/main/download.yml index 9fd0e9f86..f50f8088a 100644 --- a/roles/kubespray_defaults/defaults/main/download.yml +++ b/roles/kubespray_defaults/defaults/main/download.yml @@ -237,7 +237,8 @@ netcheck_agent_image_repo: "{{ docker_image_repo }}/mirantis/k8s-netchecker-agen netcheck_agent_image_tag: "v{{ netcheck_version }}" netcheck_server_image_repo: "{{ docker_image_repo }}/mirantis/k8s-netchecker-server" netcheck_server_image_tag: "v{{ netcheck_version }}" -netcheck_etcd_image_tag: "{{ etcd_image_tag }}" +# netchecker doesn't work with etcd>=3.6 because etcd v2 API is removed +netcheck_etcd_image_tag: "v{{ (etcd_binary_checksums['amd64'].keys() | select('version', '3.6', '<'))[0] }}" cilium_image_repo: "{{ quay_image_repo }}/cilium/cilium" cilium_image_tag: "v{{ cilium_version }}" cilium_operator_image_repo: "{{ quay_image_repo }}/cilium/operator" diff --git a/roles/kubespray_defaults/vars/main/checksums.yml b/roles/kubespray_defaults/vars/main/checksums.yml index 811908950..0b7497de5 100644 --- a/roles/kubespray_defaults/vars/main/checksums.yml +++ b/roles/kubespray_defaults/vars/main/checksums.yml @@ -284,6 +284,15 @@ kubeadm_checksums: 1.33.0: sha256:26cb7ac57d522a59c84c4784b176097d23c7b4e61874fab84ae719d0e43ac0bc etcd_binary_checksums: arm64: + 3.6.8: sha256:438f56a700d17ce761510a3e63e6fa5c1d587b2dd4d7a22c179c09a649366760 + 3.6.7: sha256:ef5fc443cf7cc5b82738f3c28363704896551900af90a6d622cae740b5644270 + 3.6.6: sha256:8a15f5427c111ff4692753682374970fb68878401d946c2c28bdad6857db652f + 3.6.5: sha256:7010161787077b07de29b15b76825ceacbbcedcb77fe2e6832f509be102cab6b + 3.6.4: sha256:323421fa279f4f3d7da4c7f2dfa17d9e49529cb2b4cdf40899a7416bccdde42d + 3.6.3: sha256:4b39989093699da7502d1cdd649c412055a2bddd26b3d80ed87d0db31957075c + 3.6.2: sha256:79d0a2488967aa07ecfde79158b1dab458158522f834810c2827eecac4695a31 + 3.6.1: sha256:5f8ed6e314df44128c218decbf0d146cf882583d05c6f6d9023ce905d232aaec + 3.6.0: sha256:81477b120ef66ff338fe7de63d894e5feec17e6e1f1d98507676832e089d9b58 3.5.27: sha256:1277309f540c5a0329c428f95455c9f76d24f768c8d28fd2753e891c379053fa 3.5.26: sha256:93ac1667df0e178ea6d152476ce4088df4075604fe4bc7f85f4719e863cd030b 3.5.25: sha256:419dce0b679df31cc45201ef2449b7a6a48e9d241af01741957c9ac86a35badc @@ -307,6 +316,15 @@ etcd_binary_checksums: 3.5.7: sha256:1a35314900da7db006b198dd917e923459b462128101736c63a3cda57ecdbf51 3.5.6: sha256:888e25c9c94702ac1254c7655709b44bb3711ebaabd3cb05439f3dd1f2b51a87 amd64: + 3.6.8: sha256:cf9cfe91a4856cb90eed9c99e6aee4b708db2c7888b88a6f116281f04b0ea693 + 3.6.7: sha256:cf8af880c5a01ee5363cefa14a3e0cb7e5308dcf4ed17a6973099c9a7aee5a9a + 3.6.6: sha256:887afaa4a99f22d802ccdfbe65730a5e79aa5c9ce2c8799c67e9d804c50ecedb + 3.6.5: sha256:66bad39ed920f6fc15fd74adcb8bfd38ba9a6412f8c7852d09eb11670e88cac3 + 3.6.4: sha256:4d5f3101daa534e45ccaf3eec8d21c19b7222db377bcfd5e5a9144155238c105 + 3.6.3: sha256:3f3b4aa9785d86322c50b296eebdc7a0a57b27065190154b5858bf6a7512ac10 + 3.6.2: sha256:4b5d55d61e2218fab7c1cc1c00b341c469159ecde8cedd575fa858683f67e9f4 + 3.6.1: sha256:1324664bfe56d178d1362a57462ca5a7b26a6d2cbe9e1c94b6820e32cb82d673 + 3.6.0: sha256:42305b0dcbba7b6fdff0382d0c7b99c42026c88c44847a619ab58cde216725d9 3.5.27: sha256:0aad9a9e4e0817a021e933f9806a2b2960a62f949ad5a3d6436d8886945cb1bc 3.5.26: sha256:0a682a91201dc8351d507210bc30b021a11e254eab806f03224b51e8fad29abb 3.5.25: sha256:168af82b59772e1811a9af7b358d42f5c6df44e0d9767afb006ecf12c4bbd607 @@ -330,6 +348,15 @@ etcd_binary_checksums: 3.5.7: sha256:a43119af79c592a874e8f59c4f23832297849d0c479338f9df36e196b86bc396 3.5.6: sha256:4db32e3bc06dd0999e2171f76a87c1cffed8369475ec7aa7abee9023635670fb ppc64le: + 3.6.8: sha256:3b9bb486b0eb8d79b30410749ec26e174db075956c9ecb533b313b9263e7ba78 + 3.6.7: sha256:de3b1ed50fc8868cdd56b12b0cd81d6740bf53edbca570400a78e530e4829b7b + 3.6.6: sha256:e4f528b63a731e9b96f5d10f55ce096223fb4e1bc1778aa2535a3d47e9a129e5 + 3.6.5: sha256:3cf99879c7c5b8678a0ec2edf9102b268ea934584db2850f049d89ed8e36b61c + 3.6.4: sha256:2910fc73e42e1eeb9cc7da8080b821c7649558465e0e6122e49afce832e4b9da + 3.6.3: sha256:de8ee412ee2669483fd9c730e915c5bd4fe113ba33be4a70305d13ff35e1f919 + 3.6.2: sha256:bf79b9d4c7e9f86e611e73de9fe54a195bc0ad54aeb17200b1c8bda3c4119705 + 3.6.1: sha256:bb87fcd0ea4b9fabf502703512c416ca1d9f4082679cb7f6dbc34bed3dfc13f6 + 3.6.0: sha256:1180d06e3a3787ab65078d9a488f778a4712c59cc82d614abde80c5d06efe38f 3.5.27: sha256:b41d488dcd579e780f49f5bd747e9386e17e1376ffb77bfff061f7944818a678 3.5.26: sha256:9678ddaced9fcd4878b76b0b76c9c2a3638a70bdc362c9f4cb25ecc48de2c6d3 3.5.25: sha256:0dee64e99a43a06dd9541a40a18b52c7309eb1682a2a32740d4bdf358296c007 diff --git a/roles/kubespray_defaults/vars/main/main.yml b/roles/kubespray_defaults/vars/main/main.yml index e351c0d6d..87d83c679 100644 --- a/roles/kubespray_defaults/vars/main/main.yml +++ b/roles/kubespray_defaults/vars/main/main.yml @@ -12,7 +12,7 @@ pod_infra_supported_versions: '1.33': '3.10' etcd_supported_versions: - '1.35': "{{ (etcd_binary_checksums['amd64'].keys() | select('version', '3.6', '<'))[0] }}" + '1.35': "{{ (etcd_binary_checksums['amd64'].keys() | select('version', '3.7', '<'))[0] }}" '1.34': "{{ (etcd_binary_checksums['amd64'].keys() | select('version', '3.6', '<'))[0] }}" '1.33': "{{ (etcd_binary_checksums['amd64'].keys() | select('version', '3.6', '<'))[0] }}" # Kubespray constants