Update etcd to 3.6 (#12634)

* [etcd] Update etcd 3.5.x to 3.6.5

- Add hashes for etcd 3.6.5
- Remove etcd v2 backup task for etcd 3.6
  The etcd 3.6 removes 'etcdctl backup' command with ETCDCTL_API=2
- Downgrade etcd to 3.5 in netchecker
  The netchecker does not work with etcd 3.6 becaust it removes v2 API support (--enable-v2).
  And netchekcer does not support v3 API.

* Fix: Change etcd config to clean up v2 store before upgrading etcd to 3.6

* Bump etcd to 3.6.8

roles/etcd/handlers/backup.yml

@@ -34,6 +34,7 @@
   when:
     - etcd_data_dir_member.stat.exists
     - etcd_cluster_is_healthy.rc == 0
+    - etcd_version is version('3.6.0', '<')
   command: >-
     {{ bin_dir }}/etcdctl backup
       --data-dir {{ etcd_data_dir }}

roles/etcd/tasks/clean_v2_store.yml

@@ -0,0 +1,43 @@
+---
+# When upgrading from etcd 3.5 to 3.6, need to clean up v2 store before upgrading.
+# Without this, etcd 3.6 will crash with following error:
+#   "panic: detected disallowed v2 WAL for stage --v2-deprecation=write-only [recovered]"
+- name: Cleanup v2 store when upgrade etcd from <3.6 to >=3.6
+  when:
+    - etcd_cluster_setup
+    - etcd_current_version != ''
+    - etcd_current_version is version('3.6.0', '<')
+    - etcd_version is version('3.6.0', '>=')
+  block:
+    - name: Ensure etcd version is >=3.5.26
+      when:
+        - etcd_current_version is version('3.5.26', '<')
+      fail:
+        msg: "You need to upgrade etcd to 3.5.26 or later before upgrade to 3.6. Current version is {{ etcd_current_version }}."
+
+    # Workarounds:
+    # Disable --enable-v2 (recommended in 20289) and do workaround of 20231 (MAX_WALS=1 and SNAPSHOT_COUNT=1)
+    # - https://github.com/etcd-io/etcd/issues/20809
+    # - https://github.com/etcd-io/etcd/discussions/20231#discussioncomment-13958051
+    - name: Change etcd configuration temporally to limit number of WALs and snapshots to clean up v2 store
+      ansible.builtin.lineinfile:
+        path: /etc/etcd.env
+        regexp: "{{ item.regexp }}"
+        line: "{{ item.line }}"
+      loop:
+        - { regexp: '^ETCD_SNAPSHOT_COUNT=', line: 'ETCD_SNAPSHOT_COUNT=1' }
+        - { regexp: '^ETCD_MAX_WALS=', line: 'ETCD_MAX_WALS=1' }
+        - { regexp: '^ETCD_MAX_SNAPSHOTS=', line: 'ETCD_MAX_SNAPSHOTS=1' }
+        - { regexp: '^ETCD_ENABLE_V2=', line: 'ETCD_ENABLE_V2=false' }
+
+    # Restart etcd to apply temporal configuration and prevent some upgrade failures
+    # See also: https://etcd.io/blog/2025/upgrade_from_3.5_to_3.6_issue_followup/
+    - name: Stop etcd
+      service:
+        name: etcd
+        state: stopped
+
+    - name: Start etcd
+      service:
+        name: etcd
+        state: started

roles/etcd/tasks/install_docker.yml

@@ -23,6 +23,14 @@
     - etcd_events_cluster_setup
     - etcd_image_tag not in etcd_events_current_docker_image.stdout | default('')
 
+- name: Get currently-deployed etcd version as x.y.z format
+  set_fact:
+    etcd_current_version: "{{ (etcd_current_docker_image.stdout | regex_search('.*:v([0-9]+\\.[0-9]+\\.[0-9]+)', '\\1'))[0] | default('') }}"
+  when: etcd_cluster_setup
+
+- name: Cleanup v2 store data
+  import_tasks: clean_v2_store.yml
+
 - name: Install etcd launch script
   template:
     src: etcd.j2

roles/etcd/tasks/install_host.yml

@@ -21,6 +21,14 @@
     - etcd_events_cluster_setup
     - etcd_version not in etcd_current_host_version.stdout | default('')
 
+- name: Get currently-deployed etcd version as x.y.z format
+  set_fact:
+    etcd_current_version: "{{ (etcd_current_host_version.stdout | regex_search('etcd Version: ([0-9]+\\.[0-9]+\\.[0-9]+)', '\\1'))[0] | default('') }}"
+  when: etcd_cluster_setup
+
+- name: Cleanup v2 store data
+  import_tasks: clean_v2_store.yml
+
 - name: Install | Copy etcd binary from download dir
   copy:
     src: "{{ local_release_dir }}/etcd-v{{ etcd_version }}-linux-{{ host_architecture }}/{{ item }}"

roles/etcd/tasks/main.yml

@@ -53,6 +53,12 @@
     - control-plane
     - network
 
+- name: Install etcd
+  include_tasks: "install_{{ etcd_deployment_type }}.yml"
+  when: ('etcd' in group_names)
+  tags:
+    - upgrade
+
 - name: Install etcdctl and etcdutl binary
   import_role:
     name: etcdctl_etcdutl
@@ -64,12 +70,6 @@
     - ('etcd' in group_names)
     - etcd_cluster_setup
 
-- name: Install etcd
-  include_tasks: "install_{{ etcd_deployment_type }}.yml"
-  when: ('etcd' in group_names)
-  tags:
-    - upgrade
-
 - name: Configure etcd
   include_tasks: configure.yml
   when: ('etcd' in group_names)

roles/etcd/templates/etcd.env.j2

@@ -25,8 +25,6 @@ ETCD_MAX_REQUEST_BYTES={{ etcd_max_request_bytes }}
 ETCD_LOG_LEVEL={{ etcd_log_level }}
 ETCD_MAX_SNAPSHOTS={{ etcd_max_snapshots }}
 ETCD_MAX_WALS={{ etcd_max_wals }}
-# Flannel need etcd v2 API
-ETCD_ENABLE_V2=true
 
 # TLS settings
 ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem