Fix recover-control-plane to work with etcd 3.3.x and add CI (#5500)

* Fix recover-control-plane to work with etcd 3.3.x and add CI

* Set default values for testcase

* Add actual test jobs

* Attempt to satisty gitlab ci linter

* Fix ansible targets

* Set etcd_member_name as stated in the docs...

* Recovering from 0 masters is not supported yet

* Add other master to broken_kube-master group as well

* Increase number of retries to see if etcd needs more time to heal

* Make number of retries for ETCD loops configurable, increase it for recovery CI and document it

roles/etcd/defaults/main.yml

@@ -62,3 +62,6 @@ etcd_secure_client: true
 
 # Enable peer client cert authentication
 etcd_peer_client_auth: true
+
+# Number of loop retries
+etcd_retries: 4

roles/etcd/tasks/configure.yml

@@ -67,7 +67,7 @@
   shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_client_url }} cluster-health | grep -q 'cluster is healthy'"
   register: etcd_cluster_is_healthy
   until: etcd_cluster_is_healthy.rc == 0
-  retries: 4
+  retries: "{{ etcd_retries }}"
   delay: "{{ retry_stagger | random + 3 }}"
   ignore_errors: false
   changed_when: false
@@ -88,7 +88,7 @@
   shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_client_url }} cluster-health | grep -q 'cluster is healthy'"
   register: etcd_events_cluster_is_healthy
   until: etcd_events_cluster_is_healthy.rc == 0
-  retries: 4
+  retries: "{{ etcd_retries }}"
   delay: "{{ retry_stagger | random + 3 }}"
   ignore_errors: false
   changed_when: false

roles/etcd/tasks/install_docker.yml

@@ -6,7 +6,7 @@
            {{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy"
   register: etcd_task_result
   until: etcd_task_result.rc == 0
-  retries: 4
+  retries: "{{ etcd_retries }}"
   delay: "{{ retry_stagger | random + 3 }}"
   changed_when: false
   when: etcd_cluster_setup

roles/etcd/tasks/join_etcd-events_member.yml

@@ -3,7 +3,7 @@
   shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} member add {{ etcd_member_name }} {{ etcd_events_peer_url }}"
   register: member_add_result
   until: member_add_result.rc == 0
-  retries: 4
+  retries: "{{ etcd_retries }}"
   delay: "{{ retry_stagger | random + 3 }}"
   when: target_node == inventory_hostname
   environment:

roles/etcd/tasks/join_etcd_member.yml

@@ -3,7 +3,7 @@
   shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} member add {{ etcd_member_name }} {{ etcd_peer_url }}"
   register: member_add_result
   until: member_add_result.rc == 0
-  retries: 4
+  retries: "{{ etcd_retries }}"
   delay: "{{ retry_stagger | random + 3 }}"
   when: target_node == inventory_hostname
   environment:

roles/recover_control_plane/etcd/tasks/main.yml

@@ -1,7 +1,78 @@
 ---
-- include_tasks: prepare.yml
+- name: Get etcd endpoint health
+  shell: "{{ bin_dir }}/etcdctl --cacert {{ etcd_cert_dir }}/ca.pem --cert {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem --key {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem --endpoints={{ etcd_access_addresses }} endpoint health"
+  register: etcd_endpoint_health
+  ignore_errors: true
+  changed_when: false
+  check_mode: no
+  environment:
+    - ETCDCTL_API: 3
+  when:
+    - groups['broken_etcd']
+
+- name: Set healthy fact
+  set_fact:
+    healthy: "{{ etcd_endpoint_health.stderr | match('Error: unhealthy cluster') }}"
+  when:
+    - groups['broken_etcd']
+
+- name: Set has_quorum fact
+  set_fact:
+    has_quorum: "{{ etcd_endpoint_health.stdout_lines | select('match', '.*is healthy.*') | list | length >= etcd_endpoint_health.stderr_lines | select('match', '.*is unhealthy.*') | list | length }}"
 
 - include_tasks: recover_lost_quorum.yml
   when:
-    - has_etcdctl
-    - not etcd_cluster_is_healthy
+    - groups['broken_etcd']
+    - not has_quorum
+
+- name: Remove etcd data dir
+  file:
+    path: "{{ etcd_data_dir }}"
+    state: absent
+  delegate_to: "{{ item }}"
+  with_items: "{{ groups['broken_etcd'] }}"
+  when:
+    - groups['broken_etcd']
+    - has_quorum
+
+- name: Delete old certificates
+  # noqa 302 - rm is ok here for now
+  shell: "rm {{ etcd_cert_dir }}/*{{ item }}*"
+  with_items: "{{ groups['broken_etcd'] }}"
+  register: delete_old_cerificates
+  ignore_errors: true
+  when: groups['broken_etcd']
+
+- name: Fail if unable to delete old certificates
+  fail:
+    msg: "Unable to delete old certificates for: {{ item.item }}"
+  loop: "{{ delete_old_cerificates.results }}"
+  changed_when: false
+  when:
+    - groups['broken_etcd']
+    - "item.rc != 0 and not 'No such file or directory' in item.stderr"
+
+- name: Get etcd cluster members
+  shell: "{{ bin_dir }}/etcdctl --cacert {{ etcd_cert_dir }}/ca.pem --cert {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem --key {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem member list"
+  register: member_list
+  changed_when: false
+  check_mode: no
+  environment:
+    - ETCDCTL_API: 3
+  when:
+    - groups['broken_etcd']
+    - not healthy
+    - has_quorum
+
+- name: Remove broken cluster members
+  shell: "{{ bin_dir }}/etcdctl --cacert {{ etcd_cert_dir }}/ca.pem --cert {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem --key {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem --endpoints={{ etcd_access_addresses }} member remove {{ item[1].replace(' ','').split(',')[0] }}"
+  environment:
+    - ETCDCTL_API: 3
+  with_nested:
+    - "{{ groups['broken_etcd'] }}"
+    - "{{ member_list.stdout_lines }}"
+  when:
+    - groups['broken_etcd']
+    - not healthy
+    - has_quorum
+    - hostvars[item[0]]['etcd_member_name'] == item[1].replace(' ','').split(',')[2]

roles/recover_control_plane/etcd/tasks/prepare.yml

@@ -1,48 +0,0 @@
----
-- name: Delete old certificates
-  # noqa 302 - rm is ok here for now
-  shell: "rm /etc/ssl/etcd/ssl/*{{ item }}* /etc/kubernetes/ssl/etcd/*{{ item }}*"
-  with_items: "{{ old_etcds.split(',') }}"
-  register: delete_old_cerificates
-  ignore_errors: true
-  when: old_etcds is defined
-
-- name: Fail if unable to delete old certificates
-  fail:
-    msg: "Unable to delete old certificates for: {{ item.item }}"
-  loop: "{{ delete_old_cerificates.results }}"
-  changed_when: false
-  when:
-    - old_etcds is defined
-    - "item.rc != 0 and not 'No such file or directory' in item.stderr"
-
-- name: Get etcd cluster members
-  shell: "{{ bin_dir }}/etcdctl member list"
-  register: member_list
-  changed_when: false
-  check_mode: no
-  environment:
-    - ETCDCTL_API: 3
-    - ETCDCTL_CA_FILE: /etc/ssl/etcd/ssl/ca.pem
-    - ETCDCTL_CERT: "/etc/ssl/etcd/ssl/admin-{{ inventory_hostname }}.pem"
-    - ETCDCTL_KEY: "/etc/ssl/etcd/ssl/admin-{{ inventory_hostname }}-key.pem"
-  when:
-    - has_etcdctl
-    - etcd_cluster_is_healthy
-    - old_etcd_members is defined
-
-- name: Remove old cluster members
-  shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} member remove {{ item[1].replace(' ','').split(',')[0] }}"
-  environment:
-    - ETCDCTL_API: 3
-    - ETCDCTL_CA_FILE: /etc/ssl/etcd/ssl/ca.pem
-    - ETCDCTL_CERT: "/etc/ssl/etcd/ssl/admin-{{ inventory_hostname }}.pem"
-    - ETCDCTL_KEY: "/etc/ssl/etcd/ssl/admin-{{ inventory_hostname }}-key.pem"
-  with_nested:
-    - "{{ old_etcd_members.split(',') }}"
-    - "{{ member_list.stdout_lines }}"
-  when:
-    - has_etcdctl
-    - etcd_cluster_is_healthy
-    - old_etcd_members is defined
-    - item[0] == item[1].replace(' ','').split(',')[2]

roles/recover_control_plane/etcd/tasks/recover_lost_quorum.yml

@@ -1,11 +1,8 @@
 ---
 - name: Save etcd snapshot
-  shell: "{{ bin_dir }}/etcdctl snapshot save /tmp/snapshot.db"
+  shell: "{{ bin_dir }}/etcdctl --cacert {{ etcd_cert_dir }}/ca.pem --cert {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem --key {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem snapshot save /tmp/snapshot.db"
   environment:
     - ETCDCTL_API: 3
-    - ETCDCTL_CA_FILE: /etc/ssl/etcd/ssl/ca.pem
-    - ETCDCTL_CERT: "/etc/ssl/etcd/ssl/member-{{ inventory_hostname }}.pem"
-    - ETCDCTL_KEY: "/etc/ssl/etcd/ssl/member-{{ inventory_hostname }}-key.pem"
   when: etcd_snapshot is not defined
 
 - name: Transfer etcd snapshot to host
@@ -25,12 +22,9 @@
     state: absent
 
 - name: Restore etcd snapshot
-  shell: "{{ bin_dir }}/etcdctl snapshot restore /tmp/snapshot.db --name {{ etcd_member_name }} --initial-cluster {{ etcd_member_name }}={{ etcd_peer_url }} --initial-cluster-token k8s_etcd --initial-advertise-peer-urls {{ etcd_peer_url }} --data-dir {{ etcd_data_dir }}"
+  shell: "{{ bin_dir }}/etcdctl --cacert {{ etcd_cert_dir }}/ca.pem --cert {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem --key {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem snapshot restore /tmp/snapshot.db --name {{ etcd_member_name }} --initial-cluster {{ etcd_member_name }}={{ etcd_peer_url }} --initial-cluster-token k8s_etcd --initial-advertise-peer-urls {{ etcd_peer_url }} --data-dir {{ etcd_data_dir }}"
   environment:
     - ETCDCTL_API: 3
-    - ETCDCTL_CA_FILE: /etc/ssl/etcd/ssl/ca.pem
-    - ETCDCTL_CERT: "/etc/ssl/etcd/ssl/member-{{ inventory_hostname }}.pem"
-    - ETCDCTL_KEY: "/etc/ssl/etcd/ssl/member-{{ inventory_hostname }}-key.pem"
 
 - name: Remove etcd snapshot
   file:

roles/recover_control_plane/master/tasks/main.yml

@@ -8,21 +8,22 @@
   retries: 6
   delay: 10
   changed_when: false
+  when: groups['broken_kube-master']
 
-- name: Delete old kube-master nodes from cluster
+- name: Delete broken kube-master nodes from cluster
   shell: "{{ bin_dir }}/kubectl delete node {{ item }}"
   environment:
     - KUBECONFIG: "{{ ansible_env.HOME | default('/root') }}/.kube/config"
-  with_items: "{{ old_kube_masters.split(',') }}"
-  register: delete_old_kube_masters
+  with_items: "{{ groups['broken_kube-master'] }}"
+  register: delete_broken_kube_masters
   failed_when: false
-  when: old_kube_masters is defined
+  when: groups['broken_kube-master']
 
-- name: Fail if unable to delete old kube-master nodes from cluster
+- name: Fail if unable to delete broken kube-master nodes from cluster
   fail:
-    msg: "Unable to delete old kube-master node: {{ item.item }}"
-  loop: "{{ delete_old_kube_masters.results }}"
+    msg: "Unable to delete broken kube-master node: {{ item.item }}"
+  loop: "{{ delete_broken_kube_masters.results }}"
   changed_when: false
   when:
-    - old_kube_masters is defined
+    - groups['broken_kube-master']
     - "item.rc != 0 and not 'NotFound' in item.stderr"

roles/recover_control_plane/pre-recover/defaults/main.yml

@@ -1,2 +0,0 @@
----
-control_plane_is_converged: "{{ groups['etcd'] | sort == groups['kube-master'] | sort | bool }}"

roles/recover_control_plane/pre-recover/tasks/main.yml

@@ -1,36 +0,0 @@
----
-- name: Check for etcdctl binary
-  raw: "test -e {{ bin_dir }}/etcdctl"
-  register: test_etcdctl
-
-- name: Set has_etcdctl fact
-  set_fact:
-    has_etcdctl: "{{ test_etcdctl.rc == 0 | bool }}"
-
-- name: Check if etcd cluster is healthy
-  shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
-  register: etcd_cluster_health
-  ignore_errors: true
-  changed_when: false
-  check_mode: no
-  environment:
-    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
-    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
-    ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
-  when: has_etcdctl
-
-- name: Set etcd_cluster_is_healthy fact
-  set_fact:
-    etcd_cluster_is_healthy: "{{ etcd_cluster_health.rc == 0 | bool }}"
-
-- name: Abort if etcd cluster is healthy and old_etcd_members is undefined
-  assert:
-    that: "{{ old_etcd_members is defined }}"
-    msg: "'old_etcd_members' must be defined when the etcd cluster has quorum."
-  when: etcd_cluster_is_healthy
-
-- name: Warn for untested recovery
-  debug:
-    msg: Control plane recovery of split control planes is UNTESTED! Abort or continue at your own risk.
-  delay: 30
-  when: not control_plane_is_converged