Fix recover-control-plane to work with etcd 3.3.x and add CI (#5500)

* Fix recover-control-plane to work with etcd 3.3.x and add CI

* Set default values for testcase

* Add actual test jobs

* Attempt to satisty gitlab ci linter

* Fix ansible targets

* Set etcd_member_name as stated in the docs...

* Recovering from 0 masters is not supported yet

* Add other master to broken_kube-master group as well

* Increase number of retries to see if etcd needs more time to heal

* Make number of retries for ETCD loops configurable, increase it for recovery CI and document it

roles/etcd/defaults/main.yml

@@ -62,3 +62,6 @@ etcd_secure_client: true
 
 # Enable peer client cert authentication
 etcd_peer_client_auth: true
+
+# Number of loop retries
+etcd_retries: 4

roles/etcd/tasks/configure.yml

@@ -67,7 +67,7 @@
   shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_client_url }} cluster-health | grep -q 'cluster is healthy'"
   register: etcd_cluster_is_healthy
   until: etcd_cluster_is_healthy.rc == 0
-  retries: 4
+  retries: "{{ etcd_retries }}"
   delay: "{{ retry_stagger | random + 3 }}"
   ignore_errors: false
   changed_when: false
@@ -88,7 +88,7 @@
   shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_client_url }} cluster-health | grep -q 'cluster is healthy'"
   register: etcd_events_cluster_is_healthy
   until: etcd_events_cluster_is_healthy.rc == 0
-  retries: 4
+  retries: "{{ etcd_retries }}"
   delay: "{{ retry_stagger | random + 3 }}"
   ignore_errors: false
   changed_when: false

roles/etcd/tasks/install_docker.yml

@@ -6,7 +6,7 @@
            {{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy"
   register: etcd_task_result
   until: etcd_task_result.rc == 0
-  retries: 4
+  retries: "{{ etcd_retries }}"
   delay: "{{ retry_stagger | random + 3 }}"
   changed_when: false
   when: etcd_cluster_setup

roles/etcd/tasks/join_etcd-events_member.yml

@@ -3,7 +3,7 @@
   shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} member add {{ etcd_member_name }} {{ etcd_events_peer_url }}"
   register: member_add_result
   until: member_add_result.rc == 0
-  retries: 4
+  retries: "{{ etcd_retries }}"
   delay: "{{ retry_stagger | random + 3 }}"
   when: target_node == inventory_hostname
   environment:

roles/etcd/tasks/join_etcd_member.yml

@@ -3,7 +3,7 @@
   shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} member add {{ etcd_member_name }} {{ etcd_peer_url }}"
   register: member_add_result
   until: member_add_result.rc == 0
-  retries: 4
+  retries: "{{ etcd_retries }}"
   delay: "{{ retry_stagger | random + 3 }}"
   when: target_node == inventory_hostname
   environment: