Fix recover-control-plane to work with etcd 3.3.x and add CI (#5500)

* Fix recover-control-plane to work with etcd 3.3.x and add CI

* Set default values for testcase

* Add actual test jobs

* Attempt to satisty gitlab ci linter

* Fix ansible targets

* Set etcd_member_name as stated in the docs...

* Recovering from 0 masters is not supported yet

* Add other master to broken_kube-master group as well

* Increase number of retries to see if etcd needs more time to heal

* Make number of retries for ETCD loops configurable, increase it for recovery CI and document it

tests/cloud_playbooks/roles/packet-ci/tasks/main.yml

@@ -5,7 +5,7 @@
 
 - name: Set VM count needed for CI test_id
   set_fact:
-    vm_count: "{%- if mode in ['separate', 'separate-scale', 'ha', 'ha-scale'] -%}{{ 3|int }}{%- elif mode == 'aio' -%}{{ 1|int }}{%- else -%}{{ 2|int }}{%- endif -%}"
+    vm_count: "{%- if mode in ['separate', 'separate-scale', 'ha', 'ha-scale', 'ha-recover', 'ha-recover-noquorum'] -%}{{ 3|int }}{%- elif mode == 'aio' -%}{{ 1|int }}{%- else -%}{{ 2|int }}{%- endif -%}"
 
 - import_tasks: create-vms.yml
   when:

tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2

@@ -45,6 +45,45 @@ instance-1
 
 [vault]
 instance-1
+{% elif mode == "ha-recover" %}
+[kube-master]
+instance-1
+instance-2
+
+[kube-node]
+instance-3
+
+[etcd]
+instance-3
+instance-1
+instance-2
+
+[broken_kube-master]
+instance-2
+
+[broken_etcd]
+instance-2 etcd_member_name=etcd3
+{% elif mode == "ha-recover-noquorum" %}
+[kube-master]
+instance-3
+instance-1
+instance-2
+
+[kube-node]
+instance-3
+
+[etcd]
+instance-3
+instance-1
+instance-2
+
+[broken_kube-master]
+instance-1
+instance-2
+
+[broken_etcd]
+instance-1 etcd_member_name=etcd2
+instance-2 etcd_member_name=etcd3
 {% endif %}
 
 [k8s-cluster:children]