Etcd cluster setup makeover

The current way to setup the etc cluster is messy and buggy.

- It checks for cluster is healthy before the cluster is even created.
- The unit files are started on handlers, not in the task, so you mess with "flush handlers".
- The join_member.yml is not used.
- etcd events cluster is not configured for kubeadm
- remove duplicate runs between running the role on etcd nodes and k8s nodes

roles/etcd/tasks/configure.yml

@@ -1,11 +1,104 @@
 ---
+- name: Configure | Check if etcd cluster is healthy
+  shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
+  register: etcd_cluster_is_healthy
+  ignore_errors: true
+  changed_when: false
+  check_mode: no
+  when: is_etcd_master and etcd_cluster_setup
+  tags:
+    - facts
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+
+- name: Configure | Check if etcd-events cluster is healthy
+  shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
+  register: etcd_events_cluster_is_healthy
+  ignore_errors: true
+  changed_when: false
+  check_mode: no
+  when: is_etcd_master and etcd_events_cluster_setup
+  tags:
+    - facts
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+
+- include_tasks: refresh_config.yml
+  when: is_etcd_master
+
+- name: Configure | Copy etcd.service systemd file
+  template:
+    src: "etcd-{{ etcd_deployment_type }}.service.j2"
+    dest: /etc/systemd/system/etcd.service
+    backup: yes
+  when: is_etcd_master and etcd_cluster_setup
+
+- name: Configure | Copy etcd-events.service systemd file
+  template:
+    src: "etcd-events-{{ etcd_deployment_type }}.service.j2"
+    dest: /etc/systemd/system/etcd-events.service
+    backup: yes
+  when: is_etcd_master and etcd_events_cluster_setup
+
+- name: Configure | reload systemd
+  command: systemctl daemon-reload
+  when: is_etcd_master
+
+- name: Configure | Ensure etcd is running
+  service:
+    name: etcd
+    state: started
+    enabled: yes
+  when: is_etcd_master and etcd_cluster_setup
+
+- name: Configure | Ensure etcd-events is running
+  service:
+    name: etcd-events
+    state: started
+    enabled: yes
+  when: is_etcd_master and etcd_events_cluster_setup
+
+- name: Configure | Check if etcd cluster is healthy
+  shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
+  register: etcd_cluster_is_healthy
+  until: etcd_cluster_is_healthy.rc == 0
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  ignore_errors: false
+  changed_when: false
+  check_mode: no
+  when: is_etcd_master and etcd_cluster_setup
+  tags:
+    - facts
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+
+- name: Configure | Check if etcd-events cluster is healthy
+  shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
+  register: etcd_events_cluster_is_healthy
+  until: etcd_events_cluster_is_healthy.rc == 0
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  ignore_errors: false
+  changed_when: false
+  check_mode: no
+  when: is_etcd_master and etcd_events_cluster_setup
+  tags:
+    - facts
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+
 - name: Configure | Check if member is in etcd cluster
   shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}"
   register: etcd_member_in_cluster
   ignore_errors: true
   changed_when: false
   check_mode: no
-  when: is_etcd_master
+  when: is_etcd_master and etcd_cluster_setup
   tags:
     - facts
   environment:
@@ -25,44 +118,16 @@
     ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
     ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
 
-- name: Configure | Copy etcd.service systemd file
-  template:
-    src: "etcd-{{ etcd_deployment_type }}.service.j2"
-    dest: /etc/systemd/system/etcd.service
-    backup: yes
-  when: is_etcd_master
-  notify: restart etcd
-
-- name: Configure | Copy etcd-events.service systemd file
-  template:
-    src: "etcd-events-host.service.j2"
-    dest: /etc/systemd/system/etcd-events.service
-    backup: yes
-  when: is_etcd_master and etcd_deployment_type == "host" and etcd_events_cluster_setup
-  notify: restart etcd-events
-
-- name: Configure | Copy etcd-events.service systemd file
-  template:
-    src: "etcd-events-docker.service.j2"
-    dest: /etc/systemd/system/etcd-events.service
-    backup: yes
-  when: is_etcd_master and etcd_deployment_type == "docker" and etcd_events_cluster_setup
-  notify: restart etcd-events
-
 - name: Configure | Join member(s) to etcd cluster one at a time
   include_tasks: join_etcd_member.yml
   vars:
     target_node: "{{ item }}"
-  loop_control:
-    pause: 10
   with_items: "{{ groups['etcd'] }}"
-  when: inventory_hostname == item and etcd_member_in_cluster.rc != 0 and etcd_cluster_is_healthy.rc == 0
+  when: inventory_hostname == item and etcd_cluster_setup and etcd_member_in_cluster.rc != 0 and etcd_cluster_is_healthy.rc == 0
 
 - name: Configure | Join member(s) to etcd-events cluster one at a time
-  include_tasks: join_etcd-evetns_member.yml
+  include_tasks: join_etcd-events_member.yml
   vars:
     target_node: "{{ item }}"
-  loop_control:
-    pause: 10
   with_items: "{{ groups['etcd'] }}"
   when: inventory_hostname == item and etcd_events_cluster_setup and etcd_events_member_in_cluster.rc != 0 and etcd_events_cluster_is_healthy.rc == 0