Updated etcd cert check tasks to detect when new cert gen is required (#7219)

* Added force_etcd_cert_refresh var to maintain existing functionality. Broke out etcd node cert syncing from member and admin cert sync logic. Now first etcd will sync node certs to other etcd members on every run to keep all etcds up to date after adding additional worker nodes to the cluster

* Updated etcd cert check tasks to better detect when new certificates need to be generated

* Move usage of force_etcd_cert_refresh var to gen_certs fact set

* Force etcd cert generation per server if force_etcd_cert_refresh is set to true

* Include gathering of node certs even if k8s-cluster member and in etcd group.

* Removed run_once due to when statement

roles/etcd/tasks/gen_certs_script.yml

@@ -66,11 +66,10 @@
               {% endfor %}"
   run_once: yes
   delegate_to: "{{ groups['etcd'][0] }}"
-  when:
-    - gen_certs|default(false)
+  when: gen_certs|default(false)
   notify: set etcd_secret_changed
 
-- name: Gen_certs | Gather etcd master certs
+- name: Gen_certs | Gather etcd member and admin certs from first etcd node
   slurp:
     src: "{{ item }}"
   register: etcd_master_certs
@@ -83,10 +82,6 @@
         '{{ etcd_cert_dir }}/member-{{ node }}.pem',
         '{{ etcd_cert_dir }}/member-{{ node }}-key.pem',
         {% endfor %}]"
-    - "[{% for node in (groups['k8s-cluster'] + groups['calico-rr']|default([]))|unique %}
-        '{{ etcd_cert_dir }}/node-{{ node }}.pem',
-        '{{ etcd_cert_dir }}/node-{{ node }}-key.pem',
-        {% endfor %}]"
   delegate_to: "{{ groups['etcd'][0] }}"
   when:
     - inventory_hostname in groups['etcd']
@@ -94,7 +89,7 @@
     - inventory_hostname != groups['etcd'][0]
   notify: set etcd_secret_changed
 
-- name: Gen_certs | Write etcd master certs
+- name: Gen_certs | Write etcd member and admin certs to other etcd nodes
   copy:
     dest: "{{ item.item }}"
     content: "{{ item.content | b64decode }}"
@@ -109,6 +104,35 @@
   loop_control:
     label: "{{ item.item }}"
 
+- name: Gen_certs | Gather node certs from first etcd node
+  slurp:
+    src: "{{ item }}"
+  register: etcd_master_node_certs
+  with_items:
+    - "[{% for node in (groups['k8s-cluster'] + groups['calico-rr']|default([]))|unique %}
+        '{{ etcd_cert_dir }}/node-{{ node }}.pem',
+        '{{ etcd_cert_dir }}/node-{{ node }}-key.pem',
+        {% endfor %}]"
+  delegate_to: "{{ groups['etcd'][0] }}"
+  when:
+    - inventory_hostname in groups['etcd']
+    - inventory_hostname != groups['etcd'][0]
+  notify: set etcd_secret_changed
+
+- name: Gen_certs | Write node certs to other etcd nodes
+  copy:
+    dest: "{{ item.item }}"
+    content: "{{ item.content | b64decode }}"
+    group: "{{ etcd_cert_group }}"
+    owner: kube
+    mode: 0640
+  with_items: "{{ etcd_master_node_certs.results }}"
+  when:
+    - inventory_hostname in groups['etcd']
+    - inventory_hostname != groups['etcd'][0]
+  loop_control:
+    label: "{{ item.item }}"
+
 - name: Gen_certs | Set cert names per node
   set_fact:
     my_etcd_node_certs: [ 'ca.pem',