Add etcd-events cluster for kube-apiserver (#2385)

Add etcd-events cluster for kube-apiserver

roles/etcd/tasks/configure.yml

@@ -1,5 +1,5 @@
 ---
-- name: Configure | Check if member is in cluster
+- name: Configure | Check if member is in etcd cluster
   shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}"
   register: etcd_member_in_cluster
   ignore_errors: true
@@ -12,6 +12,19 @@
     ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
     ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
 
+- name: Configure | Check if member is in etcd-events cluster
+  shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_access_addresses }} member list | grep -q {{ etcd_access_address }}"
+  register: etcd_events_member_in_cluster
+  ignore_errors: true
+  changed_when: false
+  check_mode: no
+  when: is_etcd_master and etcd_events_cluster_setup
+  tags:
+    - facts
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
+
 - name: Configure | Copy etcd.service systemd file
   template:
     src: "etcd-{{ etcd_deployment_type }}.service.j2"
@@ -20,11 +33,36 @@
   when: is_etcd_master
   notify: restart etcd
 
-- name: Configure | Join member(s) to cluster one at a time
-  include_tasks: join_member.yml
+- name: Configure | Copy etcd-events.service systemd file
+  template:
+    src: "etcd-events-host.service.j2"
+    dest: /etc/systemd/system/etcd-events.service
+    backup: yes
+  when: is_etcd_master and etcd_deployment_type == "host" and etcd_events_cluster_setup
+  notify: restart etcd-events
+
+- name: Configure | Copy etcd-events.service systemd file
+  template:
+    src: "etcd-events-docker.service.j2"
+    dest: /etc/systemd/system/etcd-events.service
+    backup: yes
+  when: is_etcd_master and etcd_deployment_type == "docker" and etcd_events_cluster_setup
+  notify: restart etcd-events
+
+- name: Configure | Join member(s) to etcd cluster one at a time
+  include_tasks: join_etcd_member.yml
   vars:
     target_node: "{{ item }}"
   loop_control:
     pause: 10
   with_items: "{{ groups['etcd'] }}"
   when: inventory_hostname == item and etcd_member_in_cluster.rc != 0 and etcd_cluster_is_healthy.rc == 0
+
+- name: Configure | Join member(s) to etcd-events cluster one at a time
+  include_tasks: join_etcd-evetns_member.yml
+  vars:
+    target_node: "{{ item }}"
+  loop_control:
+    pause: 10
+  with_items: "{{ groups['etcd'] }}"
+  when: inventory_hostname == item and etcd_events_cluster_setup and etcd_events_member_in_cluster.rc != 0 and etcd_events_cluster_is_healthy.rc == 0

roles/etcd/tasks/install_docker.yml

@@ -18,3 +18,13 @@
     mode: 0755
     backup: yes
   notify: restart etcd
+
+- name: Install etcd-events launch script
+  template:
+    src: etcd-events.j2
+    dest: "{{ bin_dir }}/etcd-events"
+    owner: 'root'
+    mode: 0755
+    backup: yes
+  when: etcd_events_cluster_setup
+  notify: restart etcd-events

roles/etcd/tasks/join_etcd-events_member.yml

@@ -0,0 +1,47 @@
+---
+- name: Join Member | Add member to cluster
+  shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} member add {{ etcd_member_name }} {{ etcd_events_peer_url }}"
+  register: member_add_result
+  until: member_add_result.rc == 0
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  when: target_node == inventory_hostname
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
+
+- include_tasks: refresh_config.yml
+  vars:
+    etcd_events_peer_addresses: >-
+      {% for host in groups['etcd'] -%}
+        {%- if hostvars[host]['etcd_events_member_in_cluster'].rc == 0 -%}
+          {{ "etcd"+loop.index|string }}=https://{{ hostvars[host].access_ip | default(hostvars[host].ip | default(hostvars[host].ansible_default_ipv4['address'])) }}:2382,
+        {%- endif -%}
+        {%- if loop.last -%}
+          {{ etcd_member_name }}={{ etcd_events_peer_url }}
+        {%- endif -%}
+      {%- endfor -%}
+  when: target_node == inventory_hostname
+
+- name: Join Member | reload systemd
+  command: systemctl daemon-reload
+  when: target_node == inventory_hostname
+
+- name: Join Member | Ensure etcd-events is running
+  service:
+    name: etcd-events
+    state: started
+    enabled: yes
+  when: target_node == inventory_hostname
+
+- name: Join Member | Ensure member is in etcd-events cluster
+  shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_access_addresses }} member list | grep -q {{ etcd_events_access_address }}"
+  register: etcd_events_member_in_cluster
+  changed_when: false
+  check_mode: no
+  tags:
+    - facts
+  when: target_node == inventory_hostname
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"

roles/etcd/tasks/join_etcd_member.yml

@@ -0,0 +1,47 @@
+---
+- name: Join Member | Add member to cluster
+  shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} member add {{ etcd_member_name }} {{ etcd_peer_url }}"
+  register: member_add_result
+  until: member_add_result.rc == 0
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  when: target_node == inventory_hostname
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
+
+- include_tasks: refresh_config.yml
+  vars:
+    etcd_peer_addresses: >-
+      {% for host in groups['etcd'] -%}
+        {%- if hostvars[host]['etcd_member_in_cluster'].rc == 0 -%}
+          {{ "etcd"+loop.index|string }}=https://{{ hostvars[host].access_ip | default(hostvars[host].ip | default(hostvars[host].ansible_default_ipv4['address'])) }}:2380,
+        {%- endif -%}
+        {%- if loop.last -%}
+          {{ etcd_member_name }}={{ etcd_peer_url }}
+        {%- endif -%}
+      {%- endfor -%}
+  when: target_node == inventory_hostname
+
+- name: Join Member | reload systemd
+  command: systemctl daemon-reload
+  when: target_node == inventory_hostname
+
+- name: Join Member | Ensure etcd is running
+  service:
+    name: etcd
+    state: started
+    enabled: yes
+  when: target_node == inventory_hostname
+
+- name: Join Member | Ensure member is in cluster
+  shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}"
+  register: etcd_member_in_cluster
+  changed_when: false
+  check_mode: no
+  tags:
+    - facts
+  when: target_node == inventory_hostname
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"

roles/etcd/tasks/main.yml

@@ -43,6 +43,11 @@
   notify: restart etcd
   when: is_etcd_master and etcd_secret_changed|default(false)
 
+- name: Restart etcd-events if certs changed
+  command: /bin/true
+  notify: restart etcd
+  when: is_etcd_master and etcd_events_cluster_setup and etcd_secret_changed|default(false)
+
 # reload-systemd
 - meta: flush_handlers
 
@@ -53,6 +58,13 @@
     enabled: yes
   when: is_etcd_master and etcd_cluster_setup
 
+- name: Ensure etcd-events is running
+  service:
+    name: etcd-events
+    state: started
+    enabled: yes
+  when: is_etcd_master and etcd_events_cluster_setup
+
 # After etcd cluster is assembled, make sure that
 # initial state of the cluster is in `existing`
 # state insted of `new`.

roles/etcd/tasks/refresh_config.yml

@@ -5,3 +5,10 @@
     dest: /etc/etcd.env
   notify: restart etcd
   when: is_etcd_master
+
+- name: Refresh config | Create etcd-events config file
+  template:
+    src: etcd-events.env.j2
+    dest: /etc/etcd-events.env
+  notify: restart etcd-events
+  when: is_etcd_master and etcd_events_cluster_setup

roles/etcd/tasks/set_cluster_health.yml

@@ -1,5 +1,5 @@
 ---
-- name: Configure | Check if cluster is healthy
+- name: Configure | Check if etcd cluster is healthy
   shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
   register: etcd_cluster_is_healthy
   ignore_errors: true
@@ -11,3 +11,16 @@
   environment:
     ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
     ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
+
+- name: Configure | Check if etcd-events cluster is healthy
+  shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
+  register: etcd_events_cluster_is_healthy
+  ignore_errors: true
+  changed_when: false
+  check_mode: no
+  when: is_etcd_master and etcd_events_cluster_setup
+  tags:
+    - facts
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"