Add etcd-events cluster for kube-apiserver (#2385)

Add etcd-events cluster for kube-apiserver

roles/etcd/defaults/main.yml

@@ -4,6 +4,7 @@ etcd_cluster_setup: true
 
 etcd_backup_prefix: "/var/backups"
 etcd_data_dir: "/var/lib/etcd"
+etcd_events_data_dir: "/var/lib/etcd-events"
 
 etcd_config_dir: /etc/ssl/etcd
 etcd_cert_dir: "{{ etcd_config_dir }}/ssl"

roles/etcd/handlers/main.yml

@@ -7,17 +7,33 @@
     - reload etcd
     - wait for etcd up
 
+- name: restart etcd-events
+  command: /bin/true
+  notify:
+    - etcd-events | reload systemd
+    - reload etcd-events
+    - wait for etcd-events up
+
 - import_tasks: backup.yml
 
 - name: etcd | reload systemd
   command: systemctl daemon-reload
 
+- name: etcd-events | reload systemd
+  command: systemctl daemon-reload
+
 - name: reload etcd
   service:
     name: etcd
     state: restarted
   when: is_etcd_master
 
+- name: reload etcd-events
+  service:
+    name: etcd-events
+    state: restarted
+  when: is_etcd_master
+
 - name: wait for etcd up
   uri:
     url: "https://{% if is_etcd_master %}{{ etcd_address }}{% else %}127.0.0.1{% endif %}:2379/health"
@@ -29,6 +45,17 @@
   retries: 10
   delay: 5
 
+- name: wait for etcd-events up
+  uri:
+    url: "https://{% if is_etcd_master %}{{ etcd_address }}{% else %}127.0.0.1{% endif %}:2381/health"
+    validate_certs: no
+    client_cert: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem"
+    client_key: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem"
+  register: result
+  until: result.status is defined and result.status == 200
+  retries: 10
+  delay: 5
+
 - name: set etcd_secret_changed
   set_fact:
     etcd_secret_changed: true

roles/etcd/tasks/configure.yml

@@ -1,5 +1,5 @@
 ---
-- name: Configure | Check if member is in cluster
+- name: Configure | Check if member is in etcd cluster
   shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}"
   register: etcd_member_in_cluster
   ignore_errors: true
@@ -12,6 +12,19 @@
     ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
     ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
 
+- name: Configure | Check if member is in etcd-events cluster
+  shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_access_addresses }} member list | grep -q {{ etcd_access_address }}"
+  register: etcd_events_member_in_cluster
+  ignore_errors: true
+  changed_when: false
+  check_mode: no
+  when: is_etcd_master and etcd_events_cluster_setup
+  tags:
+    - facts
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
+
 - name: Configure | Copy etcd.service systemd file
   template:
     src: "etcd-{{ etcd_deployment_type }}.service.j2"
@@ -20,11 +33,36 @@
   when: is_etcd_master
   notify: restart etcd
 
-- name: Configure | Join member(s) to cluster one at a time
-  include_tasks: join_member.yml
+- name: Configure | Copy etcd-events.service systemd file
+  template:
+    src: "etcd-events-host.service.j2"
+    dest: /etc/systemd/system/etcd-events.service
+    backup: yes
+  when: is_etcd_master and etcd_deployment_type == "host" and etcd_events_cluster_setup
+  notify: restart etcd-events
+
+- name: Configure | Copy etcd-events.service systemd file
+  template:
+    src: "etcd-events-docker.service.j2"
+    dest: /etc/systemd/system/etcd-events.service
+    backup: yes
+  when: is_etcd_master and etcd_deployment_type == "docker" and etcd_events_cluster_setup
+  notify: restart etcd-events
+
+- name: Configure | Join member(s) to etcd cluster one at a time
+  include_tasks: join_etcd_member.yml
   vars:
     target_node: "{{ item }}"
   loop_control:
     pause: 10
   with_items: "{{ groups['etcd'] }}"
   when: inventory_hostname == item and etcd_member_in_cluster.rc != 0 and etcd_cluster_is_healthy.rc == 0
+
+- name: Configure | Join member(s) to etcd-events cluster one at a time
+  include_tasks: join_etcd-evetns_member.yml
+  vars:
+    target_node: "{{ item }}"
+  loop_control:
+    pause: 10
+  with_items: "{{ groups['etcd'] }}"
+  when: inventory_hostname == item and etcd_events_cluster_setup and etcd_events_member_in_cluster.rc != 0 and etcd_events_cluster_is_healthy.rc == 0

roles/etcd/tasks/install_docker.yml

@@ -18,3 +18,13 @@
     mode: 0755
     backup: yes
   notify: restart etcd
+
+- name: Install etcd-events launch script
+  template:
+    src: etcd-events.j2
+    dest: "{{ bin_dir }}/etcd-events"
+    owner: 'root'
+    mode: 0755
+    backup: yes
+  when: etcd_events_cluster_setup
+  notify: restart etcd-events

roles/etcd/tasks/join_etcd-events_member.yml

@@ -0,0 +1,47 @@
+---
+- name: Join Member | Add member to cluster
+  shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} member add {{ etcd_member_name }} {{ etcd_events_peer_url }}"
+  register: member_add_result
+  until: member_add_result.rc == 0
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  when: target_node == inventory_hostname
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
+
+- include_tasks: refresh_config.yml
+  vars:
+    etcd_events_peer_addresses: >-
+      {% for host in groups['etcd'] -%}
+        {%- if hostvars[host]['etcd_events_member_in_cluster'].rc == 0 -%}
+          {{ "etcd"+loop.index|string }}=https://{{ hostvars[host].access_ip | default(hostvars[host].ip | default(hostvars[host].ansible_default_ipv4['address'])) }}:2382,
+        {%- endif -%}
+        {%- if loop.last -%}
+          {{ etcd_member_name }}={{ etcd_events_peer_url }}
+        {%- endif -%}
+      {%- endfor -%}
+  when: target_node == inventory_hostname
+
+- name: Join Member | reload systemd
+  command: systemctl daemon-reload
+  when: target_node == inventory_hostname
+
+- name: Join Member | Ensure etcd-events is running
+  service:
+    name: etcd-events
+    state: started
+    enabled: yes
+  when: target_node == inventory_hostname
+
+- name: Join Member | Ensure member is in etcd-events cluster
+  shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_access_addresses }} member list | grep -q {{ etcd_events_access_address }}"
+  register: etcd_events_member_in_cluster
+  changed_when: false
+  check_mode: no
+  tags:
+    - facts
+  when: target_node == inventory_hostname
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"

roles/etcd/tasks/join_etcd_member.yml

@@ -0,0 +1,47 @@
+---
+- name: Join Member | Add member to cluster
+  shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} member add {{ etcd_member_name }} {{ etcd_peer_url }}"
+  register: member_add_result
+  until: member_add_result.rc == 0
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  when: target_node == inventory_hostname
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
+
+- include_tasks: refresh_config.yml
+  vars:
+    etcd_peer_addresses: >-
+      {% for host in groups['etcd'] -%}
+        {%- if hostvars[host]['etcd_member_in_cluster'].rc == 0 -%}
+          {{ "etcd"+loop.index|string }}=https://{{ hostvars[host].access_ip | default(hostvars[host].ip | default(hostvars[host].ansible_default_ipv4['address'])) }}:2380,
+        {%- endif -%}
+        {%- if loop.last -%}
+          {{ etcd_member_name }}={{ etcd_peer_url }}
+        {%- endif -%}
+      {%- endfor -%}
+  when: target_node == inventory_hostname
+
+- name: Join Member | reload systemd
+  command: systemctl daemon-reload
+  when: target_node == inventory_hostname
+
+- name: Join Member | Ensure etcd is running
+  service:
+    name: etcd
+    state: started
+    enabled: yes
+  when: target_node == inventory_hostname
+
+- name: Join Member | Ensure member is in cluster
+  shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}"
+  register: etcd_member_in_cluster
+  changed_when: false
+  check_mode: no
+  tags:
+    - facts
+  when: target_node == inventory_hostname
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"

roles/etcd/tasks/main.yml

@@ -43,6 +43,11 @@
   notify: restart etcd
   when: is_etcd_master and etcd_secret_changed|default(false)
 
+- name: Restart etcd-events if certs changed
+  command: /bin/true
+  notify: restart etcd
+  when: is_etcd_master and etcd_events_cluster_setup and etcd_secret_changed|default(false)
+
 # reload-systemd
 - meta: flush_handlers
 
@@ -53,6 +58,13 @@
     enabled: yes
   when: is_etcd_master and etcd_cluster_setup
 
+- name: Ensure etcd-events is running
+  service:
+    name: etcd-events
+    state: started
+    enabled: yes
+  when: is_etcd_master and etcd_events_cluster_setup
+
 # After etcd cluster is assembled, make sure that
 # initial state of the cluster is in `existing`
 # state insted of `new`.

roles/etcd/tasks/refresh_config.yml

@@ -5,3 +5,10 @@
     dest: /etc/etcd.env
   notify: restart etcd
   when: is_etcd_master
+
+- name: Refresh config | Create etcd-events config file
+  template:
+    src: etcd-events.env.j2
+    dest: /etc/etcd-events.env
+  notify: restart etcd-events
+  when: is_etcd_master and etcd_events_cluster_setup

roles/etcd/tasks/set_cluster_health.yml

@@ -1,5 +1,5 @@
 ---
-- name: Configure | Check if cluster is healthy
+- name: Configure | Check if etcd cluster is healthy
   shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
   register: etcd_cluster_is_healthy
   ignore_errors: true
@@ -11,3 +11,16 @@
   environment:
     ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
     ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
+
+- name: Configure | Check if etcd-events cluster is healthy
+  shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
+  register: etcd_events_cluster_is_healthy
+  ignore_errors: true
+  changed_when: false
+  check_mode: no
+  when: is_etcd_master and etcd_events_cluster_setup
+  tags:
+    - facts
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"

roles/etcd/templates/etcd-events-docker.service.j2

@@ -0,0 +1,18 @@
+[Unit]
+Description=etcd docker wrapper
+Wants=docker.socket
+After=docker.service
+
+[Service]
+User=root
+PermissionsStartOnly=true
+EnvironmentFile=-/etc/etcd-events.env
+ExecStart={{ bin_dir }}/etcd-events
+ExecStartPre=-{{ docker_bin_dir }}/docker rm -f {{ etcd_member_name }}-events
+ExecStop={{ docker_bin_dir }}/docker stop {{ etcd_member_name }}-events
+Restart=always
+RestartSec=15s
+TimeoutStartSec=30s
+
+[Install]
+WantedBy=multi-user.target

roles/etcd/templates/etcd-events-host.service.j2

@@ -0,0 +1,16 @@
+[Unit]
+Description=etcd
+After=network.target
+
+[Service]
+Type=notify
+User=root
+EnvironmentFile=/etc/etcd-events.env
+ExecStart={{ bin_dir }}/etcd
+NotifyAccess=all
+Restart=always
+RestartSec=10s
+LimitNOFILE=40000
+
+[Install]
+WantedBy=multi-user.target

roles/etcd/templates/etcd-events.env.j2

@@ -0,0 +1,26 @@
+ETCD_DATA_DIR={{ etcd_events_data_dir }}
+ETCD_ADVERTISE_CLIENT_URLS={{ etcd_events_client_url }}
+ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_events_peer_url }}
+ETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc != 0 | bool %}new{% else %}existing{% endif %}
+
+ETCD_METRICS={{ etcd_metrics }}
+ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address }}:2381,https://127.0.0.1:2381
+ETCD_ELECTION_TIMEOUT={{ etcd_election_timeout }}
+ETCD_HEARTBEAT_INTERVAL={{ etcd_heartbeat_interval }}
+ETCD_INITIAL_CLUSTER_TOKEN=k8s_events_etcd
+ETCD_LISTEN_PEER_URLS=https://{{ etcd_address }}:2382
+ETCD_NAME={{ etcd_member_name }}-events
+ETCD_PROXY=off
+ETCD_INITIAL_CLUSTER={{ etcd_events_peer_addresses }}
+ETCD_AUTO_COMPACTION_RETENTION={{ etcd_compaction_retention }}
+
+# TLS settings
+ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
+ETCD_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem
+ETCD_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem
+ETCD_CLIENT_CERT_AUTH={{ etcd_secure_client | lower}}
+
+ETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
+ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem
+ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem
+ETCD_PEER_CLIENT_CERT_AUTH={{ etcd_peer_client_auth }}

roles/etcd/templates/etcd-events.j2

@@ -0,0 +1,22 @@
+#!/bin/bash
+{{ docker_bin_dir }}/docker run \
+  --restart=on-failure:5 \
+  --env-file=/etc/etcd-events.env \
+  --net=host \
+  -v /etc/ssl/certs:/etc/ssl/certs:ro \
+  -v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \
+  -v {{ etcd_events_data_dir }}:{{ etcd_events_data_dir }}:rw \
+  {% if etcd_memory_limit is defined %}
+  --memory={{ etcd_memory_limit|regex_replace('Mi', 'M') }} \
+  {% endif %}
+  --oom-kill-disable \
+  {% if etcd_cpu_limit is defined %}
+  --cpu-shares={{ etcd_cpu_limit|regex_replace('m', '') }} \
+  {% endif %}
+  {% if etcd_blkio_weight is defined %}
+  --blkio-weight={{ etcd_blkio_weight }} \
+  {% endif %}
+  --name={{ etcd_member_name }}-events \
+  {{ etcd_image_repo }}:{{ etcd_image_tag }} \
+  /usr/local/bin/etcd \
+  "$@"