Add etcd TLS support

2026-02-14 01:34:51 -03:30 · 2016-11-09 13:44:41 +03:00
parent 95b460ae94
commit a32cd85eb7
25 changed files with 408 additions and 35 deletions
--- a/roles/etcd/defaults/main.yml
+++ b/roles/etcd/defaults/main.yml
@@ -1,2 +1,8 @@
 ---
 etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/"
+
+etcd_config_dir: /etc/ssl/etcd
+etcd_cert_dir: "{{ etcd_config_dir }}/ssl"
+etcd_cert_group: root
+
+etcd_script_dir: "{{ bin_dir }}/etcd-scripts"
--- a/roles/etcd/files/make-ssl-etcd.sh
+++ b/roles/etcd/files/make-ssl-etcd.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# Author: Smana smainklh@gmail.com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o pipefail
+
+usage()
+{
+    cat << EOF
+Create self signed certificates
+
+Usage : $(basename $0) -f <config> [-d <ssldir>]
+      -h | --help         : Show this message
+      -f | --config       : Openssl configuration file
+      -d | --ssldir       : Directory where the certificates will be installed
+
+               ex :
+               $(basename $0) -f openssl.conf -d /srv/ssl
+EOF
+}
+
+# Options parsing
+while (($#)); do
+    case "$1" in
+        -h | --help)   usage;   exit 0;;
+        -f | --config) CONFIG=${2}; shift 2;;
+        -d | --ssldir) SSLDIR="${2}"; shift 2;;
+        *)
+            usage
+            echo "ERROR : Unknown option"
+            exit 3
+        ;;
+    esac
+done
+
+if [ -z ${CONFIG} ]; then
+    echo "ERROR: the openssl configuration file is missing. option -f"
+    exit 1
+fi
+if [ -z ${SSLDIR} ]; then
+    SSLDIR="/etc/ssl/etcd"
+fi
+
+tmpdir=$(mktemp -d /tmp/etcd_cacert.XXXXXX)
+trap 'rm -rf "${tmpdir}"' EXIT
+cd "${tmpdir}"
+
+mkdir -p "${SSLDIR}"
+
+# Root CA
+openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1
+openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=etcd-ca" > /dev/null 2>&1
+
+# ETCD member
+openssl genrsa -out member-key.pem 2048 > /dev/null 2>&1
+openssl req -new -key member-key.pem -out member.csr -subj "/CN=etcd-member" -config ${CONFIG} > /dev/null 2>&1
+openssl x509 -req -in member.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member.pem -days 365 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
+
+# Nodes and Admin
+for i in node admin; do
+    openssl genrsa -out ${i}-key.pem 2048 > /dev/null 2>&1
+    openssl req -new -key ${i}-key.pem -out ${i}.csr -subj "/CN=kube-${i}" > /dev/null 2>&1
+    openssl x509 -req -in ${i}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}.pem -days 365 -extensions ssl_client  -extfile ${CONFIG} > /dev/null 2>&1
+done
+
+# Install certs
+mv *.pem ${SSLDIR}/
--- a/roles/etcd/handlers/main.yml
+++ b/roles/etcd/handlers/main.yml
@@ -11,7 +11,7 @@
  when: ansible_service_mgr == "systemd"

 - name: wait for etcd up
-  uri: url="http://{% if is_etcd_master %}{{ etcd_address }}{% else %}127.0.0.1{% endif %}:2379/health" 
+  uri: url="https://{% if is_etcd_master %}{{ etcd_address }}{% else %}127.0.0.1{% endif %}:2379/health" validate_certs=no
  register: result
  until: result.status is defined and result.status == 200
  retries: 10
@@ -22,3 +22,8 @@
    name: etcd
    state: restarted
  when: is_etcd_master
+
+- name: set etcd_secret_changed
+  set_fact:
+    etcd_secret_changed: true
+
--- a/roles/etcd/tasks/check_certs.yml
+++ b/roles/etcd/tasks/check_certs.yml
@@ -0,0 +1,36 @@
+---
+- name: "Check_certs | check if the certs have already been generated on first master"
+  stat:
+    path: "{{ etcd_cert_dir }}/ca.pem"
+  delegate_to: "{{groups['etcd'][0]}}"
+  register: etcdcert_master
+  run_once: true
+
+- name: "Check_certs | Set default value for 'sync_certs' and 'gen_certs' to false"
+  set_fact:
+    sync_certs: false
+    gen_certs: false
+
+- name: "Check_certs | Set 'sync_certs' and 'gen_certs' to true"
+  set_fact:
+    gen_certs: true
+  when: not etcdcert_master.stat.exists
+  run_once: true
+
+- name: "Check certs | check if a cert already exists"
+  stat:
+    path: "{{ etcd_cert_dir }}/ca.pem"
+  register: etcdcert
+
+- name: "Check_certs | Set 'sync_certs' to true"
+  set_fact:
+    sync_certs: true
+  when: >-
+      {%- set certs = {'sync': False} -%}
+      {%- for server in play_hosts
+         if (not hostvars[server].etcdcert.stat.exists|default(False)) or
+         (hostvars[server].etcdcert.stat.checksum|default('') != etcdcert_master.stat.checksum|default('')) -%}
+         {%- set _ = certs.update({'sync': True}) -%}
+      {%- endfor -%}
+      {{ certs.sync }}
+  run_once: true
--- a/roles/etcd/tasks/gen_certs.yml
+++ b/roles/etcd/tasks/gen_certs.yml
@@ -0,0 +1,111 @@
+---
+
+- name: Gen_certs | create etcd script dir
+  file:
+    path: "{{ etcd_script_dir }}"
+    state: directory
+    owner: root
+  when: inventory_hostname == groups['etcd'][0]
+
+- name: Gen_certs | create etcd cert dir
+  file:
+    path={{ etcd_cert_dir }}
+    group={{ etcd_cert_group }}
+    state=directory
+    owner=root
+    recurse=yes
+
+- name: Gen_certs | write openssl config
+  template:
+    src: "openssl.conf.j2"
+    dest: "{{ etcd_config_dir }}/openssl.conf"
+  run_once: yes
+  delegate_to: "{{groups['etcd'][0]}}"
+  when: gen_certs|default(false)
+
+- name: Gen_certs | copy certs generation script
+  copy:
+    src: "make-ssl-etcd.sh"
+    dest: "{{ etcd_script_dir }}/make-ssl-etcd.sh"
+    mode: 0700
+  run_once: yes
+  delegate_to: "{{groups['etcd'][0]}}"
+  when: gen_certs|default(false)
+
+- name: Gen_certs | run cert generation script
+  command: "{{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}"
+  run_once: yes
+  delegate_to: "{{groups['etcd'][0]}}"
+  when: gen_certs|default(false)
+  notify: set etcd_secret_changed
+
+- set_fact:
+    master_certs: ['ca-key.pem', 'admin.pem', 'admin-key.pem', 'member.pem', 'member-key.pem']
+    node_certs: ['ca.pem', 'node.pem', 'node-key.pem']
+
+- name: Gen_certs | Gather etcd master certs
+  shell: "tar cfz - -C {{ etcd_cert_dir }} {{ master_certs|join(' ') }} {{ node_certs|join(' ') }}| base64 --wrap=0"
+  register: etcd_master_cert_data
+  delegate_to: "{{groups['etcd'][0]}}"
+  run_once: true
+  when: sync_certs|default(false)
+  notify: set etcd_secret_changed
+
+- name: Gen_certs | Gather etcd node certs
+  shell: "tar cfz - -C {{ etcd_cert_dir }} {{ node_certs|join(' ') }} | base64 --wrap=0"
+  register: etcd_node_cert_data
+  delegate_to: "{{groups['etcd'][0]}}"
+  run_once: true
+  when: sync_certs|default(false)
+  notify: set etcd_secret_changed
+
+- name: Gen_certs | Copy certs on masters
+  shell: "echo '{{etcd_master_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ etcd_cert_dir }}"
+  changed_when: false
+  when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and
+        inventory_hostname != groups['etcd'][0]
+
+- name: Gen_certs | Copy certs on nodes
+  shell: "echo '{{etcd_node_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ etcd_cert_dir }}"
+  changed_when: false
+  when: inventory_hostname in groups['k8s-cluster'] and sync_certs|default(false) and
+        inventory_hostname not in groups['etcd']
+
+- name: Gen_certs | check certificate permissions
+  file:
+    path={{ etcd_cert_dir }}
+    group={{ etcd_cert_group }}
+    state=directory
+    owner=kube
+    recurse=yes
+
+- name: Gen_certs | set permissions on keys
+  shell: chmod 0600 {{ etcd_cert_dir}}/*key.pem
+  when: inventory_hostname in groups['etcd']
+  changed_when: false
+
+- name: Gen_certs | target ca-certificates directory
+  set_fact:
+    ca_cert_dir: |-
+      {% if ansible_os_family == "Debian" -%}
+      /usr/local/share/ca-certificates
+      {%- elif ansible_os_family == "RedHat" -%}
+      /etc/pki/ca-trust/source/anchors
+      {%- elif ansible_os_family == "CoreOS" -%}
+      /etc/ssl/certs
+      {%- endif %}
+
+- name: Gen_certs | add CA to trusted CA dir
+  copy:
+    src: "{{ etcd_cert_dir }}/ca.pem"
+    dest: "{{ ca_cert_dir }}/etcd-ca.crt"
+    remote_src: true
+  register: etcd_ca_cert
+
+- name: Gen_certs | update ca-certificates (Debian/Ubuntu/CoreOS)
+  command: update-ca-certificates
+  when: etcd_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS"]
+
+- name: Gen_certs | update ca-certificatesa (RedHat)
+  command: update-ca-trust extract
+  when: etcd_ca_cert.changed and ansible_os_family == "RedHat"
--- a/roles/etcd/tasks/main.yml
+++ b/roles/etcd/tasks/main.yml
@@ -1,5 +1,7 @@
 ---
 - include: pre_upgrade.yml
+- include: check_certs.yml
+- include: gen_certs.yml
 - include: install.yml
  when: is_etcd_master
 - include: set_cluster_health.yml
--- a/roles/etcd/tasks/pre_upgrade.yml
+++ b/roles/etcd/tasks/pre_upgrade.yml
@@ -0,0 +1,34 @@
+- name: "Pre-upgrade | check for etcd-proxy unit file"
+  stat:
+    path: /etc/systemd/system/etcd-proxy.service
+  register: kube_apiserver_service_file
+
+- name: "Pre-upgrade | check for etcd-proxy init script"
+  stat:
+    path: /etc/init.d/etcd-proxy
+  register: kube_apiserver_init_script
+
+- name: "Pre-upgrade | stop etcd-proxy if service defined"
+  service:
+    name: etcd-proxy
+    state: stopped
+  when: (kube_apiserver_service_file.stat.exists|default(False) or kube_apiserver_init_script.stat.exists|default(False))
+
+- name: "Pre-upgrade | remove etcd-proxy service definition"
+  file:
+    path: "{{ item }}"
+    state: absent
+  when: (kube_apiserver_service_file.stat.exists|default(False) or kube_apiserver_init_script.stat.exists|default(False))
+  with_items:
+    - /etc/systemd/system/etcd-proxy.service
+    - /etc/init.d/etcd-proxy
+
+- name: "Pre-upgrade | find etcd-proxy container"
+  command: docker ps -aq --filter "name=etcd-proxy*"
+  register: etcd_proxy_container
+  ignore_errors: true
+
+- name: "Pre-upgrade | remove etcd-proxy if it exists"
+  command: "docker rm -f {{item}}"
+  with_items: "{{etcd_proxy_container.stdout_lines}}"
+
--- a/roles/etcd/templates/deb-etcd-docker.initd.j2
+++ b/roles/etcd/templates/deb-etcd-docker.initd.j2
@@ -19,8 +19,9 @@ DAEMON={{ docker_bin_dir | default("/usr/bin") }}/docker
 DAEMON_EXEC=`basename $DAEMON`
 DAEMON_ARGS="run --restart=always --env-file=/etc/etcd.env \
 --net=host \
-v /usr/share/ca-certificates/:/etc/ssl/certs:ro \
+-v /etc/ssl/certs:/etc/ssl/certs:ro \
 -v /var/lib/etcd:/var/lib/etcd:rw \
+-v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \
 --name={{ etcd_member_name | default("etcd") }} \
 {{ etcd_image_repo }}:{{ etcd_image_tag }} \
 {% if etcd_after_v3 %}
--- a/roles/etcd/templates/etcd-docker.service.j2
+++ b/roles/etcd/templates/etcd-docker.service.j2
@@ -11,7 +11,8 @@ ExecStart={{ docker_bin_dir | default("/usr/bin") }}/docker run --restart=always
 {# TODO(mattymo): Allow docker IP binding and disable in envfile
   -p 2380:2380 -p 2379:2379 #}
 --net=host \
-v /usr/share/ca-certificates/:/etc/ssl/certs:ro \
+-v /etc/ssl/certs:/etc/ssl/certs:ro \
+-v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \
 -v /var/lib/etcd:/var/lib/etcd:rw \
 --name={{ etcd_member_name | default("etcd") }} \
 {{ etcd_image_repo }}:{{ etcd_image_tag }} \
--- a/roles/etcd/templates/etcd.j2
+++ b/roles/etcd/templates/etcd.j2
@@ -3,10 +3,19 @@ ETCD_ADVERTISE_CLIENT_URLS={{ etcd_client_url }}
 ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_peer_url }}
 ETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc != 0 | bool %}new{% else %}existing{% endif %}

-ETCD_LISTEN_CLIENT_URLS=http://{{ etcd_address }}:2379,http://127.0.0.1:2379
+ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address }}:2379,https://127.0.0.1:2379
 ETCD_ELECTION_TIMEOUT=10000
 ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd
-ETCD_LISTEN_PEER_URLS=http://{{ etcd_address }}:2380
+ETCD_LISTEN_PEER_URLS=https://{{ etcd_address }}:2380
 ETCD_NAME={{ etcd_member_name }}
 ETCD_PROXY=off
 ETCD_INITIAL_CLUSTER={{ etcd_peer_addresses }}
+
+# TLS settings
+ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
+ETCD_CERT_FILE={{ etcd_cert_dir }}/node.pem
+ETCD_KEY_FILE={{ etcd_cert_dir }}/node-key.pem
+ETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
+ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member.pem
+ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-key.pem
+ETCD_PEER_CLIENT_CERT_AUTH=true
--- a/roles/etcd/templates/openssl.conf.j2
+++ b/roles/etcd/templates/openssl.conf.j2
@@ -0,0 +1,39 @@
+[req]
+req_extensions = v3_req
+distinguished_name = req_distinguished_name
+
+[req_distinguished_name]
+
+[ v3_req ]
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+
+[ ssl_client ]
+extendedKeyUsage = clientAuth, serverAuth
+basicConstraints = CA:FALSE
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid,issuer
+subjectAltName = @alt_names
+
+[ v3_ca ]
+basicConstraints = CA:TRUE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+authorityKeyIdentifier=keyid:always,issuer
+
+[alt_names]
+DNS.1 = localhost
+{% for host in groups['etcd'] %}
+DNS.{{ 1 + loop.index }} = {{ host }}
+{% endfor %}
+{% if loadbalancer_apiserver is defined  and apiserver_loadbalancer_domain_name is defined %}
+{% set idx =  groups['etcd'] | length | int + 1 %}
+DNS.{{ idx | string }} = {{ apiserver_loadbalancer_domain_name }}
+{% endif %}
+{% for host in groups['etcd'] %}
+IP.{{ 2 * loop.index - 1 }} = {{ hostvars[host]['access_ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+IP.{{ 2 * loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+{% endfor %}
+{% set idx =  groups['etcd'] | length | int * 2 + 1 %}
+IP.{{ idx }} = 127.0.0.1