External-Mirrors/kubespray
2
0
Fork 0
mirror of https://github.com/kubernetes-sigs/kubespray.git synced 2026-01-31 17:19:17 -03:30
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #591 from kubernetes-incubator/etcdtls

Browse Source 
Add etcd tls support
This commit is contained in:
Matthew Mosesohn
2016-11-10 12:32:13 +03:00
committed by GitHub
parent 57e467c03c a32cd85eb7
commit 9ea9604b3f
37 changed files with 457 additions and 409 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
roles/download/defaults/main.yml
View File

@@ -10,7 +10,7 @@ kube_version: v1.4.3
etcd_version: v3.0.6
#TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
# after migration to container download
calico_version: v0.22.0
calico_version: v1.0.0-beta
calico_cni_version: v1.4.2
weave_version: v1.6.1
flannel_version: v0.6.2
@@ -39,7 +39,8 @@ flannel_server_helper_image_tag: "{{ flannel_server_helper_version }}"
flannel_image_repo: "quay.io/coreos/flannel"
flannel_image_tag: "{{ flannel_version }}"
calicoctl_image_repo: "calico/ctl"
calicoctl_image_tag: "{{ calico_version }}"
# TODO(mattymo): v1.0.0-beta has different syntax. Needs work to upgrade
calicoctl_image_tag: "v0.22.0"
calico_node_image_repo: "calico/node"
calico_node_image_tag: "{{ calico_version }}"
calico_cni_image_repo: "calico/cni"

6
roles/etcd/defaults/main.yml
View File

@@ -1,2 +1,8 @@
---
etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/"
etcd_config_dir: /etc/ssl/etcd
etcd_cert_dir: "{{ etcd_config_dir }}/ssl"
etcd_cert_group: root
etcd_script_dir: "{{ bin_dir }}/etcd-scripts"

80
roles/etcd/files/make-ssl-etcd.sh Executable file
View File

@@ -0,0 +1,80 @@
#!/bin/bash
# Author: Smana smainklh@gmail.com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o pipefail
usage()
{
cat << EOF
Create self signed certificates
Usage : $(basename $0) -f <config> [-d <ssldir>]
-h | --help : Show this message
-f | --config : Openssl configuration file
-d | --ssldir : Directory where the certificates will be installed
ex :
$(basename $0) -f openssl.conf -d /srv/ssl
EOF
}
# Options parsing
while (($#)); do
case "$1" in
-h | --help) usage; exit 0;;
-f | --config) CONFIG=${2}; shift 2;;
-d | --ssldir) SSLDIR="${2}"; shift 2;;
*)
usage
echo "ERROR : Unknown option"
exit 3
;;
esac
done
if [ -z ${CONFIG} ]; then
echo "ERROR: the openssl configuration file is missing. option -f"
exit 1
fi
if [ -z ${SSLDIR} ]; then
SSLDIR="/etc/ssl/etcd"
fi
tmpdir=$(mktemp -d /tmp/etcd_cacert.XXXXXX)
trap 'rm -rf "${tmpdir}"' EXIT
cd "${tmpdir}"
mkdir -p "${SSLDIR}"
# Root CA
openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1
openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=etcd-ca" > /dev/null 2>&1
# ETCD member
openssl genrsa -out member-key.pem 2048 > /dev/null 2>&1
openssl req -new -key member-key.pem -out member.csr -subj "/CN=etcd-member" -config ${CONFIG} > /dev/null 2>&1
openssl x509 -req -in member.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member.pem -days 365 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
# Nodes and Admin
for i in node admin; do
openssl genrsa -out ${i}-key.pem 2048 > /dev/null 2>&1
openssl req -new -key ${i}-key.pem -out ${i}.csr -subj "/CN=kube-${i}" > /dev/null 2>&1
openssl x509 -req -in ${i}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}.pem -days 365 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
done
# Install certs
mv *.pem ${SSLDIR}/

20
roles/etcd/handlers/main.yml
View File

@@ -6,21 +6,14 @@
- reload etcd
- wait for etcd up
- name: restart etcd-proxy
command: /bin/true
notify:
- etcd | reload systemd
- reload etcd-proxy
- wait for etcd up
- name: etcd | reload systemd
command: systemctl daemon-reload
when: ansible_service_mgr == "systemd"
- name: wait for etcd up
uri: url="http://{% if is_etcd_master %}{{ etcd_address }}{% else %}127.0.0.1{% endif %}:2379/health"
uri: url="https://{% if is_etcd_master %}{{ etcd_address }}{% else %}127.0.0.1{% endif %}:2379/health" validate_certs=no
register: result
until: result.status == 200
until: result.status is defined and result.status == 200
retries: 10
delay: 5
@@ -30,8 +23,7 @@
state: restarted
when: is_etcd_master
- name: reload etcd-proxy
service:
name: etcd-proxy
state: restarted
when: is_etcd_proxy
- name: set etcd_secret_changed
set_fact:
etcd_secret_changed: true

36
roles/etcd/tasks/check_certs.yml Normal file
View File

@@ -0,0 +1,36 @@
---
- name: "Check_certs | check if the certs have already been generated on first master"
stat:
path: "{{ etcd_cert_dir }}/ca.pem"
delegate_to: "{{groups['etcd'][0]}}"
register: etcdcert_master
run_once: true
- name: "Check_certs | Set default value for 'sync_certs' and 'gen_certs' to false"
set_fact:
sync_certs: false
gen_certs: false
- name: "Check_certs | Set 'sync_certs' and 'gen_certs' to true"
set_fact:
gen_certs: true
when: not etcdcert_master.stat.exists
run_once: true
- name: "Check certs | check if a cert already exists"
stat:
path: "{{ etcd_cert_dir }}/ca.pem"
register: etcdcert
- name: "Check_certs | Set 'sync_certs' to true"
set_fact:
sync_certs: true
when: >-
{%- set certs = {'sync': False} -%}
{%- for server in play_hosts
if (not hostvars[server].etcdcert.stat.exists|default(False)) or
(hostvars[server].etcdcert.stat.checksum|default('') != etcdcert_master.stat.checksum|default('')) -%}
{%- set _ = certs.update({'sync': True}) -%}
{%- endfor -%}
{{ certs.sync }}
run_once: true

16
roles/etcd/tasks/configure.yml
View File

@@ -26,19 +26,3 @@
mode: 0755
when: ansible_service_mgr in ["sysvinit","upstart"] and ansible_os_family == "Debian" and is_etcd_master
notify: restart etcd
- name: Configure | Copy etcd-proxy.service systemd file
template:
src: "etcd-proxy-{{ etcd_deployment_type }}.service.j2"
dest: /etc/systemd/system/etcd-proxy.service
backup: yes
when: ansible_service_mgr == "systemd" and is_etcd_proxy
notify: restart etcd-proxy
- name: Configure | Write etcd-proxy initd script
template:
src: "deb-etcd-proxy-{{ etcd_deployment_type }}.initd.j2"
dest: /etc/init.d/etcd-proxy
owner: root
mode: 0755
when: ansible_service_mgr in ["sysvinit","upstart"] and ansible_os_family == "Debian" and is_etcd_proxy
notify: restart etcd-proxy

111
roles/etcd/tasks/gen_certs.yml Normal file
View File

@@ -0,0 +1,111 @@
---
- name: Gen_certs | create etcd script dir
file:
path: "{{ etcd_script_dir }}"
state: directory
owner: root
when: inventory_hostname == groups['etcd'][0]
- name: Gen_certs | create etcd cert dir
file:
path={{ etcd_cert_dir }}
group={{ etcd_cert_group }}
state=directory
owner=root
recurse=yes
- name: Gen_certs | write openssl config
template:
src: "openssl.conf.j2"
dest: "{{ etcd_config_dir }}/openssl.conf"
run_once: yes
delegate_to: "{{groups['etcd'][0]}}"
when: gen_certs|default(false)
- name: Gen_certs | copy certs generation script
copy:
src: "make-ssl-etcd.sh"
dest: "{{ etcd_script_dir }}/make-ssl-etcd.sh"
mode: 0700
run_once: yes
delegate_to: "{{groups['etcd'][0]}}"
when: gen_certs|default(false)
- name: Gen_certs | run cert generation script
command: "{{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}"
run_once: yes
delegate_to: "{{groups['etcd'][0]}}"
when: gen_certs|default(false)
notify: set etcd_secret_changed
- set_fact:
master_certs: ['ca-key.pem', 'admin.pem', 'admin-key.pem', 'member.pem', 'member-key.pem']
node_certs: ['ca.pem', 'node.pem', 'node-key.pem']
- name: Gen_certs | Gather etcd master certs
shell: "tar cfz - -C {{ etcd_cert_dir }} {{ master_certs|join(' ') }} {{ node_certs|join(' ') }}| base64 --wrap=0"
register: etcd_master_cert_data
delegate_to: "{{groups['etcd'][0]}}"
run_once: true
when: sync_certs|default(false)
notify: set etcd_secret_changed
- name: Gen_certs | Gather etcd node certs
shell: "tar cfz - -C {{ etcd_cert_dir }} {{ node_certs|join(' ') }} | base64 --wrap=0"
register: etcd_node_cert_data
delegate_to: "{{groups['etcd'][0]}}"
run_once: true
when: sync_certs|default(false)
notify: set etcd_secret_changed
- name: Gen_certs | Copy certs on masters
shell: "echo '{{etcd_master_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ etcd_cert_dir }}"
changed_when: false
when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and
inventory_hostname != groups['etcd'][0]
- name: Gen_certs | Copy certs on nodes
shell: "echo '{{etcd_node_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ etcd_cert_dir }}"
changed_when: false
when: inventory_hostname in groups['k8s-cluster'] and sync_certs|default(false) and
inventory_hostname not in groups['etcd']
- name: Gen_certs | check certificate permissions
file:
path={{ etcd_cert_dir }}
group={{ etcd_cert_group }}
state=directory
owner=kube
recurse=yes
- name: Gen_certs | set permissions on keys
shell: chmod 0600 {{ etcd_cert_dir}}/*key.pem
when: inventory_hostname in groups['etcd']
changed_when: false
- name: Gen_certs | target ca-certificates directory
set_fact:
ca_cert_dir: |-
{% if ansible_os_family == "Debian" -%}
/usr/local/share/ca-certificates
{%- elif ansible_os_family == "RedHat" -%}
/etc/pki/ca-trust/source/anchors
{%- elif ansible_os_family == "CoreOS" -%}
/etc/ssl/certs
{%- endif %}
- name: Gen_certs | add CA to trusted CA dir
copy:
src: "{{ etcd_cert_dir }}/ca.pem"
dest: "{{ ca_cert_dir }}/etcd-ca.crt"
remote_src: true
register: etcd_ca_cert
- name: Gen_certs | update ca-certificates (Debian/Ubuntu/CoreOS)
command: update-ca-certificates
when: etcd_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS"]
- name: Gen_certs | update ca-certificatesa (RedHat)
command: update-ca-trust extract
when: etcd_ca_cert.changed and ansible_os_family == "RedHat"

21
roles/etcd/tasks/main.yml
View File

@@ -1,8 +1,15 @@
---
- include: pre_upgrade.yml
- include: check_certs.yml
- include: gen_certs.yml
- include: install.yml
when: is_etcd_master
- include: set_cluster_health.yml
when: is_etcd_master
- include: configure.yml
when: is_etcd_master
- include: refresh_config.yml
when: is_etcd_master
- name: Ensure etcd is running
service:
@@ -11,23 +18,11 @@
enabled: yes
when: is_etcd_master
- name: Ensure etcd-proxy is running
service:
name: etcd-proxy
state: started
enabled: yes
when: is_etcd_proxy
- name: Restart etcd if binary changed
command: /bin/true
notify: restart etcd
when: etcd_deployment_type == "host" and etcd_copy.stdout_lines and is_etcd_master
- name: Restart etcd-proxy if binary changed
command: /bin/true
notify: restart etcd-proxy
when: etcd_deployment_type == "host" and etcd_copy.stdout_lines and is_etcd_proxy
# Reload systemd before starting service
- meta: flush_handlers
@@ -35,4 +30,6 @@
# initial state of the cluster is in `existing`
# state insted of `new`.
- include: set_cluster_health.yml
when: is_etcd_master
- include: refresh_config.yml
when: is_etcd_master

34
roles/etcd/tasks/pre_upgrade.yml Normal file
View File

@@ -0,0 +1,34 @@
- name: "Pre-upgrade | check for etcd-proxy unit file"
stat:
path: /etc/systemd/system/etcd-proxy.service
register: kube_apiserver_service_file
- name: "Pre-upgrade | check for etcd-proxy init script"
stat:
path: /etc/init.d/etcd-proxy
register: kube_apiserver_init_script
- name: "Pre-upgrade | stop etcd-proxy if service defined"
service:
name: etcd-proxy
state: stopped
when: (kube_apiserver_service_file.stat.exists|default(False) or kube_apiserver_init_script.stat.exists|default(False))
- name: "Pre-upgrade | remove etcd-proxy service definition"
file:
path: "{{ item }}"
state: absent
when: (kube_apiserver_service_file.stat.exists|default(False) or kube_apiserver_init_script.stat.exists|default(False))
with_items:
- /etc/systemd/system/etcd-proxy.service
- /etc/init.d/etcd-proxy
- name: "Pre-upgrade | find etcd-proxy container"
command: docker ps -aq --filter "name=etcd-proxy*"
register: etcd_proxy_container
ignore_errors: true
- name: "Pre-upgrade | remove etcd-proxy if it exists"
command: "docker rm -f {{item}}"
with_items: "{{etcd_proxy_container.stdout_lines}}"

7
roles/etcd/tasks/refresh_config.yml
View File

@@ -5,10 +5,3 @@
dest: /etc/etcd.env
notify: restart etcd
when: is_etcd_master
- name: Refresh config | Create etcd-proxy config file
template:
src: etcd-proxy.j2
dest: /etc/etcd-proxy.env
notify: restart etcd-proxy
when: is_etcd_proxy

3
roles/etcd/templates/deb-etcd-docker.initd.j2
View File

@@ -19,8 +19,9 @@ DAEMON={{ docker_bin_dir | default("/usr/bin") }}/docker
DAEMON_EXEC=`basename $DAEMON`
DAEMON_ARGS="run --restart=always --env-file=/etc/etcd.env \
--net=host \
-v /usr/share/ca-certificates/:/etc/ssl/certs:ro \
-v /etc/ssl/certs:/etc/ssl/certs:ro \
-v /var/lib/etcd:/var/lib/etcd:rw \
-v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \
--name={{ etcd_member_name | default("etcd") }} \
{{ etcd_image_repo }}:{{ etcd_image_tag }} \
{% if etcd_after_v3 %}

120
roles/etcd/templates/deb-etcd-proxy-docker.initd.j2
View File

@@ -1,120 +0,0 @@
#!/bin/sh
set -a
### BEGIN INIT INFO
# Provides: etcd-proxy
# Required-Start: $local_fs $network $syslog
# Required-Stop:
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: etcd-proxy
# Description:
# etcd-proxy is a proxy for etcd: distributed, consistent key-value store for shared configuration and service discovery
### END INIT INFO
PATH=/sbin:/usr/sbin:/bin/:/usr/bin
DESC="etcd-proxy"
NAME=etcd-proxy
DAEMON={{ docker_bin_dir | default("/usr/bin") }}/docker
DAEMON_EXEC=`basename $DAEMON`
DAEMON_ARGS="run --restart=always --env-file=/etc/etcd-proxy.env \
--net=host \
--stop-signal=SIGKILL \
-v /usr/share/ca-certificates/:/etc/ssl/certs:ro \
--name={{ etcd_proxy_member_name | default("etcd-proxy") }} \
{{ etcd_image_repo }}:{{ etcd_image_tag }} \
{% if etcd_after_v3 %}
{{ etcd_container_bin_dir }}etcd
{% endif %}"
SCRIPTNAME=/etc/init.d/$NAME
DAEMON_USER=root
STOP_SCHEDULE="${STOP_SCHEDULE:-QUIT/5/TERM/5/KILL/5}"
PID=/var/run/etcd-proxy.pid
# Exit if the binary is not present
[ -x "$DAEMON" ] || exit 0
# Define LSB log_* functions.
# Depend on lsb-base (>= 3.2-14) to ensure that this file is present
# and status_of_proc is working.
. /lib/lsb/init-functions
do_status()
{
status_of_proc -p $PID "$DAEMON" "$NAME" && exit 0 || exit $?
}
# Function that starts the daemon/service
#
do_start()
{
{{ docker_bin_dir | default("/usr/bin") }}/docker rm -f {{ etcd_proxy_member_name | default("etcd-proxy") }} &>/dev/null || true
sleep 1
start-stop-daemon --background --start --quiet --make-pidfile --pidfile $PID --user $DAEMON_USER --exec $DAEMON -- \
$DAEMON_ARGS \
|| return 2
}
#
# Function that stops the daemon/service
#
do_stop()
{
start-stop-daemon --stop --quiet --retry=$STOP_SCHEDULE --pidfile $PID --name $DAEMON_EXEC
RETVAL="$?"
sleep 1
return "$RETVAL"
}
case "$1" in
start)
log_daemon_msg "Starting $DESC" "$NAME"
do_start
case "$?" in
0|1) log_end_msg 0 || exit 0 ;;
2) log_end_msg 1 || exit 1 ;;
esac
;;
stop)
log_daemon_msg "Stopping $DESC" "$NAME"
if do_stop; then
log_end_msg 0
else
log_failure_msg "Can't stop etcd-proxy"
log_end_msg 1
fi
;;
status)
if do_status; then
log_end_msg 0
else
log_failure_msg "etcd-proxy is not running"
log_end_msg 1
fi
;;
restart|force-reload)
log_daemon_msg "Restarting $DESC" "$NAME"
if do_stop; then
if do_start; then
log_end_msg 0
exit 0
else
rc="$?"
fi
else
rc="$?"
fi
log_failure_msg "Can't restart etcd-proxy"
log_end_msg ${rc}
;;
*)
echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
exit 3
;;
esac

110
roles/etcd/templates/deb-etcd-proxy-host.initd.j2
View File

@@ -1,110 +0,0 @@
#!/bin/sh
set -a
### BEGIN INIT INFO
# Provides: etcd-proxy
# Required-Start: $local_fs $network $syslog
# Required-Stop:
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: etcd-proxy
# Description:
# etcd-proxy is a proxy for etcd: distributed, consistent key-value store for shared configuration and service discovery
### END INIT INFO
PATH=/sbin:/usr/sbin:/bin:/usr/bin
DESC="etcd-proxy"
NAME=etcd-proxy
DAEMON={{ bin_dir }}/etcd
DAEMON_ARGS=""
SCRIPTNAME=/etc/init.d/$NAME
DAEMON_USER=etcd
STOP_SCHEDULE="${STOP_SCHEDULE:-QUIT/5/TERM/5/KILL/5}"
PID=/var/run/etcd-proxy.pid
# Exit if the binary is not present
[ -x "$DAEMON" ] || exit 0
# Read configuration variable file if it is present
[ -f /etc/etcd-proxy.env ] && . /etc/etcd-proxy.env
# Define LSB log_* functions.
# Depend on lsb-base (>= 3.2-14) to ensure that this file is present
# and status_of_proc is working.
. /lib/lsb/init-functions
do_status()
{
status_of_proc -p $PID "$DAEMON" "$NAME" && exit 0 || exit $?
}
# Function that starts the daemon/service
#
do_start()
{
start-stop-daemon --background --start --quiet --make-pidfile --pidfile $PID --user $DAEMON_USER --exec $DAEMON -- \
$DAEMON_ARGS \
|| return 2
}
#
# Function that stops the daemon/service
#
do_stop()
{
start-stop-daemon --stop --quiet --retry=$STOP_SCHEDULE --pidfile $PID --name $NAME
RETVAL="$?"
sleep 1
return "$RETVAL"
}
case "$1" in
start)
log_daemon_msg "Starting $DESC" "$NAME"
do_start
case "$?" in
0|1) log_end_msg 0 || exit 0 ;;
2) log_end_msg 1 || exit 1 ;;
esac
;;
stop)
log_daemon_msg "Stopping $DESC" "$NAME"
if do_stop; then
log_end_msg 0
else
log_failure_msg "Can't stop etcd-proxy"
log_end_msg 1
fi
;;
status)
if do_status; then
log_end_msg 0
else
log_failure_msg "etcd-proxy is not running"
log_end_msg 1
fi
;;
restart|force-reload)
log_daemon_msg "Restarting $DESC" "$NAME"
if do_stop; then
if do_start; then
log_end_msg 0
exit 0
else
rc="$?"
fi
else
rc="$?"
fi
log_failure_msg "Can't restart etcd-proxy"
log_end_msg ${rc}
;;
*)
echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
exit 3
;;
esac

3
roles/etcd/templates/etcd-docker.service.j2
View File

@@ -11,7 +11,8 @@ ExecStart={{ docker_bin_dir | default("/usr/bin") }}/docker run --restart=always
{# TODO(mattymo): Allow docker IP binding and disable in envfile
-p 2380:2380 -p 2379:2379 #}
--net=host \
-v /usr/share/ca-certificates/:/etc/ssl/certs:ro \
-v /etc/ssl/certs:/etc/ssl/certs:ro \
-v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \
-v /var/lib/etcd:/var/lib/etcd:rw \
--name={{ etcd_member_name | default("etcd") }} \
{{ etcd_image_repo }}:{{ etcd_image_tag }} \

28
roles/etcd/templates/etcd-proxy-docker.service.j2
View File

@@ -1,28 +0,0 @@
[Unit]
Description=etcd-proxy docker wrapper
Wants=docker.socket
After=docker.service
[Service]
User=root
PermissionsStartOnly=true
ExecStart={{ docker_bin_dir | default("/usr/bin") }}/docker run --restart=always \
--env-file=/etc/etcd-proxy.env \
{# TODO(mattymo): Allow docker IP binding and disable in envfile
-p 2380:2380 -p 2379:2379 #}
--net=host \
--stop-signal=SIGKILL \
-v /usr/share/ca-certificates/:/etc/ssl/certs:ro \
--name={{ etcd_proxy_member_name | default("etcd-proxy") }} \
{{ etcd_image_repo }}:{{ etcd_image_tag }} \
{% if etcd_after_v3 %}
{{ etcd_container_bin_dir }}etcd
{% endif %}
ExecStartPre=-{{ docker_bin_dir | default("/usr/bin") }}/docker rm -f {{ etcd_proxy_member_name | default("etcd-proxy") }}
ExecReload={{ docker_bin_dir | default("/usr/bin") }}/docker restart {{ etcd_proxy_member_name | default("etcd-proxy") }}
ExecStop={{ docker_bin_dir | default("/usr/bin") }}/docker stop {{ etcd_proxy_member_name | default("etcd-proxy") }}
Restart=always
RestartSec=15s
[Install]
WantedBy=multi-user.target

19
roles/etcd/templates/etcd-proxy-host.service.j2
View File

@@ -1,19 +0,0 @@
[Unit]
Description=etcd-proxy
After=network.target
[Service]
Type=notify
User=etcd
PermissionsStartOnly=true
EnvironmentFile=/etc/etcd-proxy.env
ExecStart={{ bin_dir }}/etcd
ExecStartPre=/bin/mkdir -p /var/lib/etcd-proxy
ExecStartPre=/bin/chown -R etcd: /var/lib/etcd-proxy
NotifyAccess=all
Restart=always
RestartSec=10s
LimitNOFILE=40000
[Install]
WantedBy=multi-user.target

5
roles/etcd/templates/etcd-proxy.j2
View File

@@ -1,5 +0,0 @@
ETCD_DATA_DIR=/var/lib/etcd-proxy
ETCD_PROXY=on
ETCD_LISTEN_CLIENT_URLS={{ etcd_access_endpoint }}
ETCD_NAME={{ etcd_proxy_member_name | default("etcd-proxy") }}
ETCD_INITIAL_CLUSTER={{ etcd_peer_addresses }}

17
roles/etcd/templates/etcd.j2
View File

@@ -3,14 +3,19 @@ ETCD_ADVERTISE_CLIENT_URLS={{ etcd_client_url }}
ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_peer_url }}
ETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc != 0 | bool %}new{% else %}existing{% endif %}
{% if not is_etcd_proxy %}
ETCD_LISTEN_CLIENT_URLS=http://{{ etcd_address }}:2379,http://127.0.0.1:2379
{% else %}
ETCD_LISTEN_CLIENT_URLS=http://{{ etcd_address }}:2379
{% endif %}
ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address }}:2379,https://127.0.0.1:2379
ETCD_ELECTION_TIMEOUT=10000
ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd
ETCD_LISTEN_PEER_URLS=http://{{ etcd_address }}:2380
ETCD_LISTEN_PEER_URLS=https://{{ etcd_address }}:2380
ETCD_NAME={{ etcd_member_name }}
ETCD_PROXY=off
ETCD_INITIAL_CLUSTER={{ etcd_peer_addresses }}
# TLS settings
ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
ETCD_CERT_FILE={{ etcd_cert_dir }}/node.pem
ETCD_KEY_FILE={{ etcd_cert_dir }}/node-key.pem
ETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member.pem
ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-key.pem
ETCD_PEER_CLIENT_CERT_AUTH=true

39
roles/etcd/templates/openssl.conf.j2 Normal file
View File

@@ -0,0 +1,39 @@
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
[ ssl_client ]
extendedKeyUsage = clientAuth, serverAuth
basicConstraints = CA:FALSE
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid,issuer
subjectAltName = @alt_names
[ v3_ca ]
basicConstraints = CA:TRUE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
authorityKeyIdentifier=keyid:always,issuer
[alt_names]
DNS.1 = localhost
{% for host in groups['etcd'] %}
DNS.{{ 1 + loop.index }} = {{ host }}
{% endfor %}
{% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %}
{% set idx = groups['etcd'] | length | int + 1 %}
DNS.{{ idx | string }} = {{ apiserver_loadbalancer_domain_name }}
{% endif %}
{% for host in groups['etcd'] %}
IP.{{ 2 * loop.index - 1 }} = {{ hostvars[host]['access_ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
IP.{{ 2 * loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
{% endfor %}
{% set idx = groups['etcd'] | length | int * 2 + 1 %}
IP.{{ idx }} = 127.0.0.1

17
roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2
View File

@@ -26,7 +26,13 @@ spec:
image: calico/kube-policy-controller:latest
env:
- name: ETCD_ENDPOINTS
value: "{{ etcd_endpoint }}"
value: "{{ etcd_access_endpoint }}"
- name: ETCD_CA_CERT_FILE
value: "{{ etcd_cert_dir }}/ca.pem"
- name: ETCD_CERT_FILE
value: "{{ etcd_cert_dir }}/node.pem"
- name: ETCD_KEY_FILE
value: "{{ etcd_cert_dir }}/node-key.pem"
# Location of the Kubernetes API - this shouldn't need to be
# changed so long as it is used in conjunction with
# CONFIGURE_ETC_HOSTS="true".
@@ -38,3 +44,12 @@ spec:
# This removes the need for KubeDNS to resolve the Service.
- name: CONFIGURE_ETC_HOSTS
value: "true"
volumeMounts:
- mountPath: {{ etcd_cert_dir }}
name: etcd-certs
readOnly: true
volumes:
- hostPath:
path: {{ etcd_cert_dir }}
name: etcd-certs

6
roles/kubernetes/master/defaults/main.yml
View File

@@ -28,3 +28,9 @@ kube_apiserver_insecure_bind_address: 127.0.0.1
# Logging directory (sysvinit systems)
kube_log_dir: "/var/log/kubernetes"
# ETCD cert dir for connecting apiserver to etcd
etcd_config_dir: /etc/ssl/etcd
etcd_cert_dir: "{{ etcd_config_dir }}/ssl"

9
roles/kubernetes/master/tasks/pre-upgrade.yml
View File

@@ -14,12 +14,3 @@
name: kube-apiserver
state: stopped
when: (kube_apiserver_service_file.stat.exists|default(False) or kube_apiserver_init_script.stat.exists|default(False))
- name: "Pre-upgrade | remove kube-apiserver service definition"
file:
path: "{{ item }}"
state: absent
when: (kube_apiserver_service_file.stat.exists|default(False) or kube_apiserver_init_script.stat.exists|default(False))
with_items:
- /etc/systemd/system/kube-apiserver.service
- /etc/init.d/kube-apiserver

9
roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
View File

@@ -14,6 +14,9 @@ spec:
- --advertise-address={{ ip | default(ansible_default_ipv4.address) }}
- --etcd-servers={{ etcd_access_endpoint }}
- --etcd-quorum-read=true
- --etcd-cafile={{ etcd_cert_dir }}/ca.pem
- --etcd-certfile={{ etcd_cert_dir }}/node.pem
- --etcd-keyfile={{ etcd_cert_dir }}/node-key.pem
- --insecure-bind-address={{ kube_apiserver_insecure_bind_address }}
- --apiserver-count={{ kube_apiserver_count }}
- --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota
@@ -50,6 +53,9 @@ spec:
- mountPath: /etc/ssl/certs
name: ssl-certs-host
readOnly: true
- mountPath: {{ etcd_cert_dir }}
name: etcd-certs
readOnly: true
- mountPath: /var/log/
name: logfile
volumes:
@@ -59,6 +65,9 @@ spec:
- hostPath:
path: /etc/ssl/certs/
name: ssl-certs-host
- hostPath:
path: {{ etcd_cert_dir }}
name: etcd-certs
- hostPath:
path: /var/log/
name: logfile

4
roles/kubernetes/node/templates/cni-calico.conf.j2
View File

@@ -1,6 +1,10 @@
{
"name": "calico-k8s-network",
"type": "calico",
"etcd_endpoints": "{{ etcd_access_endpoint }}",
"etcd_cert_file": "{{ etcd_cert_dir }}/node.pem",
"etcd_key_file": "{{ etcd_cert_dir }}/node-key.pem",
"etcd_ca_cert_file": "{{ etcd_cert_dir }}/ca.pem",
"log_level": "info",
"ipam": {
"type": "calico-ipam"

3
roles/kubernetes/preinstall/defaults/main.yml
View File

@@ -45,3 +45,6 @@ openstack_username: "{{ lookup('env','OS_USERNAME') }}"
openstack_password: "{{ lookup('env','OS_PASSWORD') }}"
openstack_region: "{{ lookup('env','OS_REGION_NAME') }}"
openstack_tenant_id: "{{ lookup('env','OS_TENANT_ID') }}"
# All clients access each node individually, instead of using a load balancer.
etcd_multiaccess: true

17
roles/kubernetes/preinstall/tasks/set_facts.yml
View File

@@ -23,14 +23,14 @@
- set_fact: etcd_address="{{ ip | default(ansible_default_ipv4['address']) }}"
- set_fact: etcd_access_address="{{ access_ip | default(etcd_address) }}"
- set_fact: etcd_peer_url="http://{{ etcd_access_address }}:2380"
- set_fact: etcd_client_url="http://{{ etcd_access_address }}:2379"
- set_fact: etcd_peer_url="https://{{ etcd_access_address }}:2380"
- set_fact: etcd_client_url="https://{{ etcd_access_address }}:2379"
- set_fact: etcd_authority="127.0.0.1:2379"
- set_fact: etcd_endpoint="http://{{ etcd_authority }}"
- set_fact: etcd_endpoint="https://{{ etcd_authority }}"
- set_fact:
etcd_access_addresses: |-
{% for item in groups['etcd'] -%}
http://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2379{% if not loop.last %},{% endif %}
https://{{ item }}:2379{% if not loop.last %},{% endif %}
{%- endfor %}
- set_fact: etcd_access_endpoint="{% if etcd_multiaccess %}{{ etcd_access_addresses }}{% else %}{{ etcd_endpoint }}{% endif %}"
- set_fact:
@@ -41,15 +41,8 @@
- set_fact:
etcd_peer_addresses: |-
{% for item in groups['etcd'] -%}
{{ "etcd"+loop.index|string }}=http://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2380{% if not loop.last %},{% endif %}
{{ "etcd"+loop.index|string }}=https://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2380{% if not loop.last %},{% endif %}
{%- endfor %}
- set_fact:
etcd_proxy_member_name: |-
{% for host in groups['k8s-cluster'] %}
{% if inventory_hostname == host %}{{"etcd-proxy"+loop.index|string }}{% endif %}
{% endfor %}
- set_fact:
is_etcd_proxy: "{{ inventory_hostname in groups['k8s-cluster'] }}"
- set_fact:
is_etcd_master: "{{ inventory_hostname in groups['etcd'] }}"
- set_fact:

3
roles/network_plugin/calico/defaults/main.yml
View File

@@ -8,3 +8,6 @@ ipip: false
# Set to true if you want your calico cni binaries to overwrite the
# ones from hyperkube while leaving other cni plugins intact.
overwrite_hyperkube_cni: true
calico_cert_dir: /etc/calico/certs
etcd_cert_dir: /etc/ssl/etcd/ssl

59
roles/network_plugin/calico/tasks/main.yml
View File

@@ -12,6 +12,24 @@
- meta: flush_handlers
- name: Calico | Create calico certs directory
file:
dest: "{{ calico_cert_dir }}"
state: directory
mode: 0750
owner: root
group: root
- name: Calico | Link etcd certificates for calico-node
file:
src: "{{ etcd_cert_dir }}/{{ item.s }}"
dest: "{{ calico_cert_dir }}/{{ item.d }}"
state: hard
with_items:
- {s: "ca.pem", d: "ca_cert.crt"}
- {s: "node.pem", d: "cert.crt"}
- {s: "node-key.pem", d: "key.pem"}
- name: Calico | Install calicoctl container script
template:
src: calicoctl-container.j2
@@ -41,19 +59,23 @@
when: "{{ overwrite_hyperkube_cni|bool }}"
- name: Calico | wait for etcd
uri: url=http://localhost:2379/health
uri: url=https://localhost:2379/health validate_certs=no
register: result
until: result.status == 200
until: result.status == 200 or result.status == 401
retries: 10
delay: 5
when: inventory_hostname in groups['kube-master']
delegate_to: "{{groups['etcd'][0]}}"
run_once: true
- name: Calico | Check if calico network pool has already been configured
uri:
url: "{{ etcd_endpoint }}/v2/keys/calico/v1/ipam/v4/pool"
return_content: yes
status_code: 200,404
command: |-
curl \
--cacert {{ etcd_cert_dir }}/ca.pem \
--cert {{ etcd_cert_dir}}/admin.pem \
--key {{ etcd_cert_dir }}/admin-key.pem \
https://localhost:2379/v2/keys/calico/v1/ipam/v4/pool
register: calico_conf
delegate_to: "{{groups['etcd'][0]}}"
run_once: true
- name: Calico | Define ipip pool argument
@@ -79,21 +101,29 @@
environment:
NO_DEFAULT_POOLS: true
run_once: true
when: calico_conf.status == 404 or "nodes" not in calico_conf.content
when: '"Key not found" in calico_conf.stdout or "nodes" not in calico_conf.stdout'
- name: Calico | Get calico configuration from etcd
uri:
url: "{{ etcd_endpoint }}/v2/keys/calico/v1/ipam/v4/pool"
return_content: yes
register: calico_pools
command: |-
curl \
--cacert {{ etcd_cert_dir }}/ca.pem \
--cert {{ etcd_cert_dir}}/admin.pem \
--key {{ etcd_cert_dir }}/admin-key.pem \
https://localhost:2379/v2/keys/calico/v1/ipam/v4/pool
register: calico_pools_raw
delegate_to: "{{groups['etcd'][0]}}"
run_once: true
- set_fact:
calico_pools: "{{ calico_pools_raw.stdout | from_json }}"
run_once: true
- name: Calico | Check if calico pool is properly configured
fail:
msg: 'Only one network pool must be configured and it must be the subnet {{ kube_pods_subnet }}.
Please erase calico configuration and run the playbook again ("etcdctl rm --recursive /calico/v1/ipam/v4/pool")'
when: ( calico_pools.json['node']['nodes'] | length > 1 ) or
( not calico_pools.json['node']['nodes'][0]['key'] | search(".*{{ kube_pods_subnet | ipaddr('network') }}.*") )
when: ( calico_pools['node']['nodes'] | length > 1 ) or
( not calico_pools['node']['nodes'][0]['key'] | search(".*{{ kube_pods_subnet | ipaddr('network') }}.*") )
run_once: true
- name: Calico | Write /etc/network-environment
@@ -131,4 +161,3 @@
shell: "{{ bin_dir }}/calicoctl node bgp peer add {{ item.router_id }} as {{ item.as }}"
with_items: peers
when: peer_with_router|default(false) and inventory_hostname in groups['kube-node']

4
roles/network_plugin/calico/templates/calico-node.service.j2
View File

@@ -1,8 +1,8 @@
[Unit]
Description=Calico per-node agent
Documentation=https://github.com/projectcalico/calico-docker
After=docker.service docker.socket etcd-proxy.service
Wants=docker.socket etcd-proxy.service
After=docker.service docker.socket
Wants=docker.socket
[Service]
User=root

7
roles/network_plugin/calico/templates/calicoctl-container.j2
View File

@@ -1,8 +1,13 @@
#!/bin/bash
/usr/bin/docker run --privileged --rm \
--net=host --pid=host -e ETCD_AUTHORITY={{ etcd_authority }} \
--net=host --pid=host \
-e ETCD_ENDPOINTS={{ etcd_access_endpoint }} \
-e ETCD_CA_CERT_FILE=/etc/calico/certs/ca_cert.crt \
-e ETCD_CERT_FILE=/etc/calico/certs/cert.crt \
-e ETCD_KEY_FILE=/etc/calico/certs/key.pem \
-v /usr/bin/docker:/usr/bin/docker \
-v /var/run/docker.sock:/var/run/docker.sock \
-v /var/run/calico:/var/run/calico \
-v /etc/calico/certs:/etc/calico/certs:ro \
{{ calicoctl_image_repo }}:{{ calicoctl_image_tag}} \
$@

7
roles/network_plugin/calico/templates/network-environment.j2
View File

@@ -3,7 +3,10 @@
DEFAULT_IPV4={{ip | default(ansible_default_ipv4.address) }}
# The Kubernetes master IP
KUBERNETES_MASTER={{ first_kube_master }}
KUBERNETES_MASTER={{ kube_apiserver_endpoint }}
# IP and port of etcd instance used by Calico
ETCD_AUTHORITY={{ etcd_authority }}
ETCD_ENDPOINTS={{ etcd_access_endpoint }}
ETCD_CA_CERT_FILE=/etc/calico/certs/ca_cert.crt
ETCD_CERT_FILE=/etc/calico/certs/cert.crt
ETCD_KEY_FILE=/etc/calico/certs/key.pem

12
roles/network_plugin/flannel/tasks/main.yml
View File

@@ -1,9 +1,11 @@
---
- name: Flannel | Write flannel configuration
template:
src: network.json
dest: /etc/flannel-network.json
backup: yes
- name: Flannel | Set Flannel etcd configuration
command: |-
{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} \
set /{{ cluster_name }}/network/config \
'{ "Network": "{{ kube_pods_subnet }}", "SubnetLen": {{ kube_network_node_prefix }}, "Backend": { "Type": "{{ flannel_backend_type }}" } }'
delegate_to: "{{groups['etcd'][0]}}"
run_once: true
- name: Flannel | Create flannel pod manifest
template:

18
roles/network_plugin/flannel/templates/flannel-pod.yml
View File

@@ -12,26 +12,16 @@
- name: "subnetenv"
hostPath:
path: "/run/flannel"
- name: "networkconfig"
- name: "etcd-certs"
hostPath:
path: "/etc/flannel-network.json"
path: "{{ etcd_cert_dir }}"
containers:
- name: "flannel-server-helper"
image: "{{ flannel_server_helper_image_repo }}:{{ flannel_server_helper_image_tag }}"
args:
- "--network-config=/etc/flannel-network.json"
- "--etcd-prefix=/{{ cluster_name }}/network"
- "--etcd-server={{ etcd_endpoint }}"
volumeMounts:
- name: "networkconfig"
mountPath: "/etc/flannel-network.json"
imagePullPolicy: "Always"
- name: "flannel-container"
image: "{{ flannel_image_repo }}:{{ flannel_image_tag }}"
command:
- "/bin/sh"
- "-c"
- "/opt/bin/flanneld -etcd-endpoints {{ etcd_access_endpoint }} -etcd-prefix /{{ cluster_name }}/network {% if flannel_interface is defined %}-iface {{ flannel_interface }}{% endif %} {% if flannel_public_ip is defined %}-public-ip {{ flannel_public_ip }}{% endif %}"
- "/opt/bin/flanneld -etcd-endpoints {{ etcd_access_endpoint }} -etcd-prefix /{{ cluster_name }}/network -etcd-cafile {{ etcd_cert_dir }}/ca.pem -etcd-certfile {{ etcd_cert_dir }}/node.pem -etcd-keyfile {{ etcd_cert_dir }}/node-key.pem {% if flannel_interface is defined %}-iface {{ flannel_interface }}{% endif %} {% if flannel_public_ip is defined %}-public-ip {{ flannel_public_ip }}{% endif %}"
ports:
- hostPort: 10253
containerPort: 10253
@@ -41,6 +31,8 @@
volumeMounts:
- name: "subnetenv"
mountPath: "/run/flannel"
- name: "etcd-certs"
mountPath: "{{ etcd_cert_dir }}"
securityContext:
privileged: true
hostNetwork: true

1
roles/network_plugin/flannel/templates/network.json
View File

@@ -1 +0,0 @@
{ "Network": "{{ kube_pods_subnet }}", "SubnetLen": {{ kube_network_node_prefix }}, "Backend": { "Type": "{{ flannel_backend_type }}" } }

2
roles/uploads/defaults/main.yml
View File

@@ -5,7 +5,7 @@ local_release_dir: /tmp
kube_version: v1.4.3
etcd_version: v3.0.6
calico_version: v0.22.0
calico_version: v0.23.0
calico_cni_version: v1.4.2
weave_version: v1.6.1