mirror of
https://github.com/kubernetes-sigs/kubespray.git
synced 2026-03-27 05:45:05 -02:30
Add etcd proxy support
* Enforce a etcd-proxy role to a k8s-cluster group members. This provides an HA layout for all of the k8s cluster internal clients. * Proxies to be run on each node in the group as a separate etcd instances with a readwrite proxy mode and listen the given endpoint, which is either the access_ip:2379 or the localhost:2379. * A notion for the 'kube_etcd_multiaccess' is: ignore endpoints and loadbalancers and use the etcd members IPs as a comma-separated list. Otherwise, clients shall use the local endpoint provided by a etcd-proxy instances on each etcd node. A Netwroking plugins always use that access mode. * Fix apiserver's etcd servers args to use the etcd_access_endpoint. * Fix networking plugins flannel/calico to use the etcd_endpoint. * Fix name env var for non masters to be set as well. * Fix etcd_client_url was not used anywhere and other etcd_* facts evaluation was duplicated in a few places. * Define proxy modes only in the env file, if not a master. Del an automatic proxy mode decisions for etcd nodes in init/unit scripts. * Use Wants= instead of Requires= as "This is the recommended way to hook start-up of one unit to the start-up of another unit" * Make apiserver/calico Wants= etcd-proxy to keep it always up Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com> Co-authored-by: Matthew Mosesohn <mmosesohn@mirantis.com>
This commit is contained in:
Bogdan Dobrelya
@@ -16,23 +16,16 @@ PATH=/sbin:/usr/sbin:/bin/:/usr/bin
|
||||
DESC="etcd k/v store"
|
||||
NAME=etcd
|
||||
DAEMON={{ docker_bin_dir | default("/usr/bin") }}/docker
|
||||
{% if is_etcd_master %}
|
||||
DAEMON_ARGS='--restart=always --env-file=/etc/etcd.env \
|
||||
--net=host \
|
||||
-v /usr/share/ca-certificates/:/etc/ssl/certs:ro \
|
||||
-v /var/lib/etcd:/var/lib/etcd:rw \
|
||||
--name={{ etcd_member_name | default("etcd-proxy") }} \
|
||||
--name={{ etcd_member_name | default("etcd") }} \
|
||||
{{ etcd_image_repo }}:{{ etcd_image_tag }} \
|
||||
{% if etcd_after_v3 %}
|
||||
{{ etcd_container_bin_dir }}etcd \
|
||||
{% endif %}
|
||||
{% if is_etcd_master %}
|
||||
--proxy off
|
||||
{% else %}
|
||||
--proxy on
|
||||
{{ etcd_container_bin_dir }}etcd
|
||||
{% endif %}'
|
||||
|
||||
|
||||
SCRIPTNAME=/etc/init.d/$NAME
|
||||
DAEMON_USER=etcd
|
||||
STOP_SCHEDULE="${STOP_SCHEDULE:-QUIT/5/TERM/5/KILL/5}"
|
||||
@@ -41,9 +34,6 @@ PID=/var/run/etcd.pid
|
||||
# Exit if the binary is not present
|
||||
[ -x "$DAEMON" ] || exit 0
|
||||
|
||||
# Read configuration variable file if it is present
|
||||
[ -f /etc/etcd.env ] && . /etc/etcd.env
|
||||
|
||||
# Define LSB log_* functions.
|
||||
# Depend on lsb-base (>= 3.2-14) to ensure that this file is present
|
||||
# and status_of_proc is working.
|
||||
|
||||
@@ -16,11 +16,6 @@ PATH=/sbin:/usr/sbin:/bin:/usr/bin
|
||||
DESC="etcd k/v store"
|
||||
NAME=etcd
|
||||
DAEMON={{ bin_dir }}/etcd
|
||||
{% if is_etcd_master %}
|
||||
DAEMON_ARGS=""
|
||||
{% else %}
|
||||
DAEMON_ARGS="--proxy on"
|
||||
{% endif %}
|
||||
SCRIPTNAME=/etc/init.d/$NAME
|
||||
DAEMON_USER=etcd
|
||||
STOP_SCHEDULE="${STOP_SCHEDULE:-QUIT/5/TERM/5/KILL/5}"
|
||||
|
||||
117
roles/etcd/templates/deb-etcd-proxy-docker.initd.j2
Normal file
117
roles/etcd/templates/deb-etcd-proxy-docker.initd.j2
Normal file
@@ -0,0 +1,117 @@
|
||||
#!/bin/sh
|
||||
set -a
|
||||
|
||||
### BEGIN INIT INFO
|
||||
# Provides: etcd-proxy
|
||||
# Required-Start: $local_fs $network $syslog
|
||||
# Required-Stop:
|
||||
# Default-Start: 2 3 4 5
|
||||
# Default-Stop: 0 1 6
|
||||
# Short-Description: etcd-proxy
|
||||
# Description:
|
||||
# etcd-proxy is a proxy for etcd: distributed, consistent key-value store for shared configuration and service discovery
|
||||
### END INIT INFO
|
||||
|
||||
PATH=/sbin:/usr/sbin:/bin/:/usr/bin
|
||||
DESC="etcd-proxy"
|
||||
NAME=etcd-proxy
|
||||
DAEMON={{ docker_bin_dir | default("/usr/bin") }}/docker
|
||||
DAEMON_ARGS='--restart=always --env-file=/etc/etcd-proxy.env \
|
||||
--net=host \
|
||||
--stop-signal=SIGKILL \
|
||||
-v /usr/share/ca-certificates/:/etc/ssl/certs:ro \
|
||||
--name={{ etcd_proxy_member_name | default("etcd-proxy") }} \
|
||||
{{ etcd_image_repo }}:{{ etcd_image_tag }} \
|
||||
{% if etcd_after_v3 %}
|
||||
{{ etcd_container_bin_dir }}etcd
|
||||
{% endif %}'
|
||||
|
||||
|
||||
SCRIPTNAME=/etc/init.d/$NAME
|
||||
DAEMON_USER=etcd-proxy
|
||||
STOP_SCHEDULE="${STOP_SCHEDULE:-QUIT/5/TERM/5/KILL/5}"
|
||||
PID=/var/run/etcd-proxy.pid
|
||||
|
||||
# Exit if the binary is not present
|
||||
[ -x "$DAEMON" ] || exit 0
|
||||
|
||||
# Define LSB log_* functions.
|
||||
# Depend on lsb-base (>= 3.2-14) to ensure that this file is present
|
||||
# and status_of_proc is working.
|
||||
. /lib/lsb/init-functions
|
||||
|
||||
do_status()
|
||||
{
|
||||
status_of_proc -p $PID "$DAEMON" "$NAME" && exit 0 || exit $?
|
||||
}
|
||||
|
||||
# Function that starts the daemon/service
|
||||
#
|
||||
do_start()
|
||||
{
|
||||
start-stop-daemon --background --start --quiet --make-pidfile --pidfile $PID --user $DAEMON_USER --exec $DAEMON -- \
|
||||
$DAEMON_ARGS \
|
||||
|| return 2
|
||||
}
|
||||
|
||||
#
|
||||
# Function that stops the daemon/service
|
||||
#
|
||||
do_stop()
|
||||
{
|
||||
start-stop-daemon --stop --quiet --retry=$STOP_SCHEDULE --pidfile $PID --name $NAME
|
||||
RETVAL="$?"
|
||||
|
||||
sleep 1
|
||||
return "$RETVAL"
|
||||
}
|
||||
|
||||
|
||||
case "$1" in
|
||||
start)
|
||||
log_daemon_msg "Starting $DESC" "$NAME"
|
||||
do_start
|
||||
case "$?" in
|
||||
0|1) log_end_msg 0 || exit 0 ;;
|
||||
2) log_end_msg 1 || exit 1 ;;
|
||||
esac
|
||||
;;
|
||||
stop)
|
||||
log_daemon_msg "Stopping $DESC" "$NAME"
|
||||
if do_stop; then
|
||||
log_end_msg 0
|
||||
else
|
||||
log_failure_msg "Can't stop etcd-proxy"
|
||||
log_end_msg 1
|
||||
fi
|
||||
;;
|
||||
status)
|
||||
if do_status; then
|
||||
log_end_msg 0
|
||||
else
|
||||
log_failure_msg "etcd-proxy is not running"
|
||||
log_end_msg 1
|
||||
fi
|
||||
;;
|
||||
|
||||
restart|force-reload)
|
||||
log_daemon_msg "Restarting $DESC" "$NAME"
|
||||
if do_stop; then
|
||||
if do_start; then
|
||||
log_end_msg 0
|
||||
exit 0
|
||||
else
|
||||
rc="$?"
|
||||
fi
|
||||
else
|
||||
rc="$?"
|
||||
fi
|
||||
log_failure_msg "Can't restart etcd-proxy"
|
||||
log_end_msg ${rc}
|
||||
;;
|
||||
*)
|
||||
echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
|
||||
exit 3
|
||||
;;
|
||||
esac
|
||||
|
||||
110
roles/etcd/templates/deb-etcd-proxy-host.initd.j2
Normal file
110
roles/etcd/templates/deb-etcd-proxy-host.initd.j2
Normal file
@@ -0,0 +1,110 @@
|
||||
#!/bin/sh
|
||||
set -a
|
||||
|
||||
### BEGIN INIT INFO
|
||||
# Provides: etcd-proxy
|
||||
# Required-Start: $local_fs $network $syslog
|
||||
# Required-Stop:
|
||||
# Default-Start: 2 3 4 5
|
||||
# Default-Stop: 0 1 6
|
||||
# Short-Description: etcd-proxy
|
||||
# Description:
|
||||
# etcd-proxy is a proxy for etcd: distributed, consistent key-value store for shared configuration and service discovery
|
||||
### END INIT INFO
|
||||
|
||||
PATH=/sbin:/usr/sbin:/bin:/usr/bin
|
||||
DESC="etcd-proxy"
|
||||
NAME=etcd-proxy
|
||||
DAEMON={{ bin_dir }}/etcd
|
||||
DAEMON_ARGS=""
|
||||
SCRIPTNAME=/etc/init.d/$NAME
|
||||
DAEMON_USER=etcd
|
||||
STOP_SCHEDULE="${STOP_SCHEDULE:-QUIT/5/TERM/5/KILL/5}"
|
||||
PID=/var/run/etcd-proxy.pid
|
||||
|
||||
# Exit if the binary is not present
|
||||
[ -x "$DAEMON" ] || exit 0
|
||||
|
||||
# Read configuration variable file if it is present
|
||||
[ -f /etc/etcd-proxy.env ] && . /etc/etcd-proxy.env
|
||||
|
||||
# Define LSB log_* functions.
|
||||
# Depend on lsb-base (>= 3.2-14) to ensure that this file is present
|
||||
# and status_of_proc is working.
|
||||
. /lib/lsb/init-functions
|
||||
|
||||
do_status()
|
||||
{
|
||||
status_of_proc -p $PID "$DAEMON" "$NAME" && exit 0 || exit $?
|
||||
}
|
||||
|
||||
# Function that starts the daemon/service
|
||||
#
|
||||
do_start()
|
||||
{
|
||||
start-stop-daemon --background --start --quiet --make-pidfile --pidfile $PID --user $DAEMON_USER --exec $DAEMON -- \
|
||||
$DAEMON_ARGS \
|
||||
|| return 2
|
||||
}
|
||||
|
||||
#
|
||||
# Function that stops the daemon/service
|
||||
#
|
||||
do_stop()
|
||||
{
|
||||
start-stop-daemon --stop --quiet --retry=$STOP_SCHEDULE --pidfile $PID --name $NAME
|
||||
RETVAL="$?"
|
||||
|
||||
sleep 1
|
||||
return "$RETVAL"
|
||||
}
|
||||
|
||||
|
||||
case "$1" in
|
||||
start)
|
||||
log_daemon_msg "Starting $DESC" "$NAME"
|
||||
do_start
|
||||
case "$?" in
|
||||
0|1) log_end_msg 0 || exit 0 ;;
|
||||
2) log_end_msg 1 || exit 1 ;;
|
||||
esac
|
||||
;;
|
||||
stop)
|
||||
log_daemon_msg "Stopping $DESC" "$NAME"
|
||||
if do_stop; then
|
||||
log_end_msg 0
|
||||
else
|
||||
log_failure_msg "Can't stop etcd-proxy"
|
||||
log_end_msg 1
|
||||
fi
|
||||
;;
|
||||
status)
|
||||
if do_status; then
|
||||
log_end_msg 0
|
||||
else
|
||||
log_failure_msg "etcd-proxy is not running"
|
||||
log_end_msg 1
|
||||
fi
|
||||
;;
|
||||
|
||||
restart|force-reload)
|
||||
log_daemon_msg "Restarting $DESC" "$NAME"
|
||||
if do_stop; then
|
||||
if do_start; then
|
||||
log_end_msg 0
|
||||
exit 0
|
||||
else
|
||||
rc="$?"
|
||||
fi
|
||||
else
|
||||
rc="$?"
|
||||
fi
|
||||
log_failure_msg "Can't restart etcd-proxy"
|
||||
log_end_msg ${rc}
|
||||
;;
|
||||
*)
|
||||
echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
|
||||
exit 3
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[Unit]
|
||||
Description=etcd docker wrapper
|
||||
Requires=docker.service
|
||||
Wants=docker.service
|
||||
After=docker.service
|
||||
|
||||
[Service]
|
||||
@@ -13,15 +13,10 @@ ExecStart={{ docker_bin_dir | default("/usr/bin") }}/docker run --restart=always
|
||||
--net=host \
|
||||
-v /usr/share/ca-certificates/:/etc/ssl/certs:ro \
|
||||
-v /var/lib/etcd:/var/lib/etcd:rw \
|
||||
--name={{ etcd_member_name | default("etcd-proxy") }} \
|
||||
--name={{ etcd_member_name | default("etcd") }} \
|
||||
{{ etcd_image_repo }}:{{ etcd_image_tag }} \
|
||||
{% if etcd_after_v3 %}
|
||||
{{ etcd_container_bin_dir }}etcd \
|
||||
{% endif %}
|
||||
{% if is_etcd_master %}
|
||||
--proxy off
|
||||
{% else %}
|
||||
--proxy on
|
||||
{{ etcd_container_bin_dir }}etcd
|
||||
{% endif %}
|
||||
ExecStartPre=-{{ docker_bin_dir | default("/usr/bin") }}/docker rm -f {{ etcd_member_name | default("etcd-proxy") }}
|
||||
ExecReload={{ docker_bin_dir | default("/usr/bin") }}/docker restart {{ etcd_member_name | default("etcd-proxy") }}
|
||||
|
||||
@@ -6,11 +6,7 @@ After=network.target
|
||||
Type=notify
|
||||
User=etcd
|
||||
EnvironmentFile=/etc/etcd.env
|
||||
{% if inventory_hostname in groups['etcd'] %}
|
||||
ExecStart={{ bin_dir }}/etcd
|
||||
{% else %}
|
||||
ExecStart={{ bin_dir }}/etcd -proxy on
|
||||
{% endif %}
|
||||
NotifyAccess=all
|
||||
Restart=always
|
||||
RestartSec=10s
|
||||
|
||||
28
roles/etcd/templates/etcd-proxy-docker.service.j2
Normal file
28
roles/etcd/templates/etcd-proxy-docker.service.j2
Normal file
@@ -0,0 +1,28 @@
|
||||
[Unit]
|
||||
Description=etcd-proxy docker wrapper
|
||||
Wants=docker.service
|
||||
After=docker.service
|
||||
|
||||
[Service]
|
||||
User=root
|
||||
PermissionsStartOnly=true
|
||||
ExecStart={{ docker_bin_dir | default("/usr/bin") }}/docker run --restart=always \
|
||||
--env-file=/etc/etcd-proxy.env \
|
||||
{# TODO(mattymo): Allow docker IP binding and disable in envfile
|
||||
-p 2380:2380 -p 2379:2379 #}
|
||||
--net=host \
|
||||
--stop-signal=SIGKILL \
|
||||
-v /usr/share/ca-certificates/:/etc/ssl/certs:ro \
|
||||
--name={{ etcd_proxy_member_name | default("etcd-proxy") }} \
|
||||
{{ etcd_image_repo }}:{{ etcd_image_tag }} \
|
||||
{% if etcd_after_v3 %}
|
||||
{{ etcd_container_bin_dir }}etcd
|
||||
{% endif %}
|
||||
ExecStartPre=-{{ docker_bin_dir | default("/usr/bin") }}/docker rm -f {{ etcd_proxy_member_name | default("etcd-proxy") }}
|
||||
ExecReload={{ docker_bin_dir | default("/usr/bin") }}/docker restart {{ etcd_proxy_member_name | default("etcd-proxy") }}
|
||||
ExecStop={{ docker_bin_dir | default("/usr/bin") }}/docker stop {{ etcd_proxy_member_name | default("etcd-proxy") }}
|
||||
Restart=always
|
||||
RestartSec=10s
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
19
roles/etcd/templates/etcd-proxy-host.service.j2
Normal file
19
roles/etcd/templates/etcd-proxy-host.service.j2
Normal file
@@ -0,0 +1,19 @@
|
||||
[Unit]
|
||||
Description=etcd-proxy
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=notify
|
||||
User=etcd
|
||||
PermissionsStartOnly=true
|
||||
EnvironmentFile=/etc/etcd-proxy.env
|
||||
ExecStart={{ bin_dir }}/etcd
|
||||
ExecStartPre=/bin/mkdir -p /var/lib/etcd-proxy
|
||||
ExecStartPre=/bin/chown -R etcd: /var/lib/etcd-proxy
|
||||
NotifyAccess=all
|
||||
Restart=always
|
||||
RestartSec=10s
|
||||
LimitNOFILE=40000
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
5
roles/etcd/templates/etcd-proxy.j2
Normal file
5
roles/etcd/templates/etcd-proxy.j2
Normal file
@@ -0,0 +1,5 @@
|
||||
ETCD_DATA_DIR=/var/lib/etcd-proxy
|
||||
ETCD_PROXY=on
|
||||
ETCD_LISTEN_CLIENT_URLS={{ etcd_access_endpoint }}
|
||||
ETCD_NAME={{ etcd_proxy_member_name | default("etcd-proxy") }}
|
||||
ETCD_INITIAL_CLUSTER={% for host in groups['etcd'] %}etcd{{ loop.index|string }}={{ hostvars[host]['etcd_peer_url'] }}{% if not loop.last %},{% endif %}{% endfor %}
|
||||
@@ -1,14 +1,12 @@
|
||||
ETCD_DATA_DIR=/var/lib/etcd
|
||||
{% if is_etcd_master %}
|
||||
ETCD_ADVERTISE_CLIENT_URLS=http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address'])) }}:2379
|
||||
ETCD_INITIAL_ADVERTISE_PEER_URLS=http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address'])) }}:2380
|
||||
ETCD_ADVERTISE_CLIENT_URLS={{ etcd_client_url }}
|
||||
ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_peer_url }}
|
||||
ETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc != 0 | bool %}new{% else %}existing{% endif %}
|
||||
|
||||
ETCD_LISTEN_CLIENT_URLS=http://{{ etcd_address }}:2379
|
||||
ETCD_ELECTION_TIMEOUT=10000
|
||||
ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd
|
||||
ETCD_LISTEN_PEER_URLS=http://{{ hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address']) }}:2380
|
||||
ETCD_LISTEN_PEER_URLS=http://{{ etcd_address }}:2380
|
||||
ETCD_NAME={{ etcd_member_name }}
|
||||
{% endif %}
|
||||
ETCD_PROXY=off
|
||||
ETCD_INITIAL_CLUSTER={% for host in groups['etcd'] %}etcd{{ loop.index|string }}={{ hostvars[host]['etcd_peer_url'] }}{% if not loop.last %},{% endif %}{% endfor %}
|
||||
|
||||
ETCD_LISTEN_CLIENT_URLS=http://{{ hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address']) }}:2379,http://127.0.0.1:2379
|
||||
|
||||
Reference in New Issue
Block a user