Merge pull request #528 from kubespray/proxy-nginx

Use nginx proxy on non-master nodes to proxy apiserver traffic

roles/kubernetes/node/defaults/main.yml

@@ -14,3 +14,6 @@ kube_proxy_masquerade_all: true
 # kube_api_runtime_config:
 #   - extensions/v1beta1/daemonsets=true
 #   - extensions/v1beta1/deployments=true
+
+nginx_image_repo: nginx
+nginx_image_tag: 1.11.4-alpine

roles/kubernetes/node/tasks/main.yml

@@ -1,6 +1,9 @@
 ---
 - include: install.yml
 
+- include: nginx-proxy.yml
+  when: is_kube_master == false and loadbalancer_apiserver_localhost|default(false)
+
 - name: Write Calico cni config
   template:
     src: "cni-calico.conf.j2"

roles/kubernetes/node/tasks/nginx-proxy.yml

@@ -0,0 +1,9 @@
+---
+- name: nginx-proxy | Write static pod
+  template: src=manifests/nginx-proxy.manifest.j2 dest=/etc/kubernetes/manifests/nginx-proxy.yml
+
+- name: nginx-proxy | Make nginx directory
+  file: path=/etc/nginx state=directory mode=0700 owner=root
+
+- name: nginx-proxy | Write nginx-proxy configuration
+  template: src=nginx.conf.j2 dest="/etc/nginx/nginx.conf" owner=root mode=0755 backup=yes

roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-proxy
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+  - name: nginx-proxy
+    image: {{ nginx_image_repo }}:{{ nginx_image_tag }}
+    securityContext:
+      privileged: true
+    volumeMounts:
+    - mountPath: /etc/nginx
+      name: etc-nginx
+      readOnly: true
+  volumes:
+  - name: etc-nginx
+    hostPath:
+      path: /etc/nginx

roles/kubernetes/node/templates/nginx.conf.j2

@@ -0,0 +1,26 @@
+error_log stderr notice;
+
+worker_processes auto;
+events {
+  multi_accept on;
+  use epoll;
+  worker_connections 1024;
+}
+
+stream {
+        upstream kube_apiserver {
+            least_conn;
+            {% for host in groups['kube-master'] -%}
+            server {{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }};
+            {% endfor %}
+        }
+
+        server {
+            listen        {{ kube_apiserver_port }};
+            proxy_pass    kube_apiserver;
+            proxy_timeout 3s;
+            proxy_connect_timeout 1s;
+
+        }
+
+}

roles/kubernetes/preinstall/defaults/main.yml

@@ -21,6 +21,8 @@ kube_log_dir: "/var/log/kubernetes"
 # pods on startup
 kube_manifest_dir: "{{ kube_config_dir }}/manifests"
 
+# change to 0.0.0.0 to enable insecure access from anywhere (not recommended)
+kube_apiserver_insecure_bind_address: 127.0.0.1
 
 common_required_pkgs:
   - python-httplib2

roles/kubernetes/preinstall/tasks/set_facts.yml

@@ -5,12 +5,12 @@
 - set_fact: is_kube_master="{{ inventory_hostname in groups['kube-master'] }}"
 - set_fact: first_kube_master="{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}"
 - set_fact:
-    kube_apiserver_insecure_bind_address: |-
-      {% if loadbalancer_apiserver_localhost %}{{ kube_apiserver_address }}{% else %}127.0.0.1{% endif %}
+    loadbalancer_apiserver_localhost: false
+    when: loadbalancer_apiserver is defined
 - set_fact:
     kube_apiserver_endpoint: |-
-      {% if loadbalancer_apiserver_localhost -%}
-           http://127.0.0.1:{{ kube_apiserver_insecure_port }}
+      {% if not is_kube_master and loadbalancer_apiserver_localhost -%}
+           https://localhost:{{ kube_apiserver_port }}
       {%- elif is_kube_master and loadbalancer_apiserver is not defined -%}
            http://127.0.0.1:{{ kube_apiserver_insecure_port }}
       {%- else -%}

roles/kubernetes/secrets/files/make-ssl.sh

@@ -26,8 +26,8 @@ Usage : $(basename $0) -f <config> [-d <ssldir>]
       -h | --help         : Show this message
       -f | --config       : Openssl configuration file
       -d | --ssldir       : Directory where the certificates will be installed
-               
-               ex : 
+
+               ex :
                $(basename $0) -f openssl.conf -d /srv/ssl
 EOF
 }
@@ -37,7 +37,7 @@ while (($#)); do
     case "$1" in
         -h | --help)   usage;   exit 0;;
         -f | --config) CONFIG=${2}; shift 2;;
-        -d | --ssldir) SSLDIR="${2}"; shift 2;; 
+        -d | --ssldir) SSLDIR="${2}"; shift 2;;
         *)
             usage
             echo "ERROR : Unknown option"
@@ -68,6 +68,7 @@ openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN
 openssl genrsa -out apiserver-key.pem 2048 > /dev/null 2>&1
 openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config ${CONFIG} > /dev/null 2>&1
 openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile ${CONFIG} > /dev/null 2>&1
+cat ca.pem >> apiserver.pem
 
 # Nodes and Admin
 for i in node admin; do

roles/kubernetes/secrets/tasks/gen_certs.yml

@@ -65,3 +65,30 @@
   shell: chmod 0600 {{ kube_cert_dir}}/*key.pem
   when: inventory_hostname in groups['kube-master']
   changed_when: false
+
+- name: Gen_certs | target ca-certificates directory
+  set_fact:
+    ca_cert_dir: |-
+      {% if ansible_os_family == "Debian" -%}
+      /usr/local/share/ca-certificates
+      {%- elif ansible_os_family == "RedHat" -%}
+      /etc/pki/ca-trust/source/anchors
+      {%- elif ansible_os_family == "CoreOS" -%}
+      /etc/ssl/certs
+      {%- endif %}
+
+- name: Gen_certs | add CA to trusted CA dir
+  copy:
+    src: "{{ kube_cert_dir }}/ca.pem"
+    dest: "{{ ca_cert_dir }}/kube-ca.crt"
+    remote_src: true
+  register: kube_ca_cert
+
+- name: Gen_certs | update ca-certificates (Debian/Ubuntu/CoreOS)
+  command: update-ca-certificates
+  when: kube_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS"]
+
+- name: Gen_certs | update ca-certificatesa (RedHat)
+  command: update-ca-trust extract
+  when: kube_ca_cert.changed and ansible_os_family == "RedHat"
+

roles/kubernetes/secrets/templates/openssl.conf.j2

@@ -11,12 +11,18 @@ DNS.1 = kubernetes
 DNS.2 = kubernetes.default
 DNS.3 = kubernetes.default.svc
 DNS.4 = kubernetes.default.svc.{{ dns_domain }}
+DNS.5 = localhost
+{% for host in groups['kube-master'] %}
+DNS.{{ 5 + loop.index }} = {{ host }}
+{% endfor %}
 {% if loadbalancer_apiserver is defined  and apiserver_loadbalancer_domain_name is defined %}
-DNS.5 = {{ apiserver_loadbalancer_domain_name }}
+{% set idx =  groups['kube-master'] | length | int + 5 %}
+DNS.{{ idx | string }} = {{ apiserver_loadbalancer_domain_name }}
 {% endif %}
 {% for host in groups['kube-master'] %}
 IP.{{ 2 * loop.index - 1 }} = {{ hostvars[host]['access_ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
 IP.{{ 2 * loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
 {% endfor %}
 {% set idx =  groups['kube-master'] | length | int * 2 + 1 %}
-IP.{{ idx | string }} = {{ kube_apiserver_ip }}
+IP.{{ idx }} = {{ kube_apiserver_ip }}
+IP.{{ idx + 1 }} = 127.0.0.1