Add HAProxy as internal loadbalancer (#4480)

roles/kubernetes/node/defaults/main.yml

@@ -42,9 +42,9 @@ kube_master_cpu_reserved: 200m
 
 kubelet_status_update_frequency: 10s
 
-# Requests for nginx load balancer app
-nginx_memory_requests: 32M
-nginx_cpu_requests: 25m
+# Requests for load balancer app
+loadbalancer_apiserver_memory_requests: 32M
+loadbalancer_apiserver_cpu_requests: 25m
 
 # kube_api_runtime_config:
 #   - extensions/v1beta1/daemonsets=true

roles/kubernetes/node/tasks/haproxy.yml

@@ -0,0 +1,25 @@
+---
+- name: haproxy | Cleanup potentially deployed nginx-proxy
+  file:
+    path: "{{ kube_manifest_dir }}/nginx-proxy.yml"
+    state: absent
+
+- name: haproxy | Write static pod
+  template:
+    src: manifests/haproxy.manifest.j2
+    dest: "{{ kube_manifest_dir }}/haproxy.yml"
+
+- name: haproxy | Make haproxy directory
+  file:
+    path: "{{ haproxy_config_dir }}"
+    state: directory
+    mode: 0700
+    owner: root
+
+- name: haproxy | Write haproxy configuration
+  template:
+    src: haproxy.cfg.j2
+    dest: "{{ haproxy_config_dir }}/haproxy.cfg"
+    owner: root
+    mode: 0755
+    backup: yes

roles/kubernetes/node/tasks/main.yml

@@ -18,10 +18,15 @@
     - kubelet
 
 - import_tasks: nginx-proxy.yml
-  when: is_kube_master == false and loadbalancer_apiserver_localhost
+  when: is_kube_master == false and loadbalancer_apiserver_localhost and loadbalancer_apiserver_type == 'nginx'
   tags:
     - nginx
 
+- import_tasks: haproxy.yml
+  when: is_kube_master == false and loadbalancer_apiserver_localhost and loadbalancer_apiserver_type == 'haproxy'
+  tags:
+    - haproxy
+
 - name: Make sure dynamic kubelet configuration directory is writeable
   file:
     path: "{{ dynamic_kubelet_configuration_dir }}"

roles/kubernetes/node/tasks/nginx-proxy.yml

@@ -1,8 +1,13 @@
 ---
+- name: haproxy | Cleanup potentially deployed haproxy
+  file:
+    path: "{{ kube_manifest_dir }}/haproxy.yml"
+    state: absent
+
 - name: nginx-proxy | Write static pod
   template:
     src: manifests/nginx-proxy.manifest.j2
-    dest: "{{kube_manifest_dir}}/nginx-proxy.yml"
+    dest: "{{ kube_manifest_dir }}/nginx-proxy.yml"
 
 - name: nginx-proxy | Make nginx directory
   file:

roles/kubernetes/node/templates/haproxy.cfg.j2

@@ -0,0 +1,43 @@
+global
+    maxconn                 4000
+    log                     127.0.0.1 local0
+
+defaults
+    mode                    http
+    log                     global
+    option                  httplog
+    option                  dontlognull
+    option                  http-server-close
+    option                  redispatch
+    retries                 5
+    timeout http-request    5m
+    timeout queue           5m
+    timeout connect         30s
+    timeout client          15m
+    timeout server          15m
+    timeout http-keep-alive 30s
+    timeout check           30s
+    maxconn                 4000
+
+{% if loadbalancer_apiserver_healthcheck_port is defined -%}
+frontend healthz
+  bind *:{{ loadbalancer_apiserver_healthcheck_port }}
+  mode http
+  monitor-uri /healthz
+{% endif %}
+
+frontend kube_api_frontend
+  bind *:{{ loadbalancer_apiserver_port|default(kube_apiserver_port) }}
+  mode tcp
+  option tcplog
+  default_backend kube_api_backend
+
+backend kube_api_backend
+  mode tcp
+  balance leastconn
+  default-server inter 15s downinter 15s rise 2 fall 2 slowstart 60s maxconn 1000 maxqueue 256 weight 100
+  option httpchk GET /healthz
+  http-check expect status 200
+  {% for host in groups['kube-master'] -%}
+  server {{ host }} {{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(fallback_ips[host])) }}:{{ kube_apiserver_port }} check check-ssl verify none
+  {% endfor -%}

roles/kubernetes/node/templates/manifests/haproxy.manifest.j2

@@ -0,0 +1,43 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: haproxy
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    k8s-app: kube-haproxy
+spec:
+  hostNetwork: true
+  nodeSelector:
+    beta.kubernetes.io/os: linux
+{% if kube_version is version('v1.11.1', '>=') %}
+  priorityClassName: system-node-critical
+{% endif %}
+  containers:
+  - name: haproxy
+    image: {{ haproxy_image_repo }}:{{ haproxy_image_tag }}
+    imagePullPolicy: {{ k8s_image_pull_policy }}
+    resources:
+      requests:
+        cpu: {{ loadbalancer_apiserver_cpu_requests }}
+        memory: {{ loadbalancer_apiserver_memory_requests }}
+    securityContext:
+      privileged: true
+    {% if loadbalancer_apiserver_healthcheck_port is defined -%}
+    livenessProbe:
+      httpGet:
+        path: /healthz
+        port: {{ loadbalancer_apiserver_healthcheck_port }}
+    readinessProbe:
+      httpGet:
+        path: /healthz
+        port: {{ loadbalancer_apiserver_healthcheck_port }}
+    {% endif -%}
+    volumeMounts:
+    - mountPath: /usr/local/etc/haproxy/
+      name: etc-haproxy
+      readOnly: true
+  volumes:
+  - name: etc-haproxy
+    hostPath:
+      path: {{ haproxy_config_dir }}

roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2

@@ -19,19 +19,19 @@ spec:
     imagePullPolicy: {{ k8s_image_pull_policy }}
     resources:
       requests:
-        cpu: {{ nginx_cpu_requests }}
-        memory: {{ nginx_memory_requests }}
+        cpu: {{ loadbalancer_apiserver_cpu_requests }}
+        memory: {{ loadbalancer_apiserver_memory_requests }}
     securityContext:
       privileged: true
-    {% if nginx_kube_apiserver_healthcheck_port is defined -%}
+    {% if loadbalancer_apiserver_healthcheck_port is defined -%}
     livenessProbe:
       httpGet:
         path: /healthz
-        port: {{ nginx_kube_apiserver_healthcheck_port }}
+        port: {{ loadbalancer_apiserver_healthcheck_port }}
     readinessProbe:
       httpGet:
         path: /healthz
-        port: {{ nginx_kube_apiserver_healthcheck_port }}
+        port: {{ loadbalancer_apiserver_healthcheck_port }}
     {% endif -%}
     volumeMounts:
     - mountPath: /etc/nginx

roles/kubernetes/node/templates/nginx.conf.j2

@@ -19,7 +19,7 @@ stream {
   }
 
   server {
-    listen        127.0.0.1:{{ nginx_kube_apiserver_port|default(kube_apiserver_port) }};
+    listen        {{ loadbalancer_apiserver_port|default(kube_apiserver_port) }};
     proxy_pass    kube_apiserver;
     proxy_timeout 10m;
     proxy_connect_timeout 1s;
@@ -38,13 +38,13 @@ http {
   server_tokens off;
   autoindex off;
 
-  {% if nginx_kube_apiserver_healthcheck_port is defined -%}
+  {% if loadbalancer_apiserver_healthcheck_port is defined -%}
   server {
-    listen {{ nginx_kube_apiserver_healthcheck_port }};
+    listen {{ loadbalancer_apiserver_healthcheck_port }};
     location /healthz {
       access_log off;
       return 200;
     }
   }
-  {% endif -%}
+  {% endif %}
 }