Add HA/LB endpoints for kube-apiserver

* Add auto-evaluated internal endpoints and clarify the loadbalancer_apiserver
vars and usecases.
* Add loadbalancer_apiserver_localhost (default false). If enabled, override
the external LB and expect localhost:443/8080 to be new internal only frontends.
* Add kube_apiserver_multiaccess to ignore loadbalancers, and make clients
to access the apiservers as a comma-separated list of access_ip/ip/ansible ip
(a default mode). When disabled, allow clients to use the given loadbalancers.
* Define connections security mode for kube controllers, schedulers, proxies.
It is insecure be default, which is the current deployment choice.
* Rework the groups['kube-master'][0] hardcode defining the apiserver
endpoints.
* Improve grouping of vars and add facts for kube_apiserver.
* Define kube_apiserver_insecure_bind_address as a fact, add more
facts for ease of use.

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>

roles/kubernetes/master/templates/kube-apiserver.j2

@@ -21,7 +21,7 @@ KUBE_ALLOW_PRIV="--allow_privileged=true"
 KUBE_API_PORT="--insecure-port={{kube_apiserver_insecure_port}} --secure-port={{ kube_apiserver_port }}"
 
 # Insecure API address (default is localhost)
-KUBE_API_INSECURE_BIND="--insecure-bind-address={{ kube_apiserver_insecure_bind_address | default('127.0.0.1') }}"
+KUBE_API_INSECURE_BIND="--insecure-bind-address={{ kube_apiserver_insecure_bind_address }}"
 
 # Address range to use for services
 KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range={{ kube_service_addresses }}"
@@ -42,7 +42,7 @@ KUBE_RUNTIME_CONFIG="{% if kube_api_runtime_config is defined %}{% for conf in k
 KUBE_TLS_CONFIG="--tls_cert_file={{ kube_cert_dir }}/apiserver.pem --tls_private_key_file={{ kube_cert_dir }}/apiserver-key.pem --client_ca_file={{ kube_cert_dir }}/ca.pem"
 
 # Add you own!
-KUBE_API_ARGS="--token_auth_file={{ kube_token_dir }}/known_tokens.csv --basic-auth-file={{ kube_users_dir }}/known_users.csv --service_account_key_file={{ kube_cert_dir }}/apiserver-key.pem --advertise-address={{ ip | default(ansible_default_ipv4.address) }}"
+KUBE_API_ARGS="--token_auth_file={{ kube_token_dir }}/known_tokens.csv --basic-auth-file={{ kube_users_dir }}/known_users.csv --service_account_key_file={{ kube_cert_dir }}/apiserver-key.pem --advertise-address={{ kube_apiserver_address }}"
 
 {% if cloud_provider is defined and cloud_provider == "openstack" %}
 KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config"

roles/kubernetes/master/templates/kubectl-kubeconfig.yaml.j2

@@ -5,7 +5,7 @@ preferences: {}
 clusters:
 - cluster:
     certificate-authority-data: {{ kube_node_cert|b64encode }}
-    server: https://{{ groups['kube-master'][0] }}:{{ kube_apiserver_port }}
+    server: {{ kube_apiserver_endpoint }}
   name: {{ cluster_name }}
 contexts:
 - context:

roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2

@@ -11,7 +11,11 @@ spec:
     command:
     - /hyperkube
     - controller-manager
-    - --master=http://127.0.0.1:{{kube_apiserver_insecure_port}}
+{% if kube_controller_insecure %}
+    - --master={{ kube_apiserver_insecure_endpoint }}
+{% else %}
+    - --master={{ kube_apiserver_endpoint }}
+{% endif %}
     - --leader-elect=true
     - --service-account-private-key-file={{ kube_cert_dir }}/apiserver-key.pem
     - --root-ca-file={{ kube_cert_dir }}/ca.pem

roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2

@@ -12,7 +12,11 @@ spec:
     - /hyperkube
     - scheduler
     - --leader-elect=true
-    - --master=http://127.0.0.1:{{kube_apiserver_insecure_port}}
+{% if kube_scheduler_insecure %}
+    - --master={{ kube_apiserver_insecure_endpoint }}
+{% else %}
+    - --master={{ kube_apiserver_endpoint }}
+{% endif %}
     - --v={{ kube_log_level | default('2') }}
     livenessProbe:
       httpGet:

roles/kubernetes/node/templates/kubelet.j2

@@ -8,7 +8,7 @@ KUBE_LOGGING="--logtostderr=true"
 KUBE_LOG_LEVEL="--v={{ kube_log_level | default('2') }}"
 KUBE_ALLOW_PRIV="--allow_privileged=true"
 {% if inventory_hostname in groups['kube-node'] %}
-KUBELET_API_SERVER="--api_servers={% for host in groups['kube-master'] %}https://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }}{% if not loop.last %},{% endif %}{% endfor %}"
+KUBELET_API_SERVER="--api_servers={{ kube_apiserver_access_endpoint }}"
 {% endif %}
 # The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)
 KUBELET_ADDRESS="--address={{ ip | default("0.0.0.0") }}"

roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2

@@ -12,14 +12,10 @@ spec:
     - /hyperkube
     - proxy
     - --v={{ kube_log_level | default('2') }}
-{% if inventory_hostname in groups['kube-master'] %}
-    - --master=http://127.0.0.1:{{kube_apiserver_insecure_port}}
+{% if kube_proxy_insecure %}
+    - --master={{ kube_apiserver_insecure_endpoint }}
 {% else %}
-{%   if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %}
-    - --master=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }}
-{%   else %}
-    - --master=https://{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }}
-{%   endif%}
+    - --master={{ kube_apiserver_access_endpoint }}
     - --kubeconfig=/etc/kubernetes/node-kubeconfig.yaml
 {% endif %}
     - --bind-address={{ ip | default(ansible_default_ipv4.address) }}

roles/kubernetes/preinstall/tasks/set_facts.yml

@@ -1,4 +1,38 @@
 ---
+- set_fact: kube_apiserver_count="{{ groups['kube-master'] | length }}"
+- set_fact: kube_apiserver_address="{{ ip | default(ansible_default_ipv4['address']) }}"
+- set_fact: kube_apiserver_access_address="{{ access_ip | default(kube_apiserver_address) }}"
+- set_fact:
+    kube_apiserver_insecure_bind_address: |-
+      {% if loadbalancer_apiserver_localhost %}
+      {{ kube_apiserver_address }}{% else %}127.0.0.1{% endif %}
+- set_fact:
+    apiserver_access_port: |-
+      {% if loadbalancer_apiserver is defined and loadbalancer_apiserver.port is defined %}
+      {{ loadbalancer_apiserver.port }}{% else %}{{ kube_apiserver_port }}{% endif %}
+- set_fact:
+    kube_apiserver_endpoint: |-
+      {% if loadbalancer_apiserver_localhost %}
+      https://127.0.0.1:{{ apiserver_access_port }}
+      {% else -%}
+      https://{{ apiserver_loadbalancer_domain_name|default(kube_apiserver_access_address) }}:{{ apiserver_access_port }}
+      {%- endif %}
+- set_fact: kube_apiserver_insecure_endpoint="http://{{ kube_apiserver_insecure_bind_address }}:{{ kube_apiserver_insecure_port }}"
+- set_fact:
+    kube_apiserver_access_endpoint: |-
+      {% if kube_apiserver_multiaccess %}
+      {%   for host in groups['kube-master'] -%}
+      https://{{ hostvars[host].kube_apiserver_access_address }}:{{ kube_apiserver_port }}{% if not loop.last %},{% endif %}
+      {%-  endfor %}
+      {% else %}{{ kube_apiserver_endpoint }}{% endif %}
+- set_fact:
+    kube_apiserver_insecure_access_endpoint: |-
+      {% if kube_apiserver_multiaccess %}
+      {%   for host in groups['kube-master'] -%}
+      https://{{ hostvars[host].kube_apiserver_insecure_bind_address }}:{{ kube_apiserver_insecure_port }}{% if not loop.last %},{% endif %}
+      {%-  endfor %}
+      {% else %}{{ kube_apiserver_insecure_endpoint  }}{% endif %}
+
 - set_fact: etcd_address="{{ ip | default(ansible_default_ipv4['address']) }}"
 - set_fact: etcd_access_address="{{ access_ip | default(etcd_address) }}"
 - set_fact: etcd_peer_url="http://{{ etcd_access_address }}:2380"