Use K8s 1.15 (#4905)

* Use K8s 1.15

* Use Kubernetes 1.15 and use kubeadm.k8s.io/v1beta2 for
  InitConfiguration.
* bump to v1.15.0

* Remove k8s 1.13 checksums.

* Update README kubernetes version 1.15.0.

* Update metrics server 0.3.3 for k8s 1.15

* Remove less than k8s 1.14 related code

* Use kubeadm with --upload-certs instead of --experimental-upload-certs due to depricate

* Update dnsautoscaler 1.6.0

* Skip certificateKey if it's not defined

* Add kubeadm-conftolplane.v2beta2 for k8s 1.15 or later

* Support kubeadm control plane for k8s 1.15

* Update sonobuoy version 0.15.0 for k8s 1.15

roles/kubernetes/master/defaults/main/main.yml

@@ -96,12 +96,8 @@ kube_apiserver_admission_control:
   - ServiceAccount
   - DefaultStorageClass
   - PersistentVolumeClaimResize
-  - >-
-      {%- if kube_version is version('v1.9', '<') -%}
-      GenericAdmissionWebhook
-      {%- else -%}
-      MutatingAdmissionWebhook,ValidatingAdmissionWebhook
-      {%- endif -%}
+  - MutatingAdmissionWebhook
+  - ValidatingAdmissionWebhook
   - ResourceQuota
 
 # 1.10+ admission plugins

roles/kubernetes/master/tasks/kubeadm-secondary-experimental.yml

@@ -30,8 +30,13 @@
   command: >-
     {{ bin_dir }}/kubeadm init phase
     --config {{ kube_config_dir }}/kubeadm-config.yaml
-    upload-certs --experimental-upload-certs
-    {% if kubeadm_certificate_key is defined %}
+    upload-certs
+    {% if kubeadm_version is version('v1.15.0', '<') %}
+    --experimental-upload-certs
+    {% else %}
+    --upload-certs
+    {% endif %}
+    {% if kubeadm_certificate_key is defined and kubeadm_version is version('v1.15.0', '<') %}
     --certificate-key={{ kubeadm_certificate_key }}
     {% endif %}
   register: kubeadm_upload_cert
@@ -52,7 +57,7 @@
     {{ bin_dir }}/kubeadm join
     --config {{ kube_config_dir }}/kubeadm-controlplane.yaml
     --ignore-preflight-errors=all
-    {% if kubeadm_certificate_key is defined %}
+    {% if kubeadm_certificate_key is defined and kubeadm_version is version('v1.15.0', '<') %}
     --certificate-key={{ kubeadm_certificate_key }}
     {% endif %}
   register: kubeadm_join_control_plane

roles/kubernetes/master/tasks/kubeadm-setup.yml

@@ -109,10 +109,14 @@
     --ignore-preflight-errors=all
     --skip-phases=addon/coredns
     {% if kubeadm_version is version('v1.14.0', '>=') %}
+    {% if kubeadm_version is version('v1.15.0', '<') %}
     --experimental-upload-certs
     {% if kubeadm_certificate_key is defined %}
     --certificate-key={{ kubeadm_certificate_key }}
     {% endif %}
+    {% else %}
+    --upload-certs
+    {% endif %}
     {% endif %}
   register: kubeadm_init
   # Retry is because upload config sometimes fails

roles/kubernetes/master/tasks/kubeadm-version.yml

@@ -3,25 +3,16 @@
   command: "{{ bin_dir }}/kubeadm version -o short"
   register: kubeadm_output
 
-- name: sets kubeadm api version to v1alpha2
-  set_fact:
-    kubeadmConfig_api_version: v1alpha2
-  when:
-    - kubeadm_output.stdout is version('v1.11.0', '>=')
-    - kubeadm_output.stdout is version('v1.12.0', '<')
-
-- name: sets kubeadm api version to v1alpha3
-  set_fact:
-    kubeadmConfig_api_version: v1alpha3
-  when:
-    - kubeadm_output.stdout is version('v1.12.0', '>=')
-    - kubeadm_output.stdout is version('v1.13.0', '<')
-
 - name: sets kubeadm api version to v1beta1
   set_fact:
     kubeadmConfig_api_version: v1beta1
   when: kubeadm_output.stdout is version('v1.13.0', '>=')
 
+- name: sets kubeadm api version to v1beta2
+  set_fact:
+    kubeadmConfig_api_version: v1beta2
+  when: kubeadm_output.stdout is version('v1.15.0', '>=')
+
 - name: kubeadm | Create kubeadm config
   template:
     src: "kubeadm-config.{{ kubeadmConfig_api_version }}.yaml.j2"

roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2

@@ -1,235 +0,0 @@
-apiVersion: kubeadm.k8s.io/v1alpha2
-kind: MasterConfiguration
-api:
-{% if kubeadm_config_api_fqdn is defined %}
-  controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}
-  bindPort: {{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
-{% else %}
-  advertiseAddress: {{ ip | default(fallback_ips[inventory_hostname]) }}
-  bindPort: {{ kube_apiserver_port }}
-{% endif %}
-etcd:
-  external:
-      endpoints:
-{% for endpoint in etcd_access_addresses.split(',') %}
-      - {{ endpoint }}
-{% endfor %}
-      caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }}
-      certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }}
-      keyFile: {{ etcd_cert_dir }}/{{ kube_etcd_key_file }}
-networking:
-  dnsDomain: {{ dns_domain }}
-  serviceSubnet: {{ kube_service_addresses }}
-  podSubnet: {{ kube_pods_subnet }}
-kubernetesVersion: {{ kube_version }}
-kubeProxy:
-  config:
-    mode: {{ kube_proxy_mode }}
-{% if kube_proxy_nodeport_addresses %}
-    nodePortAddresses: {{ kube_proxy_nodeport_addresses }}
-{% endif %}
-    resourceContainer: ""
-authorizationModes:
-{% for mode in authorization_modes %}
-- {{ mode }}
-{% endfor %}
-apiServerExtraArgs:
-  bind-address: {{ kube_apiserver_bind_address }}
-{% if kube_apiserver_insecure_port|string != "0" %}
-  insecure-bind-address: {{ kube_apiserver_insecure_bind_address }}
-{% endif %}
-  insecure-port: "{{ kube_apiserver_insecure_port }}"
-{% if kube_version is version('v1.10', '<') %}
-  admission-control: {{ kube_apiserver_admission_control | join(',') }}
-{% else %}
-{% if kube_apiserver_enable_admission_plugins|length > 0 %}
-  enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }}
-{% endif %}
-{% if kube_apiserver_disable_admission_plugins|length > 0 %}
-  disable-admission-plugins: {{ kube_apiserver_disable_admission_plugins | join(',') }}
-{% endif %}
-{% endif %}
-  apiserver-count: "{{ kube_apiserver_count }}"
-{% if kube_version is version('v1.9', '>=') %}
-  endpoint-reconciler-type: lease
-{% endif %}
-{% if etcd_events_cluster_enabled %}
-  etcd-servers-overrides: "/events#{{ etcd_events_access_addresses_semicolon }}"
-{% endif %}
-  service-node-port-range: {{ kube_apiserver_node_port_range }}
-  kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}"
-  profiling: "{{ kube_profiling }}"
-  request-timeout: "{{ kube_apiserver_request_timeout }}"
-  repair-malformed-updates: "false"
-  enable-aggregator-routing: "{{ kube_api_aggregator_routing }}"
-{% if kube_api_anonymous_auth is defined and kube_version is version('v1.5', '>=')  %}
-  anonymous-auth: "{{ kube_api_anonymous_auth }}"
-{% endif %}
-{% if kube_basic_auth|default(true) %}
-  basic-auth-file: {{ kube_users_dir }}/known_users.csv
-{% endif %}
-{% if kube_token_auth|default(true) %}
-  token-auth-file: {{ kube_token_dir }}/known_tokens.csv
-{% endif %}
-{% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %}
-  oidc-issuer-url: {{ kube_oidc_url }}
-  oidc-client-id: {{ kube_oidc_client_id }}
-{%   if kube_oidc_ca_file is defined %}
-  oidc-ca-file: {{ kube_oidc_ca_file }}
-{%   endif %}
-{%   if kube_oidc_username_claim is defined %}
-  oidc-username-claim: {{ kube_oidc_username_claim }}
-{%   endif %}
-{%   if kube_oidc_groups_claim is defined %}
-  oidc-groups-claim: {{ kube_oidc_groups_claim }}
-{%   endif %}
-{%   if kube_oidc_username_prefix is defined %}
-  oidc-username-prefix: "{{ kube_oidc_username_prefix }}"
-{%   endif %}
-{%   if kube_oidc_groups_prefix is defined %}
-  oidc-groups-prefix: "{{ kube_oidc_groups_prefix }}"
-{%   endif %}
-{% endif %}
-{% if kube_webhook_token_auth|default(false) %}
-  authentication-token-webhook-config-file: {{ kube_config_dir }}/webhook-token-auth-config.yaml
-{% endif %}
-{% if kube_encrypt_secret_data %}
-  experimental-encryption-provider-config: {{ kube_cert_dir }}/secrets_encryption.yaml
-{% endif %}
-  storage-backend: {{ kube_apiserver_storage_backend }}
-{% if kube_api_runtime_config is defined %}
-  runtime-config: {{ kube_api_runtime_config | join(',') }}
-{% endif %}
-  allow-privileged: "true"
-{% if kubernetes_audit %}
-  audit-log-path: "{{ audit_log_path }}"
-  audit-log-maxage: "{{ audit_log_maxage }}"
-  audit-log-maxbackup: "{{ audit_log_maxbackups }}"
-  audit-log-maxsize: "{{ audit_log_maxsize }}"
-  audit-policy-file: {{ audit_policy_file }}
-{% endif %}
-{% for key in kube_kubeadm_apiserver_extra_args %}
-  {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}"
-{% endfor %}
-{% if kube_feature_gates %}
-  feature-gates: {{ kube_feature_gates|join(',') }}
-{% endif %}
-{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
-  cloud-provider: {{cloud_provider}}
-  cloud-config: {{ kube_config_dir }}/cloud_config
-{% elif cloud_provider is defined and cloud_provider in ["external"] %}
-  cloud-config: {{ kube_config_dir }}/cloud_config
-{% endif %}
-{% if kube_network_plugin is defined and kube_network_plugin == 'cloud' %}
-  configure-cloud-routes: "true"
-{% endif %}
-controllerManagerExtraArgs:
-  node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
-  node-monitor-period: {{ kube_controller_node_monitor_period }}
-  pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }}
-  node-cidr-mask-size: "{{ kube_network_node_prefix }}"
-  profiling: "{{ kube_profiling }}"
-  terminated-pod-gc-threshold: "{{ kube_controller_terminated_pod_gc_threshold }}"
-{% if kube_feature_gates %}
-  feature-gates: {{ kube_feature_gates|join(',') }}
-{% endif %}
-{% for key in kube_kubeadm_controller_extra_args %}
-  {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
-{% endfor %}
-{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
-  cloud-provider: {{cloud_provider}}
-  cloud-config: {{ kube_config_dir }}/cloud_config
-{% elif cloud_provider is defined and cloud_provider in ["external"] %}
-  cloud-config: {{ kube_config_dir }}/cloud_config
-{% endif %}
-{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
-controllerManagerExtraVolumes:
-{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined and openstack_cacert != "" %}
-- name: openstackcacert
-  hostPath: "{{ kube_config_dir }}/openstack-cacert.pem"
-  mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
-{% endif %}
-{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
-- name: cloud-config
-  hostPath: {{ kube_config_dir }}/cloud_config
-  mountPath: {{ kube_config_dir }}/cloud_config
-{% endif %}
-{% endif %}
-{% if kubernetes_audit or kube_basic_auth|default(true) or kube_token_auth|default(true) or kube_webhook_token_auth|default(false) or ssl_ca_dirs|length %}
-apiServerExtraVolumes:
-{% if kube_basic_auth|default(true) %}
-- name: basic-auth-config
-  hostPath: {{ kube_users_dir }}
-  mountPath: {{ kube_users_dir }}
-{% endif %}
-{% if kube_token_auth|default(true) %}
-- name: token-auth-config
-  hostPath: {{ kube_token_dir }}
-  mountPath: {{ kube_token_dir }}
-{% endif %}
-{% if kube_webhook_token_auth|default(false) %}
-- name: webhook-token-auth-config
-  hostPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml
-  mountPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml
-{% endif %}
-{% if kubernetes_audit %}
-- name: {{ audit_policy_name }}
-  hostPath: {{ audit_policy_hostpath }}
-  mountPath: {{ audit_policy_mountpath }}
-{% if audit_log_path != "-" %}
-- name: {{ audit_log_name }}
-  hostPath: {{ audit_log_hostpath }}
-  mountPath: {{ audit_log_mountpath }}
-  writable: true
-{% endif %}
-{% endif %}
-{% if ssl_ca_dirs|length %}
-{% for dir in ssl_ca_dirs %}
-- name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
-  hostPath: {{ dir }}
-  mountPath: {{ dir }}
-  writable: false
-{% endfor %}
-{% endif %}
-{% endif %}
-{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
-- name: cloud-config
-  hostPath: {{ kube_config_dir }}/cloud_config
-  mountPath: {{ kube_config_dir }}/cloud_config
-{% endif %}
-schedulerExtraArgs:
-  profiling: "{{ kube_profiling }}"
-{% if kube_feature_gates %}
-  feature-gates: {{ kube_feature_gates|join(',') }}
-{% endif %}
-{% if volume_cross_zone_attachment %}
-  policy-config-file: {{ kube_config_dir }}/kube-scheduler-policy.yaml
-{% endif %}
-{% if kube_kubeadm_scheduler_extra_args|length > 0 %}
-{% for key in kube_kubeadm_scheduler_extra_args %}
-  {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}"
-{% endfor %}
-{% endif %}
-apiServerCertSANs:
-{% for san in apiserver_sans %}
-  - {{ san }}
-{% endfor %}
-certificatesDir: {{ kube_cert_dir }}
-imageRepository: {{ kube_image_repo }}
-unifiedControlPlaneImage: ""
-nodeRegistration:
-{% if kube_override_hostname|default('') %}
-  name: {{ kube_override_hostname }}
-{% endif %}
-{% if inventory_hostname not in groups['kube-node'] %}
-  taints:
-  - effect: NoSchedule
-    key: node-role.kubernetes.io/master
-{% else %}
-  taints: {}
-{% endif %}
-  criSocket: {{ cri_socket }}
-{% if dynamic_kubelet_configuration %}
-featureGates:
-  DynamicKubeletConfig: true
-{% endif %}

roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2

@@ -93,15 +93,11 @@ apiServer:
     insecure-bind-address: {{ kube_apiserver_insecure_bind_address }}
 {% endif %}
     insecure-port: "{{ kube_apiserver_insecure_port }}"
-{% if kube_version is version('v1.10', '<') %}
-    admission-control: {{ kube_apiserver_admission_control | join(',') }}
-{% else %}
 {% if kube_apiserver_enable_admission_plugins|length > 0 %}
     enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }}
 {% endif %}
 {% if kube_apiserver_disable_admission_plugins|length > 0 %}
     disable-admission-plugins: {{ kube_apiserver_disable_admission_plugins | join(',') }}
-{% endif %}
 {% endif %}
     apiserver-count: "{{ kube_apiserver_count }}"
 {% if kube_version is version('v1.9', '>=') %}
@@ -231,11 +227,7 @@ controllerManager:
     node-cidr-mask-size: "{{ kube_network_node_prefix }}"
     profiling: "{{ kube_profiling }}"
     terminated-pod-gc-threshold: "{{ kube_controller_terminated_pod_gc_threshold }}"
-{% if kube_version is version('v1.14', '<') %}
-    address: {{ kube_controller_manager_bind_address }}
-{% else %}
     bind-address: {{ kube_controller_manager_bind_address }}
-{% endif %}
 {% if kube_feature_gates %}
     feature-gates: {{ kube_feature_gates|join(',') }}
 {% endif %}
@@ -272,11 +264,7 @@ controllerManager:
 {% endif %}
 scheduler:
   extraArgs:
-{% if kube_version is version('v1.14', '<') %}
-    address: {{ kube_scheduler_bind_address }}
-{% else %}
     bind-address: {{ kube_scheduler_bind_address }}
-{% endif %}
 {% if kube_feature_gates %}
     feature-gates: {{ kube_feature_gates|join(',') }}
 {% endif %}

roles/kubernetes/master/templates/kubeadm-config.v1beta2.yaml.j2

@@ -1,22 +1,29 @@
-apiVersion: kubeadm.k8s.io/v1alpha3
+apiVersion: kubeadm.k8s.io/v1beta2
 kind: InitConfiguration
-apiEndpoint:
+localAPIEndpoint:
   advertiseAddress: {{ ip | default(fallback_ips[inventory_hostname]) }}
   bindPort: {{ kube_apiserver_port }}
+{% if kubeadm_certificate_key is defined %}
+certificateKey: {{ kubeadm_certificate_key }}
+{% endif %}
 nodeRegistration:
 {% if kube_override_hostname|default('') %}
   name: {{ kube_override_hostname }}
 {% endif %}
-{% if inventory_hostname not in groups['kube-node'] %}
+{% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %}
   taints:
   - effect: NoSchedule
     key: node-role.kubernetes.io/master
 {% else %}
-  taints: {}
+  taints: []
+{% endif %}
+{% if container_manager == 'crio' %}
+  criSocket: /var/run/crio/crio.sock
+{% else %}
+  criSocket: /var/run/dockershim.sock
 {% endif %}
-  criSocket: {{ cri_socket }}
 ---
-apiVersion: kubeadm.k8s.io/v1alpha3
+apiVersion: kubeadm.k8s.io/v1beta2
 kind: ClusterConfiguration
 clusterName: {{ cluster_name }}
 etcd:
@@ -38,212 +45,208 @@ controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.po
 {% else %}
 controlPlaneEndpoint: {{ ip | default(fallback_ips[inventory_hostname]) }}:{{ kube_apiserver_port }}
 {% endif %}
-apiServerCertSANs:
-{% for san in apiserver_sans %}
-  - {{ san }}
-{% endfor %}
 certificatesDir: {{ kube_cert_dir }}
 imageRepository: {{ kube_image_repo }}
-unifiedControlPlaneImage: ""
-apiServerExtraArgs:
+useHyperKubeImage: false
+apiServer:
+  extraArgs:
 {% if kube_api_anonymous_auth is defined and kube_version is version('v1.5', '>=')  %}
-  anonymous-auth: "{{ kube_api_anonymous_auth }}"
+    anonymous-auth: "{{ kube_api_anonymous_auth }}"
 {% endif %}
-  authorization-mode: {{ authorization_modes | join(',') }}
-  bind-address: {{ kube_apiserver_bind_address }}
+    authorization-mode: {{ authorization_modes | join(',') }}
+    bind-address: {{ kube_apiserver_bind_address }}
 {% if kube_apiserver_insecure_port|string != "0" %}
-  insecure-bind-address: {{ kube_apiserver_insecure_bind_address }}
+    insecure-bind-address: {{ kube_apiserver_insecure_bind_address }}
 {% endif %}
-  insecure-port: "{{ kube_apiserver_insecure_port }}"
-{% if kube_version is version('v1.10', '<') %}
-  admission-control: {{ kube_apiserver_admission_control | join(',') }}
-{% else %}
+    insecure-port: "{{ kube_apiserver_insecure_port }}"
 {% if kube_apiserver_enable_admission_plugins|length > 0 %}
-  enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }}
+    enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }}
 {% endif %}
 {% if kube_apiserver_disable_admission_plugins|length > 0 %}
-  disable-admission-plugins: {{ kube_apiserver_disable_admission_plugins | join(',') }}
+    disable-admission-plugins: {{ kube_apiserver_disable_admission_plugins | join(',') }}
 {% endif %}
-{% endif %}
-  apiserver-count: "{{ kube_apiserver_count }}"
+    apiserver-count: "{{ kube_apiserver_count }}"
 {% if kube_version is version('v1.9', '>=') %}
-  endpoint-reconciler-type: lease
+    endpoint-reconciler-type: lease
 {% endif %}
 {% if etcd_events_cluster_enabled %}
-  etcd-servers-overrides: "/events#{{ etcd_events_access_addresses_semicolon }}"
+    etcd-servers-overrides: "/events#{{ etcd_events_access_addresses_semicolon }}"
 {% endif %}
-  service-node-port-range: {{ kube_apiserver_node_port_range }}
-  kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}"
-  profiling: "{{ kube_profiling }}"
-  request-timeout: "{{ kube_apiserver_request_timeout }}"
+    service-node-port-range: {{ kube_apiserver_node_port_range }}
+    kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}"
+    profiling: "{{ kube_profiling }}"
+    request-timeout: "{{ kube_apiserver_request_timeout }}"
+    enable-aggregator-routing: "{{ kube_api_aggregator_routing }}"
 {% if kube_basic_auth|default(true) %}
-  basic-auth-file: {{ kube_users_dir }}/known_users.csv
+    basic-auth-file: {{ kube_users_dir }}/known_users.csv
 {% endif %}
 {% if kube_token_auth|default(true) %}
-  token-auth-file: {{ kube_token_dir }}/known_tokens.csv
+    token-auth-file: {{ kube_token_dir }}/known_tokens.csv
 {% endif %}
 {% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %}
-  oidc-issuer-url: {{ kube_oidc_url }}
-  oidc-client-id: {{ kube_oidc_client_id }}
+    oidc-issuer-url: {{ kube_oidc_url }}
+    oidc-client-id: {{ kube_oidc_client_id }}
 {%   if kube_oidc_ca_file is defined %}
-  oidc-ca-file: {{ kube_oidc_ca_file }}
+    oidc-ca-file: {{ kube_oidc_ca_file }}
 {%   endif %}
 {%   if kube_oidc_username_claim is defined %}
-  oidc-username-claim: {{ kube_oidc_username_claim }}
+    oidc-username-claim: {{ kube_oidc_username_claim }}
 {%   endif %}
 {%   if kube_oidc_groups_claim is defined %}
-  oidc-groups-claim: {{ kube_oidc_groups_claim }}
+    oidc-groups-claim: {{ kube_oidc_groups_claim }}
 {%   endif %}
 {%   if kube_oidc_username_prefix is defined %}
-  oidc-username-prefix: "{{ kube_oidc_username_prefix }}"
+    oidc-username-prefix: "{{ kube_oidc_username_prefix }}"
 {%   endif %}
 {%   if kube_oidc_groups_prefix is defined %}
-  oidc-groups-prefix: "{{ kube_oidc_groups_prefix }}"
+    oidc-groups-prefix: "{{ kube_oidc_groups_prefix }}"
 {%   endif %}
 {% endif %}
 {% if kube_webhook_token_auth|default(false) %}
-  authentication-token-webhook-config-file: {{ kube_config_dir }}/webhook-token-auth-config.yaml
+    authentication-token-webhook-config-file: {{ kube_config_dir }}/webhook-token-auth-config.yaml
 {% endif %}
 {% if kube_encrypt_secret_data %}
-  experimental-encryption-provider-config: {{ kube_cert_dir }}/secrets_encryption.yaml
+    encryption-provider-config: {{ kube_cert_dir }}/secrets_encryption.yaml
 {% endif %}
-  storage-backend: {{ kube_apiserver_storage_backend }}
+    storage-backend: {{ kube_apiserver_storage_backend }}
 {% if kube_api_runtime_config is defined %}
-  runtime-config: {{ kube_api_runtime_config | join(',') }}
+    runtime-config: {{ kube_api_runtime_config | join(',') }}
 {% endif %}
-  allow-privileged: "true"
+    allow-privileged: "true"
 {% if kubernetes_audit %}
-  audit-log-path: "{{ audit_log_path }}"
-  audit-log-maxage: "{{ audit_log_maxage }}"
-  audit-log-maxbackup: "{{ audit_log_maxbackups }}"
-  audit-log-maxsize: "{{ audit_log_maxsize }}"
-  audit-policy-file: {{ audit_policy_file }}
+    audit-log-path: "{{ audit_log_path }}"
+    audit-log-maxage: "{{ audit_log_maxage }}"
+    audit-log-maxbackup: "{{ audit_log_maxbackups }}"
+    audit-log-maxsize: "{{ audit_log_maxsize }}"
+    audit-policy-file: {{ audit_policy_file }}
 {% endif %}
 {% for key in kube_kubeadm_apiserver_extra_args %}
-  {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}"
+    {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}"
 {% endfor %}
 {% if kube_feature_gates %}
-  feature-gates: {{ kube_feature_gates|join(',') }}
+    feature-gates: {{ kube_feature_gates|join(',') }}
 {% endif %}
 {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
-  cloud-provider: {{cloud_provider}}
-  cloud-config: {{ kube_config_dir }}/cloud_config
+    cloud-provider: {{cloud_provider}}
+    cloud-config: {{ kube_config_dir }}/cloud_config
 {% elif cloud_provider is defined and cloud_provider in ["external"] %}
-  cloud-config: {{ kube_config_dir }}/cloud_config
-{% endif %}
-controllerManagerExtraArgs:
-  node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
-  node-monitor-period: {{ kube_controller_node_monitor_period }}
-  pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }}
-  node-cidr-mask-size: "{{ kube_network_node_prefix }}"
-  profiling: "{{ kube_profiling }}"
-  terminated-pod-gc-threshold: "{{ kube_controller_terminated_pod_gc_threshold }}"
-  enable-aggregator-routing: "{{ kube_api_aggregator_routing }}"
-{% if kube_version is version('v1.14', '<') %}
-  address: {{ kube_controller_manager_bind_address }}
-{% else %}
-  bind-address: {{ kube_controller_manager_bind_address }}
-{% endif %}
-{% if kube_feature_gates %}
-  feature-gates: {{ kube_feature_gates|join(',') }}
-{% endif %}
-{% for key in kube_kubeadm_controller_extra_args %}
-  {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
-{% endfor %}
-{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
-  cloud-provider: {{cloud_provider}}
-  cloud-config: {{ kube_config_dir }}/cloud_config
-{% elif cloud_provider is defined and cloud_provider in ["external"] %}
-  cloud-config: {{ kube_config_dir }}/cloud_config
-{% endif %}
-schedulerExtraArgs:
-{% if kube_version is version('v1.14', '<') %}
-  address: {{ kube_scheduler_bind_address }}
-{% else %}
-  bind-address: {{ kube_scheduler_bind_address }}
-{% endif %}
-{% if kube_feature_gates %}
-  feature-gates: {{ kube_feature_gates|join(',') }}
-{% endif %}
-{% if kube_kubeadm_scheduler_extra_args|length > 0 %}
-{% for key in kube_kubeadm_scheduler_extra_args %}
-  {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}"
-{% endfor %}
+    cloud-config: {{ kube_config_dir }}/cloud_config
 {% endif %}
 {% if kubernetes_audit or kube_basic_auth|default(true) or kube_token_auth|default(true) or kube_webhook_token_auth|default(false) or ( cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] ) or apiserver_extra_volumes or ssl_ca_dirs|length %}
-apiServerExtraVolumes:
+  extraVolumes:
 {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
-- name: cloud-config
-  hostPath: {{ kube_config_dir }}/cloud_config
-  mountPath: {{ kube_config_dir }}/cloud_config
+  - name: cloud-config
+    hostPath: {{ kube_config_dir }}/cloud_config
+    mountPath: {{ kube_config_dir }}/cloud_config
 {% endif %}
 {% if kube_basic_auth|default(true) %}
-- name: basic-auth-config
-  hostPath: {{ kube_users_dir }}
-  mountPath: {{ kube_users_dir }}
+  - name: basic-auth-config
+    hostPath: {{ kube_users_dir }}
+    mountPath: {{ kube_users_dir }}
 {% endif %}
 {% if kube_token_auth|default(true) %}
-- name: token-auth-config
-  hostPath: {{ kube_token_dir }}
-  mountPath: {{ kube_token_dir }}
+  - name: token-auth-config
+    hostPath: {{ kube_token_dir }}
+    mountPath: {{ kube_token_dir }}
 {% endif %}
 {% if kube_webhook_token_auth|default(false) %}
-- name: webhook-token-auth-config
-  hostPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml
-  mountPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml
+  - name: webhook-token-auth-config
+    hostPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml
+    mountPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml
 {% endif %}
 {% if kubernetes_audit %}
-- name: {{ audit_policy_name }}
-  hostPath: {{ audit_policy_hostpath }}
-  mountPath: {{ audit_policy_mountpath }}
+  - name: {{ audit_policy_name }}
+    hostPath: {{ audit_policy_hostpath }}
+    mountPath: {{ audit_policy_mountpath }}
 {% if audit_log_path != "-" %}
-- name: {{ audit_log_name }}
-  hostPath: {{ audit_log_hostpath }}
-  mountPath: {{ audit_log_mountpath }}
-  writable: true
+  - name: {{ audit_log_name }}
+    hostPath: {{ audit_log_hostpath }}
+    mountPath: {{ audit_log_mountpath }}
+    readOnly: false
 {% endif %}
 {% endif %}
 {% for volume in apiserver_extra_volumes %}
-- name: {{ volume.name }}
-  hostPath: {{ volume.hostPath }}
-  mountPath: {{ volume.mountPath }}
-  writable: {{ volume.writable | default(false)}}
+  - name: {{ volume.name }}
+    hostPath: {{ volume.hostPath }}
+    mountPath: {{ volume.mountPath }}
+    readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }}
 {% endfor %}
 {% if ssl_ca_dirs|length %}
 {% for dir in ssl_ca_dirs %}
-- name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
-  hostPath: {{ dir }}
-  mountPath: {{ dir }}
-  writable: false
+  - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
+    hostPath: {{ dir }}
+    mountPath: {{ dir }}
+    readOnly: true
 {% endfor %}
 {% endif %}
+{% endif %}
+  certSANs:
+{% for san in apiserver_sans %}
+  - {{ san }}
+{% endfor %}
+  timeoutForControlPlane: 5m0s
+controllerManager:
+  extraArgs:
+    node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
+    node-monitor-period: {{ kube_controller_node_monitor_period }}
+    pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }}
+    node-cidr-mask-size: "{{ kube_network_node_prefix }}"
+    profiling: "{{ kube_profiling }}"
+    terminated-pod-gc-threshold: "{{ kube_controller_terminated_pod_gc_threshold }}"
+    bind-address: {{ kube_controller_manager_bind_address }}
+{% if kube_feature_gates %}
+    feature-gates: {{ kube_feature_gates|join(',') }}
+{% endif %}
+{% for key in kube_kubeadm_controller_extra_args %}
+    {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
+{% endfor %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
+    cloud-provider: {{cloud_provider}}
+    cloud-config: {{ kube_config_dir }}/cloud_config
+{% elif cloud_provider is defined and cloud_provider in ["external"] %}
+    cloud-config: {{ kube_config_dir }}/cloud_config
+{% endif %}
+{% if kube_network_plugin is defined and kube_network_plugin not in ["cloud"] %}
+    configure-cloud-routes: "false"
 {% endif %}
 {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] or controller_manager_extra_volumes %}
-controllerManagerExtraVolumes:
+  extraVolumes:
 {% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %}
-- name: openstackcacert
-  hostPath: "{{ kube_config_dir }}/openstack-cacert.pem"
-  mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
+  - name: openstackcacert
+    hostPath: "{{ kube_config_dir }}/openstack-cacert.pem"
+    mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
 {% endif %}
 {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
-- name: cloud-config
-  hostPath: {{ kube_config_dir }}/cloud_config
-  mountPath: {{ kube_config_dir }}/cloud_config
+  - name: cloud-config
+    hostPath: {{ kube_config_dir }}/cloud_config
+    mountPath: {{ kube_config_dir }}/cloud_config
 {% endif %}
 {% for volume in controller_manager_extra_volumes %}
-- name: {{ volume.name }}
-  hostPath: {{ volume.hostPath }}
-  mountPath: {{ volume.mountPath }}
-  writable: {{ volume.writable | default(false)}}
+  - name: {{ volume.name }}
+    hostPath: {{ volume.hostPath }}
+    mountPath: {{ volume.mountPath }}
+    readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }}
 {% endfor %}
 {% endif %}
+scheduler:
+  extraArgs:
+    bind-address: {{ kube_scheduler_bind_address }}
+{% if kube_feature_gates %}
+    feature-gates: {{ kube_feature_gates|join(',') }}
+{% endif %}
+{% if kube_kubeadm_scheduler_extra_args|length > 0 %}
+{% for key in kube_kubeadm_scheduler_extra_args %}
+    {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}"
+{% endfor %}
+{% endif %}
+  extraVolumes:
 {% if scheduler_extra_volumes %}
-schedulerExtraVolumes:
+  extraVolumes:
 {% for volume in scheduler_extra_volumes %}
-- name: {{ volume.name }}
-  hostPath: {{ volume.hostPath }}
-  mountPath: {{ volume.mountPath }}
-  writable: {{ volume.writable | default(false)}}
+  - name: {{ volume.name }}
+    hostPath: {{ volume.hostPath }}
+    mountPath: {{ volume.mountPath }}
+    readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }}
 {% endfor %}
 {% endif %}
 ---
@@ -259,7 +262,6 @@ clientConnection:
 clusterCIDR: {{ kube_pods_subnet }}
 configSyncPeriod: {{ kube_proxy_config_sync_period }}
 conntrack:
- max: {{ kube_proxy_conntrack_max }}
  maxPerCore: {{ kube_proxy_conntrack_max_per_core }}
  min: {{ kube_proxy_conntrack_min }}
  tcpCloseWaitTimeout: {{ kube_proxy_conntrack_tcp_close_wait_timeout }}
@@ -273,7 +275,7 @@ iptables:
  minSyncPeriod: {{ kube_proxy_min_sync_period }}
  syncPeriod: {{ kube_proxy_sync_period }}
 ipvs:
- excludeCIDRs: {{ "[]" if kube_proxy_exclude_cidrs is not defined or kube_proxy_exclude_cidrs == "null" or kube_proxy_exclude_cidrs | length == 0 else (kube_proxy_exclude_cidrs if kube_proxy_exclude_cidrs[0] == '[' else ("[" + kube_proxy_exclude_cidrs + "]" if (kube_proxy_exclude_cidrs[0] | length) == 1 else "[" + kube_proxy_exclude_cidrs | join(",") + "]")) }}
+ excludeCIDRs: {{ kube_proxy_exclude_cidrs }}
  minSyncPeriod: {{ kube_proxy_min_sync_period }}
  scheduler: {{ kube_proxy_scheduler }}
  syncPeriod: {{ kube_proxy_sync_period }}

roles/kubernetes/master/templates/kubeadm-controlplane.v1beta2.yaml.j2

@@ -0,0 +1,25 @@
+apiVersion: kubeadm.k8s.io/v1beta2
+kind: JoinConfiguration
+discovery:
+  bootstrapToken:
+{% if kubeadm_config_api_fqdn is defined %}
+    apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
+{% else %}
+    apiServerEndpoint: {{ kubeadm_discovery_address | replace("https://", "")}}
+{% endif %}
+    token: {{ kubeadm_token }}
+    unsafeSkipCAVerification: true
+  timeout: {{ discovery_timeout }}
+  tlsBootstrapToken: {{ kubeadm_token }}
+controlPlane:
+  localAPIEndpoint:
+    advertiseAddress: {{ kube_apiserver_address }}
+    bindPort: {{ kube_apiserver_port }}
+  certificateKey: {{ kubeadm_certificate_key }}
+nodeRegistration:
+  name: {{ kube_override_hostname|default(inventory_hostname) }}
+{% if container_manager == 'crio' %}
+  criSocket: /var/run/crio/crio.sock
+{% else %}
+  criSocket: /var/run/dockershim.sock
+{% endif %}