Remove non-kubeadm deployment (#3811)

* Remove non-kubeadm deployment

* More cleanup

* More cleanup

* More cleanup

* More cleanup

* Fix gitlab

* Try stop gce first before absent to make the delete process work

* More cleanup

* Fix bug with checking if kubeadm has already run

* Fix bug with checking if kubeadm has already run

* More fixes

* Fix test

* fix

* Fix gitlab checkout untill kubespray 2.8 is on quay

* Fixed

* Add upgrade path from non-kubeadm to kubeadm. Revert ssl path

* Readd secret checking

* Do gitlab checks from v2.7.0 test upgrade path to 2.8.0

* fix typo

* Fix CI jobs to kubeadm again. Fix broken hyperkube path

* Fix gitlab

* Fix rotate tokens

* More fixes

* More fixes

* Fix tokens

roles/kubernetes/master/handlers/main.yml

@@ -91,13 +91,16 @@
   command: /bin/true
   notify:
     - Master | set secret_changed to true
-    - Master | clear kubeconfig for root user
+    - Master | Copy new kubeconfig for root user
 
 - name: Master | set secret_changed to true
   set_fact:
     secret_changed: true
 
-- name: Master | clear kubeconfig for root user
-  file:
-    path: /root/.kube/config
-    state: absent
+- name: Master | Copy new kubeconfig for root user
+  copy:
+    src: "{{ kube_config_dir }}/admin.conf"
+    dest: "/root/.kube/config"
+    remote_src: yes
+    mode: "0600"
+    backup: yes

roles/kubernetes/master/tasks/kubeadm-cleanup-old-certs.yml

@@ -1,7 +1,7 @@
 ---
 - name: kubeadm | Retrieve files to purge
   find:
-    paths: "{{kube_cert_dir }}"
+    paths: "{{ kube_cert_dir }}"
     patterns: '*.pem'
   register: files_to_purge_for_kubeadm
 

roles/kubernetes/master/tasks/kubeadm-setup.yml

@@ -26,19 +26,22 @@
   file:
     path: "{{ kube_config_dir }}/admin.conf"
     state: absent
-  when: not kubeadm_already_run.stat.exists
+  when:
+    - not kubeadm_already_run.stat.exists
 
 - name: kubeadm | Delete old static pods
   file:
     path: "{{ kube_config_dir }}/manifests/{{item}}.manifest"
     state: absent
   with_items: ["kube-apiserver", "kube-controller-manager", "kube-scheduler", "kube-proxy"]
-  when: old_apiserver_cert.stat.exists
+  when:
+    - old_apiserver_cert.stat.exists
 
 - name: kubeadm | Forcefully delete old static pods
   shell: "docker ps -f name=k8s_{{item}} -q | xargs --no-run-if-empty docker rm -f"
   with_items: ["kube-apiserver", "kube-controller-manager", "kube-scheduler"]
-  when: old_apiserver_cert.stat.exists
+  when:
+    - old_apiserver_cert.stat.exists
 
 - name: kubeadm | aggregate all SANs
   set_fact:
@@ -220,7 +223,8 @@
 
 - name: kubeadm | cleanup old certs if necessary
   import_tasks: kubeadm-cleanup-old-certs.yml
-  when: old_apiserver_cert.stat.exists
+  when:
+    - old_apiserver_cert.stat.exists
 
 - name: kubeadm | Remove taint for master with node role
   command: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf taint node {{ inventory_hostname }} node-role.kubernetes.io/master:NoSchedule-"

roles/kubernetes/master/tasks/main.yml

@@ -4,12 +4,14 @@
     - k8s-pre-upgrade
 
 - import_tasks: users-file.yml
-  when: kube_basic_auth|default(true)
+  when:
+    - kube_basic_auth|default(true)
 
 - import_tasks: encrypt-at-rest.yml
-  when: kube_encrypt_secret_data
+  when:
+    - kube_encrypt_secret_data
 
-- name: install | Copy kubectl binary from download dir
+- name: Install | Copy kubectl binary from download dir
   synchronize:
     src: "{{ local_release_dir }}/hyperkube"
     dest: "{{ bin_dir }}/kubectl"
@@ -57,10 +59,5 @@
     kube_apiserver_enable_admission_plugins: "{{ kube_apiserver_enable_admission_plugins | difference(['SecurityContextDeny']) | union(['PodSecurityPolicy']) | unique }}"
   when: podsecuritypolicy_enabled
 
-- name: Include kubeadm setup if enabled
+- name: Include kubeadm setup
   import_tasks: kubeadm-setup.yml
-  when: kubeadm_enabled|bool|default(false)
-
-- name: Include static pod setup if not using kubeadm
-  import_tasks: static-pod-setup.yml
-  when: not kubeadm_enabled|bool|default(false)

roles/kubernetes/master/tasks/static-pod-setup.yml

@@ -1,59 +0,0 @@
----
-- name: Create audit-policy directory
-  file:
-    path: "{{ audit_policy_file | dirname }}"
-    state: directory
-  tags:
-    - kube-apiserver
-  when: kubernetes_audit|default(false)
-
-- name: Write api audit policy yaml
-  template:
-    src: apiserver-audit-policy.yaml.j2
-    dest: "{{ audit_policy_file }}"
-  notify: Master | Restart apiserver
-  tags:
-    - kube-apiserver
-  when: kubernetes_audit|default(false)
-
-- name: Write kube-apiserver manifest
-  template:
-    src: manifests/kube-apiserver.manifest.j2
-    dest: "{{ kube_manifest_dir }}/kube-apiserver.manifest"
-  notify: Master | Restart apiserver
-  tags:
-    - kube-apiserver
-
-- meta: flush_handlers
-
-- name: Write kube-scheduler kubeconfig
-  template:
-    src: kube-scheduler-kubeconfig.yaml.j2
-    dest: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"
-  tags:
-    - kube-scheduler
-
-- name: Write kube-scheduler manifest
-  template:
-    src: manifests/kube-scheduler.manifest.j2
-    dest: "{{ kube_manifest_dir }}/kube-scheduler.manifest"
-  notify: Master | Restart kube-scheduler
-  tags:
-    - kube-scheduler
-
-- name: Write kube-controller-manager kubeconfig
-  template:
-    src: kube-controller-manager-kubeconfig.yaml.j2
-    dest: "{{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml"
-  tags:
-    - kube-controller-manager
-
-- name: Write kube-controller-manager manifest
-  template:
-    src: manifests/kube-controller-manager.manifest.j2
-    dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest"
-  notify: Master | Restart kube-controller-manager
-  tags:
-    - kube-controller-manager
-
-- meta: flush_handlers

roles/kubernetes/master/tasks/users-file.yml

@@ -12,4 +12,3 @@
     dest: "{{ kube_users_dir }}/known_users.csv"
     mode: 0640
     backup: yes
-  notify: Master | set secret_changed

roles/kubernetes/master/templates/kube-controller-manager-kubeconfig.yaml.j2

@@ -1,18 +0,0 @@
-apiVersion: v1
-kind: Config
-clusters:
-- name: local
-  cluster:
-    certificate-authority: {{ kube_cert_dir }}/ca.pem
-    server: {{ kube_apiserver_endpoint }}
-users:
-- name: kube-controller-manager
-  user:
-    client-certificate: {{ kube_cert_dir }}/kube-controller-manager.pem
-    client-key: {{ kube_cert_dir }}/kube-controller-manager-key.pem
-contexts:
-- context:
-    cluster: local
-    user: kube-controller-manager
-  name: kube-controller-manager-{{ cluster_name }}
-current-context: kube-controller-manager-{{ cluster_name }}

roles/kubernetes/master/templates/kube-scheduler-kubeconfig.yaml.j2

@@ -1,18 +0,0 @@
-apiVersion: v1
-kind: Config
-clusters:
-- name: local
-  cluster:
-    certificate-authority: {{ kube_cert_dir }}/ca.pem
-    server: {{ kube_apiserver_endpoint }}
-users:
-- name: kube-scheduler
-  user:
-    client-certificate: {{ kube_cert_dir }}/kube-scheduler.pem
-    client-key: {{ kube_cert_dir }}/kube-scheduler-key.pem
-contexts:
-- context:
-    cluster: local
-    user: kube-scheduler
-  name: kube-scheduler-{{ cluster_name }}
-current-context: kube-scheduler-{{ cluster_name }}

roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2

@@ -174,7 +174,7 @@ apiServerCertSANs:
 {% for san in  apiserver_sans.split(' ') | unique %}
   - {{ san }}
 {% endfor %}
-certificatesDir: {{ kube_config_dir }}/ssl
+certificatesDir: {{ kube_cert_dir }}
 imageRepository: {{ kube_image_repo }}
 unifiedControlPlaneImage: ""
 {% if kube_override_hostname|default('') %}

roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2

@@ -192,7 +192,7 @@ apiServerCertSANs:
 {% for san in apiserver_sans.split(' ') | unique %}
   - {{ san }}
 {% endfor %}
-certificatesDir: {{ kube_config_dir }}/ssl
+certificatesDir: {{ kube_cert_dir }}
 imageRepository: {{ kube_image_repo }}
 unifiedControlPlaneImage: ""
 nodeRegistration:

roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2

@@ -47,7 +47,7 @@ apiServerCertSANs:
 {% for san in  apiserver_sans.split(' ') | unique %}
   - {{ san }}
 {% endfor %}
-certificatesDir: {{ kube_config_dir }}/ssl
+certificatesDir: {{ kube_cert_dir }}
 imageRepository: {{ kube_image_repo }}
 unifiedControlPlaneImage: ""
 apiServerExtraArgs:

roles/kubernetes/master/templates/kubectl-kubeconfig.yaml.j2

@@ -1,18 +0,0 @@
-apiVersion: v1
-kind: Config
-current-context: kubectl-to-{{ cluster_name }}
-preferences: {}
-clusters:
-- cluster:
-    certificate-authority-data: {{ kube_node_cert|b64encode }}
-    server: {{ kube_apiserver_endpoint }}
-  name: {{ cluster_name }}
-contexts:
-- context:
-    cluster: {{ cluster_name }}
-    user: kubectl
-  name: kubectl-to-{{ cluster_name }}
-users:
-- name: kubectl
-  user:
-    token: {{ kubectl_token }}

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -1,237 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: kube-apiserver
-  namespace: kube-system
-  labels:
-    k8s-app: kube-apiserver
-    kubespray: v2
-  annotations:
-    kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}"
-    kubespray.apiserver-cert/serial: "{{ apiserver_cert_serial }}"
-spec:
-  hostNetwork: true
-{% if kube_version is version('v1.6', '>=')  %}
-  dnsPolicy: ClusterFirst
-{% endif %}
-{% if kube_version is version('v1.11.1', '>=') %}
-  priorityClassName: system-node-critical
-{% endif %}
-  containers:
-  - name: kube-apiserver
-    image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}
-    imagePullPolicy: {{ k8s_image_pull_policy }}
-    resources:
-      limits:
-        cpu: {{ kube_apiserver_cpu_limit }}
-        memory: {{ kube_apiserver_memory_limit }}
-      requests:
-        cpu: {{ kube_apiserver_cpu_requests }}
-        memory: {{ kube_apiserver_memory_requests }}
-    command:
-    - /hyperkube
-    - apiserver
-{% if kubernetes_audit %}
-    - --audit-log-path={{ audit_log_path }}
-    - --audit-log-maxage={{ audit_log_maxage }}
-    - --audit-log-maxbackup={{ audit_log_maxbackups }}
-    - --audit-log-maxsize={{ audit_log_maxsize }}
-    - --audit-policy-file={{ audit_policy_file }}
-{% endif %}
-    - --advertise-address={{ ip | default(ansible_default_ipv4.address) }}
-    - --etcd-servers={{ etcd_access_addresses }}
-{%   if etcd_events_cluster_enabled %}
-    - --etcd-servers-overrides=/events#{{ etcd_events_access_addresses_semicolon }}
-{% endif %}
-{%   if kube_version is version('v1.9', '<')  %}
-    - --etcd-quorum-read=true
-{% endif %}
-    - --etcd-cafile={{ etcd_cert_dir }}/ca.pem
-    - --etcd-certfile={{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem
-    - --etcd-keyfile={{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem
-{% if kube_apiserver_insecure_port|string != "0" %}
-    - --insecure-bind-address={{ kube_apiserver_insecure_bind_address }}
-{% endif %}
-    - --bind-address={{ kube_apiserver_bind_address }}
-    - --apiserver-count={{ kube_apiserver_count }}
-{% if kube_version is version('v1.9', '>=') %}
-    - --endpoint-reconciler-type=lease
-{% endif %}
-{% if kube_version is version('v1.10', '<') %}
-    - --admission-control={{ kube_apiserver_admission_control | join(',') }}
-{% else %}
-{% if kube_apiserver_enable_admission_plugins|length > 0 %}
-    - --enable-admission-plugins={{ kube_apiserver_enable_admission_plugins | join(',') }}
-{% endif %}
-{% if kube_apiserver_disable_admission_plugins|length > 0 %}
-    - --disable-admission-plugins={{ kube_apiserver_disable_admission_plugins | join(',') }}
-{% endif %}
-{% endif %}
-    - --service-cluster-ip-range={{ kube_service_addresses }}
-    - --service-node-port-range={{ kube_apiserver_node_port_range }}
-    - --client-ca-file={{ kube_cert_dir }}/ca.pem
-    - --profiling={{ kube_profiling }}
-    - --repair-malformed-updates=false
-    - --kubelet-client-certificate={{ kube_cert_dir }}/node-{{ inventory_hostname }}.pem
-    - --kubelet-client-key={{ kube_cert_dir }}/node-{{ inventory_hostname }}-key.pem
-    - --service-account-lookup=true
-    - --kubelet-preferred-address-types={{ kubelet_preferred_address_types }}
-    - --request-timeout={{ kube_apiserver_request_timeout }}
-{% if kube_basic_auth|default(true) %}
-    - --basic-auth-file={{ kube_users_dir }}/known_users.csv
-{% endif %}
-    - --tls-cert-file={{ kube_cert_dir }}/apiserver.pem
-    - --tls-private-key-file={{ kube_cert_dir }}/apiserver-key.pem
-{% if kube_token_auth|default(true) %}
-    - --token-auth-file={{ kube_token_dir }}/known_tokens.csv
-{% endif %}
-    - --service-account-key-file={{ kube_cert_dir }}/service-account-key.pem
-{% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %}
-    - --oidc-issuer-url={{ kube_oidc_url }}
-    - --oidc-client-id={{ kube_oidc_client_id }}
-{%   if kube_oidc_ca_file is defined %}
-    - --oidc-ca-file={{ kube_oidc_ca_file }}
-{%   endif %}
-{%   if kube_oidc_username_claim is defined %}
-    - --oidc-username-claim={{ kube_oidc_username_claim }}
-{%   endif %}
-{%   if kube_oidc_username_prefix is defined %}
-    - "--oidc-username-prefix={{ kube_oidc_username_prefix }}"
-{%   endif %}
-{%   if kube_oidc_groups_claim is defined %}
-    - --oidc-groups-claim={{ kube_oidc_groups_claim }}
-{%   endif %}
-{%   if kube_oidc_groups_prefix is defined %}
-    - "--oidc-groups-prefix={{ kube_oidc_groups_prefix }}"
-{%   endif %}
-{% endif %}
-    - --secure-port={{ kube_apiserver_port }}
-    - --insecure-port={{ kube_apiserver_insecure_port }}
-    - --storage-backend={{ kube_apiserver_storage_backend }}
-{% if kube_api_runtime_config is defined %}
-{%   for conf in kube_api_runtime_config %}
-    - --runtime-config={{ conf }}
-{%   endfor %}
-{% endif %}
-{% if enable_network_policy %}
-{%   if kube_version is version('v1.8', '<')  %}
-    - --runtime-config=extensions/v1beta1/networkpolicies=true
-{%   endif %}
-{% endif %}
-    - --v={{ kube_log_level }}
-    - --allow-privileged=true
-{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
-    - --cloud-provider={{ cloud_provider }}
-    - --cloud-config={{ kube_config_dir }}/cloud_config
-{% endif %}
-{% if kube_api_anonymous_auth is defined and kube_version is version('v1.5', '>=')  %}
-    - --anonymous-auth={{ kube_api_anonymous_auth }}
-{% endif %}
-{% if authorization_modes %}
-    - --authorization-mode={{ authorization_modes|join(',') }}
-{% endif %}
-{% if kube_encrypt_secret_data %}
-    - --experimental-encryption-provider-config={{ kube_config_dir }}/ssl/secrets_encryption.yaml
-{% endif %}
-{% if kube_feature_gates %}
-    - --feature-gates={{ kube_feature_gates|join(',') }}
-{% endif %}
-{% if kube_version is version('v1.9', '>=') %}
-    - --requestheader-client-ca-file={{ kube_cert_dir }}/{{ kube_front_proxy_ca }}
-{# FIXME(mattymo): Vault certs do not work with front-proxy-client #}
-{% if cert_management == "vault" %}
-    - --requestheader-allowed-names=
-{% else %}
-    - --requestheader-allowed-names=front-proxy-client
-{% endif %}
-    - --requestheader-extra-headers-prefix=X-Remote-Extra-
-    - --requestheader-group-headers=X-Remote-Group
-    - --requestheader-username-headers=X-Remote-User
-    - --enable-aggregator-routing={{ kube_api_aggregator_routing }}
-    - --proxy-client-cert-file={{ kube_cert_dir }}/front-proxy-client.pem
-    - --proxy-client-key-file={{ kube_cert_dir }}/front-proxy-client-key.pem
-{% else %}
-    - --proxy-client-cert-file={{ kube_cert_dir }}/apiserver.pem
-    - --proxy-client-key-file={{ kube_cert_dir }}/apiserver-key.pem
-{% endif %}
-{% if apiserver_custom_flags is string %}
-    - {{ apiserver_custom_flags }}
-{% else %}
-{% for flag in apiserver_custom_flags %}
-    - {{ flag }}
-{% endfor %}
-{% endif %}
-    livenessProbe:
-      httpGet:
-        host: 127.0.0.1
-        path: /healthz
-{% if kube_apiserver_insecure_port|int == 0 %}
-        port: {{ kube_apiserver_port }}
-        scheme: HTTPS
-{% else %}
-        port: {{ kube_apiserver_insecure_port }}
-{% endif %}
-      failureThreshold: 8
-      initialDelaySeconds: 15
-      periodSeconds: 10
-      successThreshold: 1
-      timeoutSeconds: 15
-    volumeMounts:
-    - mountPath: {{ kube_config_dir }}
-      name: kubernetes-config
-      readOnly: true
-    - mountPath: /etc/ssl
-      name: ssl-certs-host
-      readOnly: true
-{% for dir in ssl_ca_dirs %}
-    - mountPath: {{ dir }}
-      name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
-      readOnly: true
-{% endfor %}
-    - mountPath: {{ etcd_cert_dir }}
-      name: etcd-certs
-      readOnly: true
-{% if cloud_provider is defined and cloud_provider == 'aws' and ansible_os_family == 'RedHat' %}
-    - mountPath: /etc/ssl/certs/ca-bundle.crt
-      name: rhel-ca-bundle
-      readOnly: true
-{% endif %}
-{% if kubernetes_audit %}
-{% if audit_log_path != "-" %}
-    - mountPath: {{ audit_log_mountpath }}
-      name: {{ audit_log_name }}
-      Writable: true
-{% endif %}
-    - mountPath: {{ audit_policy_mountpath }}
-      name: {{ audit_policy_name }}
-{% endif %}
-  volumes:
-  - hostPath:
-      path: {{ kube_config_dir }}
-    name: kubernetes-config
-  - name: ssl-certs-host
-    hostPath:
-      path: /etc/ssl
-{% for dir in ssl_ca_dirs %}
-  - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
-    hostPath:
-      path: {{ dir }}
-{% endfor %}
-  - hostPath:
-      path: {{ etcd_cert_dir }}
-    name: etcd-certs
-{% if cloud_provider is defined and cloud_provider == 'aws' and ansible_os_family == 'RedHat' %}
-  - hostPath:
-      path: /etc/ssl/certs/ca-bundle.crt
-    name: rhel-ca-bundle
-{% endif %}
-{% if kubernetes_audit %}
-{% if audit_log_path != "-" %}
-  - hostPath:
-      path: {{ audit_log_hostpath }}
-    name: {{ audit_log_name }}
-{% endif %}
-  - hostPath:
-      path: {{ audit_policy_hostpath }}
-    name: {{ audit_policy_name }}
-{% endif %}

roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2

@@ -1,132 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: kube-controller-manager
-  namespace: kube-system
-  labels:
-    k8s-app: kube-controller-manager
-  annotations:
-    kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}"
-    kubespray.controller-manager-cert/serial: "{{ controller_manager_cert_serial }}"
-spec:
-  hostNetwork: true
-{% if kube_version is version('v1.6', '>=') %}
-  dnsPolicy: ClusterFirst
-{% endif %}
-{% if kube_version is version('v1.11.1', '>=') %}
-  priorityClassName: system-node-critical
-{% endif %}
-  containers:
-  - name: kube-controller-manager
-    image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}
-    imagePullPolicy: {{ k8s_image_pull_policy }}
-    resources:
-      limits:
-        cpu: {{ kube_controller_cpu_limit }}
-        memory: {{ kube_controller_memory_limit }}
-      requests:
-        cpu: {{ kube_controller_cpu_requests }}
-        memory: {{ kube_controller_memory_requests }}
-    command:
-    - /hyperkube
-    - controller-manager
-    - --kubeconfig={{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml
-    - --leader-elect=true
-    - --service-account-private-key-file={{ kube_cert_dir }}/service-account-key.pem
-    - --root-ca-file={{ kube_cert_dir }}/ca.pem
-    - --cluster-signing-cert-file={{ kube_cert_dir }}/ca.pem
-    - --cluster-signing-key-file={{ kube_cert_dir }}/ca-key.pem
-    - --enable-hostpath-provisioner={{ kube_hostpath_dynamic_provisioner }}
-    - --node-monitor-grace-period={{ kube_controller_node_monitor_grace_period }}
-    - --node-monitor-period={{ kube_controller_node_monitor_period }}
-    - --pod-eviction-timeout={{ kube_controller_pod_eviction_timeout }}
-    - --profiling={{ kube_profiling }}
-    - --terminated-pod-gc-threshold={{ kube_controller_terminated_pod_gc_threshold }}
-    - --v={{ kube_log_level }}
-{% if rbac_enabled %}
-    - --use-service-account-credentials=true
-{% endif %}
-{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
-    - --cloud-provider={{cloud_provider}}
-    - --cloud-config={{ kube_config_dir }}/cloud_config
-{% elif cloud_provider is defined and cloud_provider in ["external", "oci"] %}
-    - --cloud-provider=external
-{% endif %}
-{% if kube_network_plugin is defined and kube_network_plugin == 'cloud' %}
-    - --configure-cloud-routes=true
-{% else %}
-    - --configure-cloud-routes=false
-{% endif %}
-{% if kube_network_plugin is defined and kube_network_plugin in ["cloud", "flannel", "canal", "cilium", "kube-router"] %}
-    - --allocate-node-cidrs=true
-    - --cluster-cidr={{ kube_pods_subnet }}
-    - --service-cluster-ip-range={{ kube_service_addresses }}
-    - --node-cidr-mask-size={{ kube_network_node_prefix }}
-{% endif %}
-{% if kube_feature_gates %}
-    - --feature-gates={{ kube_feature_gates|join(',') }}
-{% endif %}
-{% if controller_mgr_custom_flags is string %}
-    - {{ controller_mgr_custom_flags }}
-{% else %}
-{%   for flag in controller_mgr_custom_flags %}
-    - {{ flag }}
-{%   endfor %}
-{% endif %}
-    livenessProbe:
-      httpGet:
-        host: 127.0.0.1
-        path: /healthz
-        port: 10252
-      initialDelaySeconds: 30
-      timeoutSeconds: 10
-    volumeMounts:
-    - mountPath: /etc/ssl
-      name: ssl-certs-host
-      readOnly: true
-{% for dir in ssl_ca_dirs %}
-    - mountPath: {{ dir }}
-      name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
-      readOnly: true
-{% endfor %}
-    - mountPath: "{{kube_config_dir}}/ssl"
-      name: etc-kube-ssl
-      readOnly: true
-    - mountPath: "{{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml"
-      name: kubeconfig
-      readOnly: true
-{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
-    - mountPath: "{{ kube_config_dir }}/cloud_config"
-      name: cloudconfig
-      readOnly: true
-{% endif %}
-{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined and openstack_cacert != "" %}
-    - mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
-      name: openstackcacert
-      readOnly: true
-{% endif %}
-  volumes:
-  - name: ssl-certs-host
-    hostPath:
-      path: /etc/ssl
-{% for dir in ssl_ca_dirs %}
-  - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
-    hostPath:
-      path: {{ dir }}
-{% endfor %}
-  - name: etc-kube-ssl
-    hostPath:
-      path: "{{ kube_config_dir }}/ssl"
-  - name: kubeconfig
-    hostPath:
-      path: "{{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml"
-{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
-  - hostPath:
-      path: "{{ kube_config_dir }}/cloud_config"
-    name: cloudconfig
-{% endif %}
-{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined and openstack_cacert != "" %}
-  - hostPath:
-      path: "{{ kube_config_dir }}/openstack-cacert.pem"
-    name: openstackcacert
-{% endif %}

roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2

@@ -1,82 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: kube-scheduler
-  namespace: kube-system
-  labels:
-    k8s-app: kube-scheduler
-  annotations:
-    kubespray.scheduler-cert/serial: "{{ scheduler_cert_serial }}"
-spec:
-  hostNetwork: true
-{% if kube_version is version('v1.6', '>=') %}
-  dnsPolicy: ClusterFirst
-{% endif %}
-{% if kube_version is version('v1.11.1', '>=') %}
-  priorityClassName: system-node-critical
-{% endif %}
-  containers:
-  - name: kube-scheduler
-    image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}
-    imagePullPolicy: {{ k8s_image_pull_policy }}
-    resources:
-      limits:
-        cpu: {{ kube_scheduler_cpu_limit }}
-        memory: {{ kube_scheduler_memory_limit }}
-      requests:
-        cpu: {{ kube_scheduler_cpu_requests }}
-        memory: {{ kube_scheduler_memory_requests }}
-    command:
-    - /hyperkube
-    - scheduler
-    - --leader-elect=true
-    - --kubeconfig={{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml
-    - --profiling={{ kube_profiling }}
-    - --v={{ kube_log_level }}
-{% if kube_feature_gates %}
-    - --feature-gates={{ kube_feature_gates|join(',') }}
-{% endif %}
-{% if scheduler_custom_flags is string %}
-    - {{ scheduler_custom_flags }}
-{% else %}
-{%   for flag in scheduler_custom_flags %}
-    - {{ flag }}
-{%   endfor %}
-{% endif %}
-    livenessProbe:
-      httpGet:
-        host: 127.0.0.1
-        path: /healthz
-        port: 10251
-      initialDelaySeconds: 30
-      timeoutSeconds: 10
-    volumeMounts:
-    - mountPath: /etc/ssl
-      name: ssl-certs-host
-      readOnly: true
-{% for dir in ssl_ca_dirs %}
-    - mountPath: {{ dir }}
-      name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
-      readOnly: true
-{% endfor %}
-    - mountPath: "{{ kube_config_dir }}/ssl"
-      name: etc-kube-ssl
-      readOnly: true
-    - mountPath: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"
-      name: kubeconfig
-      readOnly: true
-  volumes:
-  - name: ssl-certs-host
-    hostPath:
-      path: /etc/ssl
-{% for dir in ssl_ca_dirs %}
-  - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
-    hostPath:
-      path: {{ dir }}
-{% endfor %}
-  - name: etc-kube-ssl
-    hostPath:
-      path: "{{ kube_config_dir }}/ssl"
-  - name: kubeconfig
-    hostPath:
-      path: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"