kubeadm support (#1631)

* kubeadm support

* move k8s master to a subtask
* disable k8s secrets when using kubeadm
* fix etcd cert serial var
* move simple auth users to master role
* make a kubeadm-specific env file for kubelet
* add non-ha CI job

* change ci boolean vars to json format

* fixup

* Update create-gce.yml

* Update create-gce.yml

* Update create-gce.yml

roles/kubernetes/master/defaults/main.yml

@@ -66,3 +66,7 @@ apiserver_custom_flags: []
 controller_mgr_custom_flags: []
 
 scheduler_custom_flags: []
+
+# kubeadm settings
+# Value of 0 means it never expires
+kubeadm_token_ttl: 0

roles/kubernetes/master/handlers/main.yml

@@ -44,3 +44,7 @@
   until: result.status == 200
   retries: 20
   delay: 6
+
+- name: Master | set secret_changed
+  set_fact:
+    secret_changed: true

roles/kubernetes/master/tasks/kubeadm-setup.yml

@@ -0,0 +1,35 @@
+---
+- name: kubeadm | aggregate all SANs
+  set_fact:
+    apiserver_sans: >-
+      kubernetes
+      kubernetes.default
+      kubernetes.default.svc
+      kubernetes.default.svc.{{ dns_domain }}
+      {{ kube_apiserver_ip }}
+      localhost
+      127.0.0.1
+      {{ ' '.join(groups['kube-master']) }}
+      {%- if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %}
+      {{ apiserver_loadbalancer_domain_name }}
+      {%- endif %}
+      {%- for host in groups['kube-master'] -%}
+      {%- if hostvars[host]['access_ip'] is defined %}{{ hostvars[host]['access_ip'] }}{% endif -%}
+      {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+      {%- endfor %}
+  tags: facts
+
+- name: kubeadm | Copy etcd cert dir under k8s cert dir
+  command: "cp -TR {{ etcd_cert_dir }} {{ kube_config_dir }}/ssl/etcd"
+  changed_when: false
+
+- name: kubeadm | Create kubeadm config
+  template:
+    src: kubeadm-config.yaml.j2
+    dest: "{{ kube_config_dir }}/kubeadm-config.yaml"
+  register: kubeadm_config
+
+- name: kubeadm | Initialize cluster
+  command: timeout -k 240s 240s kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --skip-preflight-checks
+  register: kubeadm_init
+  when: kubeadm_config.changed

roles/kubernetes/master/tasks/main.yml

@@ -2,6 +2,9 @@
 - include: pre-upgrade.yml
   tags: k8s-pre-upgrade
 
+- include: users-file.yml
+  when: kube_basic_auth|default(true)
+
 - name: Copy kubectl from hyperkube container
   command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp /hyperkube /systembindir/kubectl"
   register: kube_task_result
@@ -25,63 +28,10 @@
   when: ansible_os_family in ["Debian","RedHat"]
   tags: [kubectl, upgrade]
 
-- name: Write kube-apiserver manifest
-  template:
-    src: manifests/kube-apiserver.manifest.j2
-    dest: "{{ kube_manifest_dir }}/kube-apiserver.manifest"
-  notify: Master | wait for the apiserver to be running
-  tags: kube-apiserver
+- task: Include kubeadm setup if enabled
+  include: kubeadm-setup.yml
+  when: kubeadm_enabled|bool|default(false)
 
-- meta: flush_handlers
-
-- name: Write kube system namespace manifest
-  template:
-    src: namespace.j2
-    dest: "{{kube_config_dir}}/{{system_namespace}}-ns.yml"
-  run_once: yes
-  when: inventory_hostname == groups['kube-master'][0]
-  tags: apps
-
-- name: Check if kube system namespace exists
-  command: "{{ bin_dir }}/kubectl get ns {{system_namespace}}"
-  register: 'kubesystem'
-  changed_when: False
-  failed_when: False
-  run_once: yes
-  tags: apps
-
-- name: Create kube system namespace
-  command: "{{ bin_dir }}/kubectl create -f {{kube_config_dir}}/{{system_namespace}}-ns.yml"
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
-  register: create_system_ns
-  until: create_system_ns.rc == 0
-  changed_when: False
-  when: kubesystem|failed and inventory_hostname == groups['kube-master'][0]
-  tags: apps
-
-- name: Write kube-scheduler kubeconfig
-  template:
-    src: kube-scheduler-kubeconfig.yaml.j2
-    dest: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"
-  tags: kube-scheduler
-
-- name: Write kube-scheduler manifest
-  template:
-    src: manifests/kube-scheduler.manifest.j2
-    dest: "{{ kube_manifest_dir }}/kube-scheduler.manifest"
-  notify: Master | wait for kube-scheduler
-  tags: kube-scheduler
-
-- name: Write kube-controller-manager kubeconfig
-  template:
-    src: kube-controller-manager-kubeconfig.yaml.j2
-    dest: "{{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml"
-  tags: kube-controller-manager
-
-- name: Write kube-controller-manager manifest
-  template:
-    src: manifests/kube-controller-manager.manifest.j2
-    dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest"
-  notify: Master | wait for kube-controller-manager
-  tags: kube-controller-manager
+- task: Include static pod setup if not using kubeadm
+  include: static-pod-setup.yml
+  when: not kubeadm_enabled|bool|default(false)

roles/kubernetes/master/tasks/static-pod-setup.yml

@@ -0,0 +1,61 @@
+---
+- name: Write kube-apiserver manifest
+  template:
+    src: manifests/kube-apiserver.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-apiserver.manifest"
+  notify: Master | wait for the apiserver to be running
+  tags: kube-apiserver
+
+- meta: flush_handlers
+
+- name: Write kube system namespace manifest
+  template:
+    src: namespace.j2
+    dest: "{{kube_config_dir}}/{{system_namespace}}-ns.yml"
+  run_once: yes
+  when: inventory_hostname == groups['kube-master'][0]
+  tags: apps
+
+- name: Check if kube system namespace exists
+  command: "{{ bin_dir }}/kubectl get ns {{system_namespace}}"
+  register: 'kubesystem'
+  changed_when: False
+  failed_when: False
+  run_once: yes
+  tags: apps
+
+- name: Create kube system namespace
+  command: "{{ bin_dir }}/kubectl create -f {{kube_config_dir}}/{{system_namespace}}-ns.yml"
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  register: create_system_ns
+  until: create_system_ns.rc == 0
+  changed_when: False
+  when: kubesystem|failed and inventory_hostname == groups['kube-master'][0]
+  tags: apps
+
+- name: Write kube-scheduler kubeconfig
+  template:
+    src: kube-scheduler-kubeconfig.yaml.j2
+    dest: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"
+  tags: kube-scheduler
+
+- name: Write kube-scheduler manifest
+  template:
+    src: manifests/kube-scheduler.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-scheduler.manifest"
+  notify: Master | wait for kube-scheduler
+  tags: kube-scheduler
+
+- name: Write kube-controller-manager kubeconfig
+  template:
+    src: kube-controller-manager-kubeconfig.yaml.j2
+    dest: "{{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml"
+  tags: kube-controller-manager
+
+- name: Write kube-controller-manager manifest
+  template:
+    src: manifests/kube-controller-manager.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest"
+  notify: Master | wait for kube-controller-manager
+  tags: kube-controller-manager

roles/kubernetes/master/tasks/users-file.yml

@@ -0,0 +1,14 @@
+---
+- name: Make sure the users directory exits
+  file:
+    path: "{{ kube_users_dir }}"
+    state: directory
+    mode: o-rwx
+    group: "{{ kube_cert_group }}"
+
+- name: Populate users for basic auth in API
+  template:
+    src: known_users.csv.j2
+    dest: "{{ kube_users_dir }}/known_users.csv"
+    backup: yes
+  notify: Master | set secret_changed

roles/kubernetes/master/templates/known_users.csv.j2

@@ -0,0 +1,4 @@
+{% for user in kube_users %}
+{{kube_users[user].pass}},{{user}},{{kube_users[user].role}}{% if kube_users[user].groups is defined %},{% set groups_csv = kube_users[user].groups|join(',') -%}"{{groups_csv}}"{% endif %}
+
+{% endfor %}

roles/kubernetes/master/templates/kubeadm-config.yaml.j2

@@ -0,0 +1,67 @@
+apiVersion: kubeadm.k8s.io/v1alpha1
+kind: MasterConfiguration
+api:
+  advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }}
+  bindPort: "{{ kube_apiserver_port }}"
+etcd:
+  endpoints: 
+{% for endpoint in etcd_access_endpoint.split(',') %}
+  - {{ endpoint }}
+{% endfor %}
+  caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem
+  certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem
+  keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem
+networking:
+  dnsDomain: {{ dns_domain }}
+  serviceSubnet: {{ kube_service_addresses }}
+  podSubnet: {{ kube_pods_subnet }}
+kubernetesVersion: {{ kube_version }}
+cloudProvider: {{ cloud_provider|default('') }}
+#TODO: cloud provider conf file
+authorizationModes: 
+{% for mode in authorization_modes %}
+- {{ mode }}
+{% endfor %}
+token: {{ kubeadm_token }}
+tokenTTL: {{ kubeadm_token_ttl }}
+selfHosted: false
+apiServerExtraArgs:
+  insecure-bind-address: {{ kube_apiserver_insecure_bind_address }}
+  insecure-port: "{{ kube_apiserver_insecure_port }}"
+  admission-control: {{ kube_apiserver_admission_control | join(',') }}
+  service-node-port-range: {{ kube_apiserver_node_port_range }}
+{% if kube_basic_auth|default(true) %}
+  basic-auth-file: {{ kube_users_dir }}/known_users.csv
+{% endif %}
+{% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %}
+  oidc-issuer-url: {{ kube_oidc_url }}
+  oidc-client-id: {{ kube_oidc_client_id }}
+{%   if kube_oidc_ca_file is defined %}
+  oidc-ca-file: {{ kube_oidc_ca_file }}
+{%   endif %}
+{%   if kube_oidc_username_claim is defined %}
+  oidc-username-claim: {{ kube_oidc_username_claim }}
+{%   endif %}
+{%   if kube_oidc_groups_claim is defined %}
+  oidc-groups-claim: {{ kube_oidc_groups_claim }}
+{%   endif %}
+{% endif %}
+  storage-backend: {{ kube_apiserver_storage_backend }}
+{% if kube_api_runtime_config is defined %}
+  runtime-config: {{ kube_api_runtime_config }}
+{% endif %}
+  allow-privileged: "true"
+#TODO: Custom flags compatible with kubeadm
+controllerManagerExtraArgs:
+  node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
+  node-monitor-period: {{ kube_controller_node_monitor_period }}
+  pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }}
+{% if kube_feature_gates %}
+  feature-gates: {{ kube_feature_gates|join(',') }}
+{% endif %}
+#schedulerExtraArgs:
+apiServerCertSANs:
+{% for san in  apiserver_sans.split(' ') | unique %}
+  - {{ san }}
+{% endfor %}
+certificatesDir: {{ kube_config_dir }}/ssl

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -7,7 +7,7 @@ metadata:
     k8s-app: kube-apiserver
     kubespray: v2
   annotations:
-    kubespray.etcd-cert/serial: "{{ etcd_node_cert_serial }}"
+    kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}"
     kubespray.apiserver-cert/serial: "{{ apiserver_cert_serial }}"
 spec:
   hostNetwork: true

roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2

@@ -6,7 +6,7 @@ metadata:
   labels:
     k8s-app: kube-controller
   annotations:
-    kubespray.etcd-cert/serial: "{{ etcd_node_cert_serial }}"
+    kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}"
     kubespray.controller-manager-cert/serial: "{{ controller_manager_cert_serial }}"
 spec:
   hostNetwork: true