Move kubernetes/master to kubernetes/control-plane (#7218)

This is a small step to replace "master" with "control-plane" in
Kubespray project.

roles/kubernetes/control-plane/defaults/main/etcd.yml

@@ -0,0 +1,27 @@
+---
+# Note: This does not set up DNS entries. It simply adds the following DNS
+# entries to the certificate
+etcd_cert_alt_names:
+  - "etcd.kube-system.svc.{{ dns_domain }}"
+  - "etcd.kube-system.svc"
+  - "etcd.kube-system"
+  - "etcd"
+etcd_cert_alt_ips: []
+
+etcd_heartbeat_interval: "250"
+etcd_election_timeout: "5000"
+
+# etcd_snapshot_count: "10000"
+
+etcd_metrics: "basic"
+
+## A dictionary of extra environment variables to add to etcd.env, formatted like:
+##  etcd_extra_vars:
+##    var1: "value1"
+##    var2: "value2"
+## Note this is different from the etcd role with ETCD_ prfexi, caps, and underscores
+etcd_extra_vars: {}
+
+# etcd_quota_backend_bytes: "2147483648"
+
+etcd_compaction_retention: "8"

roles/kubernetes/control-plane/defaults/main/kube-proxy.yml

@@ -0,0 +1,122 @@
+---
+# bind address for kube-proxy
+kube_proxy_bind_address: '0.0.0.0'
+
+# acceptContentTypes defines the Accept header sent by clients when connecting to a server, overriding the
+# default value of 'application/json'. This field will control all connections to the server used by a particular
+# client.
+kube_proxy_client_accept_content_types: ''
+
+# burst allows extra queries to accumulate when a client is exceeding its rate.
+kube_proxy_client_burst: 10
+
+# contentType is the content type used when sending data to the server from this client.
+kube_proxy_client_content_type: application/vnd.kubernetes.protobuf
+
+# kubeconfig is the path to a KubeConfig file.
+# Leave as empty string to generate from other fields
+kube_proxy_client_kubeconfig: ''
+
+# qps controls the number of queries per second allowed for this connection.
+kube_proxy_client_qps: 5
+
+# How often configuration from the apiserver is refreshed. Must be greater than 0.
+kube_proxy_config_sync_period: 15m0s
+
+### Conntrack
+# max is the maximum number of NAT connections to track (0 to
+# leave as-is).  This takes precedence over maxPerCore and min.
+kube_proxy_conntrack_max: 'null'
+
+# maxPerCore is the maximum number of NAT connections to track
+# per CPU core (0 to leave the limit as-is and ignore min).
+kube_proxy_conntrack_max_per_core: 32768
+
+# min is the minimum value of connect-tracking records to allocate,
+# regardless of conntrackMaxPerCore (set maxPerCore=0 to leave the limit as-is).
+kube_proxy_conntrack_min: 131072
+
+# tcpCloseWaitTimeout is how long an idle conntrack entry
+# in CLOSE_WAIT state will remain in the conntrack
+# table. (e.g. '60s'). Must be greater than 0 to set.
+kube_proxy_conntrack_tcp_close_wait_timeout: 1h0m0s
+
+# tcpEstablishedTimeout is how long an idle TCP connection will be kept open
+# (e.g. '2s').  Must be greater than 0 to set.
+kube_proxy_conntrack_tcp_established_timeout: 24h0m0s
+
+# Enables profiling via web interface on /debug/pprof handler.
+# Profiling handlers will be handled by metrics server.
+kube_proxy_enable_profiling: false
+
+# bind address for kube-proxy health check
+kube_proxy_healthz_bind_address: 0.0.0.0:10256
+
+# If using the pure iptables proxy, SNAT everything. Note that it breaks any
+# policy engine.
+kube_proxy_masquerade_all: false
+
+# If using the pure iptables proxy, the bit of the fwmark space to mark packets requiring SNAT with.
+# Must be within the range [0, 31].
+kube_proxy_masquerade_bit: 14
+
+# The minimum interval of how often the iptables or ipvs rules can be refreshed as
+# endpoints and services change (e.g. '5s', '1m', '2h22m').
+kube_proxy_min_sync_period: 0s
+
+# The maximum interval of how often iptables or ipvs rules are refreshed (e.g. '5s', '1m', '2h22m').
+# Must be greater than 0.
+kube_proxy_sync_period: 30s
+
+# A comma-separated list of CIDR's which the ipvs proxier should not touch when cleaning up IPVS rules.
+kube_proxy_exclude_cidrs: []
+
+# The ipvs scheduler type when proxy mode is ipvs
+# rr: round-robin
+# lc: least connection
+# dh: destination hashing
+# sh: source hashing
+# sed: shortest expected delay
+# nq: never queue
+kube_proxy_scheduler: rr
+
+# configure arp_ignore and arp_announce to avoid answering ARP queries from kube-ipvs0 interface
+# must be set to true for MetalLB to work
+kube_proxy_strict_arp: false
+
+# kube_proxy_tcp_timeout is the timeout value used for idle IPVS TCP sessions.
+# The default value is 0, which preserves the current timeout value on the system.
+kube_proxy_tcp_timeout: 0s
+
+# kube_proxy_tcp_fin_timeout is the timeout value used for IPVS TCP sessions after receiving a FIN.
+# The default value is 0, which preserves the current timeout value on the system.
+kube_proxy_tcp_fin_timeout: 0s
+
+# kube_proxy_udp_timeout is the timeout value used for IPVS UDP packets.
+# The default value is 0, which preserves the current timeout value on the system.
+kube_proxy_udp_timeout: 0s
+
+# The IP address and port for the metrics server to serve on
+# (set to 0.0.0.0 for all IPv4 interfaces and `::` for all IPv6 interfaces)
+kube_proxy_metrics_bind_address: 127.0.0.1:10249
+
+# A string slice of values which specify the addresses to use for NodePorts.
+# Values may be valid IP blocks (e.g. 1.2.3.0/24, 1.2.3.4/32).
+# The default empty string slice ([]) means to use all local addresses.
+kube_proxy_nodeport_addresses: >-
+  {%- if kube_proxy_nodeport_addresses_cidr is defined -%}
+  [{{ kube_proxy_nodeport_addresses_cidr }}]
+  {%- else -%}
+  []
+  {%- endif -%}
+
+# oom-score-adj value for kube-proxy process. Values must be within the range [-1000, 1000]
+kube_proxy_oom_score_adj: -999
+
+# portRange is the range of host ports (beginPort-endPort, inclusive) that may be consumed
+# in order to proxy service traffic. If unspecified, 0, or (0-0) then ports will be randomly chosen.
+kube_proxy_port_range: ''
+
+# udpIdleTimeout is how long an idle UDP connection will be kept open (e.g. '250ms', '2s').
+# Must be greater than 0. Only applicable for proxyMode=userspace.
+kube_proxy_udp_idle_timeout: 250ms

roles/kubernetes/control-plane/defaults/main/main.yml

@@ -0,0 +1,198 @@
+---
+# disable upgrade cluster
+upgrade_cluster_setup: false
+
+# Experimental kubeadm etcd deployment mode. Available only for new deployment
+etcd_kubeadm_enabled: false
+
+# change to 0.0.0.0 to enable insecure access from anywhere (not recommended)
+kube_apiserver_insecure_bind_address: 127.0.0.1
+
+# By default the external API listens on all interfaces, this can be changed to
+# listen on a specific address/interface.
+kube_apiserver_bind_address: 0.0.0.0
+
+# A port range to reserve for services with NodePort visibility.
+# Inclusive at both ends of the range.
+kube_apiserver_node_port_range: "30000-32767"
+
+# ETCD backend for k8s data
+kube_apiserver_storage_backend: etcd3
+
+kube_etcd_cacert_file: ca.pem
+kube_etcd_cert_file: node-{{ inventory_hostname }}.pem
+kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem
+
+# Associated interfaces must be reachable by the rest of the cluster, and by
+# CLI/web clients.
+kube_controller_manager_bind_address: 0.0.0.0
+kube_scheduler_bind_address: 0.0.0.0
+
+# Leader election lease durations and timeouts for scheduler and controller-manager
+kube_controller_manager_leader_elect_lease_duration: 15s
+kube_controller_manager_leader_elect_renew_deadline: 10s
+
+kube_scheduler_leader_elect_lease_duration: 15s
+kube_scheduler_leader_elect_renew_deadline: 10s
+
+# discovery_timeout modifies the discovery timeout
+discovery_timeout: 5m0s
+
+# Instruct first master to refresh kubeadm token
+kubeadm_refresh_token: true
+
+# Scale down coredns replicas to 0 if not using coredns dns_mode
+kubeadm_scale_down_coredns_enabled: true
+
+# audit support
+kubernetes_audit: false
+# path to audit log file
+audit_log_path: /var/log/audit/kube-apiserver-audit.log
+# num days
+audit_log_maxage: 30
+# the num of audit logs to retain
+audit_log_maxbackups: 1
+# the max size in MB to retain
+audit_log_maxsize: 100
+# policy file
+audit_policy_file: "{{ kube_config_dir }}/audit-policy/apiserver-audit-policy.yaml"
+# custom audit policy rules (to replace the default ones)
+# audit_policy_custom_rules: |
+#   - level: None
+#     users: []
+#     verbs: []
+#     resources: []
+
+# audit log hostpath
+audit_log_name: audit-logs
+audit_log_hostpath: /var/log/kubernetes/audit
+audit_log_mountpath: "{{ audit_log_path | dirname }}"
+
+# audit policy hostpath
+audit_policy_name: audit-policy
+audit_policy_hostpath: "{{ audit_policy_file | dirname }}"
+audit_policy_mountpath: "{{ audit_policy_hostpath }}"
+
+# audit webhook support
+kubernetes_audit_webhook: false
+
+# path to audit webhook config file
+audit_webhook_config_file: "{{ kube_config_dir }}/audit-policy/apiserver-audit-webhook-config.yaml"
+audit_webhook_server_url: "https://audit.app"
+audit_webhook_server_extra_args: {}
+audit_webhook_mode: batch
+audit_webhook_batch_max_size: 100
+audit_webhook_batch_max_wait: 1s
+
+kube_controller_node_monitor_grace_period: 40s
+kube_controller_node_monitor_period: 5s
+kube_controller_terminated_pod_gc_threshold: 12500
+kube_apiserver_request_timeout: "1m0s"
+kube_apiserver_pod_eviction_not_ready_timeout_seconds: "300"
+kube_apiserver_pod_eviction_unreachable_timeout_seconds: "300"
+
+# 1.10+ admission plugins
+kube_apiserver_enable_admission_plugins: []
+
+# 1.10+ list of disabled admission plugins
+kube_apiserver_disable_admission_plugins: []
+
+# extra runtime config
+kube_api_runtime_config: []
+
+## Enable/Disable Kube API Server Authentication Methods
+kube_token_auth: false
+kube_oidc_auth: false
+kube_webhook_token_auth: false
+kube_webhook_token_auth_url_skip_tls_verify: false
+## Variables for webhook token auth https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication
+# kube_webhook_token_auth_url: https://...
+kube_webhook_authorization: false
+## Variables for webhook token authz https://kubernetes.io/docs/reference/access-authn-authz/webhook/
+# kube_webhook_authorization_url: https://...
+kube_webhook_authorization_url_skip_tls_verify: false
+
+
+## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/
+## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...)
+
+# kube_oidc_url: https:// ...
+# kube_oidc_client_id: kubernetes
+## Optional settings for OIDC
+# kube_oidc_username_claim: sub
+# kube_oidc_username_prefix: oidc:
+# kube_oidc_groups_claim: groups
+# kube_oidc_groups_prefix: oidc:
+# Copy oidc CA file to the following path if needed
+# kube_oidc_ca_file: {{ kube_cert_dir }}/ca.pem
+# Optionally include a base64-encoded oidc CA cert
+# kube_oidc_ca_cert: c3RhY2thYnVzZS5jb20...
+
+# List of the preferred NodeAddressTypes to use for kubelet connections.
+kubelet_preferred_address_types: 'InternalDNS,InternalIP,Hostname,ExternalDNS,ExternalIP'
+
+## Extra args for k8s components passing by kubeadm
+kube_kubeadm_apiserver_extra_args: {}
+kube_kubeadm_controller_extra_args: {}
+kube_kubeadm_scheduler_extra_args: {}
+
+## Extra control plane host volume mounts
+## Example:
+# apiserver_extra_volumes:
+#  - name: name
+#    hostPath: /host/path
+#    mountPath: /mount/path
+#    readOnly: true
+apiserver_extra_volumes: {}
+controller_manager_extra_volumes: {}
+scheduler_extra_volumes: {}
+
+## Encrypting Secret Data at Rest
+kube_encrypt_secret_data: false
+kube_encrypt_token: "{{ lookup('password', credentials_dir + '/kube_encrypt_token.creds length=32 chars=ascii_letters,digits') }}"
+# Must be either: aescbc, secretbox or aesgcm
+kube_encryption_algorithm: "aescbc"
+# Which kubernetes resources to encrypt
+kube_encryption_resources: [secrets]
+
+# If non-empty, will use this string as identification instead of the actual hostname
+kube_override_hostname: >-
+  {%- if cloud_provider is defined and cloud_provider in [ 'aws' ] -%}
+  {%- else -%}
+  {{ inventory_hostname }}
+  {%- endif -%}
+
+secrets_encryption_query: "resources[*].providers[0].{{kube_encryption_algorithm}}.keys[0].secret"
+
+## Support tls min version, Possible values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13.
+# tls_min_version: ""
+
+## Support tls cipher suites.
+# tls_cipher_suites:
+#   - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA
+#   - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
+#   - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
+#   - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA
+#   - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+#   - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+#   - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA
+#   - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA
+#   - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA
+#   - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256
+#   - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
+#   - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA
+#   - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+#   - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305
+#   - TLS_ECDHE_RSA_WITH_RC4_128_SHA
+#   - TLS_RSA_WITH_3DES_EDE_CBC_SHA
+#   - TLS_RSA_WITH_AES_128_CBC_SHA
+#   - TLS_RSA_WITH_AES_128_CBC_SHA256
+#   - TLS_RSA_WITH_AES_128_GCM_SHA256
+#   - TLS_RSA_WITH_AES_256_CBC_SHA
+#   - TLS_RSA_WITH_AES_256_GCM_SHA384
+#   - TLS_RSA_WITH_RC4_128_SHA
+
+## Amount of time to retain events. (default 1h0m0s)
+event_ttl_duration: "1h0m0s"
+##  Force regeneration of kubernetes control plane certificates without the need of bumping the cluster version
+force_certificate_regeneration: false

roles/kubernetes/control-plane/handlers/main.yml

@@ -0,0 +1,141 @@
+---
+- name: Master | restart kubelet
+  command: /bin/true
+  notify:
+    - Master | reload systemd
+    - Master | reload kubelet
+    - Master | wait for master static pods
+
+- name: Master | wait for master static pods
+  command: /bin/true
+  notify:
+    - Master | wait for the apiserver to be running
+    - Master | wait for kube-scheduler
+    - Master | wait for kube-controller-manager
+
+- name: Master | Restart apiserver
+  command: /bin/true
+  notify:
+    - Master | Remove apiserver container docker
+    - Master | Remove apiserver container containerd/crio
+    - Master | wait for the apiserver to be running
+
+- name: Master | Restart kube-scheduler
+  command: /bin/true
+  notify:
+    - Master | Remove scheduler container docker
+    - Master | Remove scheduler container containerd/crio
+    - Master | wait for kube-scheduler
+
+- name: Master | Restart kube-controller-manager
+  command: /bin/true
+  notify:
+    - Master | Remove controller manager container docker
+    - Master | Remove controller manager container containerd/crio
+    - Master | wait for kube-controller-manager
+
+- name: Master | reload systemd
+  systemd:
+    daemon_reload: true
+
+- name: Master | reload kubelet
+  service:
+    name: kubelet
+    state: restarted
+
+- name: Master | Remove apiserver container docker
+  shell: docker ps -af name=k8s_kube-apiserver* -q | xargs --no-run-if-empty docker rm -f
+  register: remove_apiserver_container
+  retries: 10
+  until: remove_apiserver_container.rc == 0
+  delay: 1
+  when: container_manager == "docker"
+
+- name: Master | Remove apiserver container containerd/crio
+  shell: "{{ bin_dir }}/crictl pods --name kube-apiserver* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'"
+  register: remove_apiserver_container
+  retries: 10
+  until: remove_apiserver_container.rc == 0
+  delay: 1
+  when: container_manager in ['containerd', 'crio']
+
+- name: Master | Remove scheduler container docker
+  shell: "{{ docker_bin_dir }}/docker ps -af name=k8s_kube-scheduler* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f"
+  register: remove_scheduler_container
+  retries: 10
+  until: remove_scheduler_container.rc == 0
+  delay: 1
+  when: container_manager == "docker"
+
+- name: Master | Remove scheduler container containerd/crio
+  shell: "{{ bin_dir }}/crictl pods --name kube-scheduler* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'"
+  register: remove_scheduler_container
+  retries: 10
+  until: remove_scheduler_container.rc == 0
+  delay: 1
+  when: container_manager in ['containerd', 'crio']
+
+- name: Master | Remove controller manager container docker
+  shell: "{{ docker_bin_dir }}/docker ps -af name=k8s_kube-controller-manager* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f"
+  register: remove_cm_container
+  retries: 10
+  until: remove_cm_container.rc == 0
+  delay: 1
+  when: container_manager == "docker"
+
+- name: Master | Remove controller manager container containerd/crio
+  shell: "{{ bin_dir }}/crictl pods --name kube-controller-manager* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'"
+  register: remove_cm_container
+  retries: 10
+  until: remove_cm_container.rc == 0
+  delay: 1
+  when: container_manager in ['containerd', 'crio']
+
+- name: Master | wait for kube-scheduler
+  vars:
+    endpoint: "{{ kube_scheduler_bind_address if kube_scheduler_bind_address != '0.0.0.0' else 'localhost' }}"
+  uri:
+    url: https://{{ endpoint }}:10259/healthz
+    validate_certs: no
+  register: scheduler_result
+  until: scheduler_result.status == 200
+  retries: 60
+  delay: 1
+
+- name: Master | wait for kube-controller-manager
+  vars:
+    endpoint: "{{ kube_controller_manager_bind_address if kube_controller_manager_bind_address != '0.0.0.0' else 'localhost' }}"
+  uri:
+    url: https://{{ endpoint }}:10257/healthz
+    validate_certs: no
+  register: controller_manager_result
+  until: controller_manager_result.status == 200
+  retries: 60
+  delay: 1
+
+- name: Master | wait for the apiserver to be running
+  uri:
+    url: "{{ kube_apiserver_endpoint }}/healthz"
+    validate_certs: no
+  register: result
+  until: result.status == 200
+  retries: 60
+  delay: 1
+
+- name: Master | set secret_changed
+  command: /bin/true
+  notify:
+    - Master | set secret_changed to true
+    - Master | Copy new kubeconfig for root user
+
+- name: Master | set secret_changed to true
+  set_fact:
+    secret_changed: true
+
+- name: Master | Copy new kubeconfig for root user
+  copy:
+    src: "{{ kube_config_dir }}/admin.conf"
+    dest: "{{ ansible_env.HOME | default('/root') }}/.kube/config"
+    remote_src: yes
+    mode: "0600"
+    backup: yes

roles/kubernetes/control-plane/meta/main.yml

@@ -0,0 +1,6 @@
+---
+dependencies:
+  - role: kubernetes/tokens
+    when: kube_token_auth
+    tags:
+      - k8s-secrets

roles/kubernetes/control-plane/tasks/encrypt-at-rest.yml

@@ -0,0 +1,39 @@
+---
+- name: Check if secret for encrypting data at rest already exist
+  stat:
+    path: "{{ kube_cert_dir }}/secrets_encryption.yaml"
+  register: secrets_encryption_file
+
+- name: Slurp secrets_encryption file if it exists
+  slurp:
+    src: "{{ kube_cert_dir }}/secrets_encryption.yaml"
+  register: secret_file_encoded
+  when: secrets_encryption_file.stat.exists
+
+- name: Base 64 Decode slurped secrets_encryption.yaml file
+  set_fact:
+    secret_file_decoded: "{{ secret_file_encoded['content'] | b64decode | from_yaml }}"
+  when: secrets_encryption_file.stat.exists
+
+- name: Extract secret value from secrets_encryption.yaml
+  set_fact:
+    kube_encrypt_token_extracted: "{{ secret_file_decoded | json_query(secrets_encryption_query) | first | b64decode }}"
+  when: secrets_encryption_file.stat.exists
+
+- name: Set kube_encrypt_token across master nodes
+  set_fact:
+    kube_encrypt_token: "{{ kube_encrypt_token_extracted }}"
+  delegate_to: "{{ item }}"
+  delegate_facts: true
+  with_inventory_hostnames: kube-master
+  when: kube_encrypt_token_extracted is defined
+
+- name: Write secrets for encrypting secret data at rest
+  template:
+    src: secrets_encryption.yaml.j2
+    dest: "{{ kube_cert_dir }}/secrets_encryption.yaml"
+    owner: root
+    group: "{{ kube_cert_group }}"
+    mode: 0640
+  tags:
+    - kube-apiserver

roles/kubernetes/control-plane/tasks/kubeadm-certificate.yml

@@ -0,0 +1,15 @@
+---
+- name: Backup old certs and keys
+  copy:
+    src: "{{ kube_cert_dir }}/{{ item.src }}"
+    dest: "{{ kube_cert_dir }}/{{ item.dest }}"
+    mode: 0640
+    remote_src: yes
+  with_items:
+    - {src: apiserver.crt, dest: apiserver.crt.old}
+    - {src: apiserver.key, dest: apiserver.key.old}
+    - {src: apiserver-kubelet-client.crt, dest: apiserver-kubelet-client.crt.old}
+    - {src: apiserver-kubelet-client.key, dest: apiserver-kubelet-client.key.old}
+    - {src: front-proxy-client.crt, dest: front-proxy-client.crt.old}
+    - {src: front-proxy-client.key, dest: front-proxy-client.key.old}
+  ignore_errors: yes

roles/kubernetes/control-plane/tasks/kubeadm-cleanup-old-certs.yml

@@ -0,0 +1,17 @@
+---
+- name: kubeadm | Retrieve files to purge
+  find:
+    paths: "{{ kube_cert_dir }}"
+    patterns: '*.pem'
+  register: files_to_purge_for_kubeadm
+
+- name: kubeadm | Purge old certs
+  file:
+    path: "{{ item.path }}"
+    state: absent
+  with_items: "{{ files_to_purge_for_kubeadm.files }}"
+
+- name: kubeadm | Purge old kubeconfig
+  file:
+    path: "{{ ansible_env.HOME | default('/root') }}/.kube/config"
+    state: absent

roles/kubernetes/control-plane/tasks/kubeadm-etcd.yml

@@ -0,0 +1,32 @@
+---
+- name: Calculate etcd cert serial
+  command: "openssl x509 -in {{ kube_cert_dir }}/apiserver-etcd-client.crt -noout -serial"
+  register: "etcd_client_cert_serial_result"
+  changed_when: false
+  tags:
+    - network
+
+- name: Set etcd_client_cert_serial
+  set_fact:
+    etcd_client_cert_serial: "{{ etcd_client_cert_serial_result.stdout.split('=')[1] }}"
+  tags:
+    - network
+
+- name: Ensure etcdctl binary is installed
+  include_tasks: "{{ role_path }}/../../etcd/tasks/install_host.yml"
+  vars:
+    etcd_cluster_setup: true
+  when: etcd_deployment_type == "host" and not etcd_kubeadm_enabled
+
+- name: Ensure etcdctl binary is installed
+  include_tasks: "{{ role_path }}/../../etcd/tasks/install_etcdctl_docker.yml"
+  vars:
+    etcd_cluster_setup: true
+    etcd_retries: 4
+  when:
+    - etcd_deployment_type == "docker" and not etcd_kubeadm_enabled
+
+- name: Ensure etcdctl script is installed
+  import_role:
+    name: etcdctl
+  when: etcd_kubeadm_enabled

roles/kubernetes/control-plane/tasks/kubeadm-fix-apiserver.yml

@@ -0,0 +1,46 @@
+---
+- name: Test if correct apiserver is set in all kubeconfigs
+  shell: >-
+    grep -Fq "{{ kube_apiserver_endpoint }}" {{ kube_config_dir }}/admin.conf &&
+    grep -Fq "{{ kube_apiserver_endpoint }}" {{ kube_config_dir }}/controller-manager.conf &&
+    grep -Fq "{{ kube_apiserver_endpoint }}" {{ kube_config_dir }}/kubelet.conf &&
+    grep -Fq "{{ kube_apiserver_endpoint }}" {{ kube_config_dir }}/scheduler.conf
+  register: kubeconfig_correct_apiserver
+  changed_when: False
+  failed_when: False
+
+- name: Create temporary directory
+  tempfile:
+    state: directory
+  register: kubeconfig_temp_dir
+  when: kubeconfig_correct_apiserver.rc != 0
+
+- name: Generate new kubeconfigs with correct apiserver
+  command: >-
+    {{ bin_dir }}/kubeadm init phase kubeconfig all
+    --config {{ kube_config_dir }}/kubeadm-config.yaml
+    --kubeconfig-dir {{ kubeconfig_temp_dir.path }}
+  when: kubeconfig_correct_apiserver.rc != 0
+
+- name: Copy new kubeconfigs to kube config dir
+  copy:
+    src: "{{ kubeconfig_temp_dir.path }}/{{ item }}"
+    dest: "{{ kube_config_dir }}/{{ item }}"
+    mode: 0640
+    remote_src: yes
+  when: kubeconfig_correct_apiserver.rc != 0
+  with_items:
+    - admin.conf
+    - controller-manager.conf
+    - kubelet.conf
+    - scheduler.conf
+  notify:
+    - "Master | Restart kube-controller-manager"
+    - "Master | Restart kube-scheduler"
+    - "Master | reload kubelet"
+
+- name: Cleanup temporary directory
+  file:
+    path: "{{ kubeconfig_temp_dir.path }}"
+    state: absent
+  when: kubeconfig_correct_apiserver.rc != 0

roles/kubernetes/control-plane/tasks/kubeadm-migrate-certs.yml

@@ -0,0 +1,21 @@
+---
+- name: Copy old certs to the kubeadm expected path
+  copy:
+    src: "{{ kube_cert_dir }}/{{ item.src }}"
+    dest: "{{ kube_cert_dir }}/{{ item.dest }}"
+    mode: 0640
+    remote_src: yes
+  with_items:
+    - {src: apiserver.pem, dest: apiserver.crt}
+    - {src: apiserver-key.pem, dest: apiserver.key}
+    - {src: ca.pem, dest: ca.crt}
+    - {src: ca-key.pem, dest: ca.key}
+    - {src: front-proxy-ca.pem, dest: front-proxy-ca.crt}
+    - {src: front-proxy-ca-key.pem, dest: front-proxy-ca.key}
+    - {src: front-proxy-client.pem, dest: front-proxy-client.crt}
+    - {src: front-proxy-client-key.pem, dest: front-proxy-client.key}
+    - {src: service-account-key.pem, dest: sa.pub}
+    - {src: service-account-key.pem, dest: sa.key}
+    - {src: "node-{{ inventory_hostname }}.pem", dest: apiserver-kubelet-client.crt}
+    - {src: "node-{{ inventory_hostname }}-key.pem", dest: apiserver-kubelet-client.key}
+  register: kubeadm_copy_old_certs

roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml

@@ -0,0 +1,72 @@
+---
+- name: Set kubeadm_discovery_address
+  set_fact:
+    kubeadm_discovery_address: >-
+      {%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%}
+      {{ first_kube_master }}:{{ kube_apiserver_port }}
+      {%- else -%}
+      {{ kube_apiserver_endpoint | regex_replace('https://', '') }}
+      {%- endif %}
+  tags:
+    - facts
+
+- name: Upload certificates so they are fresh and not expired
+  command: >-
+    {{ bin_dir }}/kubeadm init phase
+    --config {{ kube_config_dir }}/kubeadm-config.yaml
+    upload-certs
+    --upload-certs
+  register: kubeadm_upload_cert
+  when:
+    - inventory_hostname == groups['kube-master']|first
+
+- name: Parse certificate key if not set
+  set_fact:
+    kubeadm_certificate_key: "{{ hostvars[groups['kube-master'][0]]['kubeadm_upload_cert'].stdout_lines[-1] | trim }}"
+  run_once: yes
+  when:
+    - hostvars[groups['kube-master'][0]]['kubeadm_upload_cert'] is defined
+    - hostvars[groups['kube-master'][0]]['kubeadm_upload_cert'] is not skipped
+
+- name: Create kubeadm ControlPlane config
+  template:
+    src: "kubeadm-controlplane.{{ kubeadmConfig_api_version }}.yaml.j2"
+    dest: "{{ kube_config_dir }}/kubeadm-controlplane.yaml"
+    mode: 0640
+    backup: yes
+  when:
+    - inventory_hostname != groups['kube-master']|first
+    - not kubeadm_already_run.stat.exists
+
+- name: Wait for k8s apiserver
+  wait_for:
+    host: "{{ kubeadm_discovery_address.split(':')[0] }}"
+    port: "{{ kubeadm_discovery_address.split(':')[1] }}"
+    timeout: 180
+
+
+- name: check already run
+  debug:
+    msg: "{{ kubeadm_already_run.stat.exists }}"
+
+- name: Joining control plane node to the cluster.
+  shell: >-
+    if [ -f /etc/kubernetes/manifests/kube-apiserver.yaml ]; then
+    {{ bin_dir }}/kubeadm reset -f --cert-dir {{ kube_cert_dir }};
+    fi &&
+    {{ bin_dir }}/kubeadm join
+    --config {{ kube_config_dir }}/kubeadm-controlplane.yaml
+    --ignore-preflight-errors=all
+  register: kubeadm_join_control_plane
+  retries: 3
+  throttle: 1
+  until: kubeadm_join_control_plane is succeeded
+  when:
+    - inventory_hostname != groups['kube-master']|first
+    - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
+
+- name: Set secret_changed to false to avoid extra token rotation
+  set_fact:
+    secret_changed: false

roles/kubernetes/control-plane/tasks/kubeadm-setup.yml

@@ -0,0 +1,232 @@
+---
+- name: kubeadm | Check if old apiserver cert exists on host
+  stat:
+    path: "{{ kube_cert_dir }}/apiserver.pem"
+  register: old_apiserver_cert
+  delegate_to: "{{ groups['kube-master'] | first }}"
+  run_once: true
+
+- name: kubeadm | Migrate old certs if necessary
+  import_tasks: kubeadm-migrate-certs.yml
+  when: old_apiserver_cert.stat.exists
+
+- name: Install OIDC certificate
+  copy:
+    content: "{{ kube_oidc_ca_cert | b64decode }}"
+    dest: "{{ kube_oidc_ca_file }}"
+    owner: root
+    group: root
+    mode: "0644"
+  when:
+    - kube_oidc_auth
+    - kube_oidc_ca_cert is defined
+
+- name: kubeadm | Check serviceaccount key
+  stat:
+    path: "{{ kube_cert_dir }}/sa.key"
+  register: sa_key_before
+  run_once: true
+
+- name: kubeadm | Check if kubeadm has already run
+  stat:
+    path: "/var/lib/kubelet/config.yaml"
+  register: kubeadm_already_run
+
+- name: kubeadm | Delete old admin.conf
+  file:
+    path: "{{ kube_config_dir }}/admin.conf"
+    state: absent
+  when:
+    - not kubeadm_already_run.stat.exists
+
+- name: kubeadm | Delete old static pods
+  file:
+    path: "{{ kube_config_dir }}/manifests/{{ item }}.manifest"
+    state: absent
+  with_items: ["kube-apiserver", "kube-controller-manager", "kube-scheduler", "kube-proxy"]
+  when:
+    - old_apiserver_cert.stat.exists
+
+- name: kubeadm | Forcefully delete old static pods
+  shell: "set -o pipefail && docker ps -f name=k8s_{{ item }} -q | xargs --no-run-if-empty docker rm -f"
+  args:
+    executable: /bin/bash
+  with_items: ["kube-apiserver", "kube-controller-manager", "kube-scheduler"]
+  when:
+    - old_apiserver_cert.stat.exists
+
+- name: kubeadm | aggregate all SANs
+  set_fact:
+    apiserver_sans: "{{ (sans_base + groups['kube-master'] + sans_lb + sans_lb_ip + sans_supp + sans_access_ip + sans_ip + sans_address + sans_override + sans_hostname + sans_fqdn) | unique }}"
+  vars:
+    sans_base:
+      - "kubernetes"
+      - "kubernetes.default"
+      - "kubernetes.default.svc"
+      - "kubernetes.default.svc.{{ dns_domain }}"
+      - "{{ kube_apiserver_ip }}"
+      - "localhost"
+      - "127.0.0.1"
+    sans_lb: "{{ [apiserver_loadbalancer_domain_name] if apiserver_loadbalancer_domain_name is defined else [] }}"
+    sans_lb_ip: "{{ [loadbalancer_apiserver.address] if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined else [] }}"
+    sans_supp: "{{ supplementary_addresses_in_ssl_keys if supplementary_addresses_in_ssl_keys is defined else [] }}"
+    sans_access_ip: "{{ groups['kube-master'] | map('extract', hostvars, 'access_ip') | list | select('defined') | list }}"
+    sans_ip: "{{ groups['kube-master'] | map('extract', hostvars, 'ip') | list | select('defined') | list }}"
+    sans_address: "{{ groups['kube-master'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | list | select('defined') | list }}"
+    sans_override: "{{ [kube_override_hostname] if kube_override_hostname else [] }}"
+    sans_hostname: "{{ groups['kube-master'] | map('extract', hostvars, ['ansible_hostname']) | list | select('defined') | list }}"
+    sans_fqdn: "{{ groups['kube-master'] | map('extract', hostvars, ['ansible_fqdn']) | list | select('defined') | list }}"
+  tags: facts
+
+- name: Create audit-policy directory
+  file:
+    path: "{{ audit_policy_file | dirname }}"
+    state: directory
+  when: kubernetes_audit|default(false) or kubernetes_audit_webhook|default(false)
+
+- name: Write api audit policy yaml
+  template:
+    src: apiserver-audit-policy.yaml.j2
+    dest: "{{ audit_policy_file }}"
+  when: kubernetes_audit|default(false) or kubernetes_audit_webhook|default(false)
+
+- name: Write api audit webhook config yaml
+  template:
+    src: apiserver-audit-webhook-config.yaml.j2
+    dest: "{{ audit_webhook_config_file }}"
+  when: kubernetes_audit_webhook|default(false)
+
+# Nginx LB(default), If kubeadm_config_api_fqdn is defined, use other LB by kubeadm controlPlaneEndpoint.
+- name: set kubeadm_config_api_fqdn define
+  set_fact:
+    kubeadm_config_api_fqdn: "{{ apiserver_loadbalancer_domain_name|default('lb-apiserver.kubernetes.local') }}"
+  when: loadbalancer_apiserver is defined
+
+- name: kubeadm | set kubeadm version
+  import_tasks: kubeadm-version.yml
+
+- name: kubeadm | Certificate management with kubeadm
+  import_tasks: kubeadm-certificate.yml
+  when:
+    - not upgrade_cluster_setup
+    - kubeadm_already_run.stat.exists
+
+- name: kubeadm | Check if apiserver.crt contains all needed SANs
+  command: openssl x509 -noout -in "{{ kube_cert_dir }}/apiserver.crt" -check{{ item|ipaddr|ternary('ip','host') }} "{{ item }}"
+  with_items: "{{ apiserver_sans }}"
+  register: apiserver_sans_check
+  changed_when: "'does match certificate' not in apiserver_sans_check.stdout"
+  when:
+    - inventory_hostname == groups['kube-master']|first
+    - kubeadm_already_run.stat.exists
+
+- name: kubeadm | regenerate apiserver cert 1/2
+  file:
+    state: absent
+    path: "{{ kube_cert_dir }}/{{ item }}"
+  with_items:
+    - apiserver.crt
+    - apiserver.key
+  when:
+    - inventory_hostname == groups['kube-master']|first
+    - kubeadm_already_run.stat.exists
+    - apiserver_sans_check.changed or force_certificate_regeneration
+
+- name: kubeadm | regenerate apiserver cert 2/2
+  command: >-
+    {{ bin_dir }}/kubeadm
+    init phase certs apiserver
+    --config={{ kube_config_dir }}/kubeadm-config.yaml
+  when:
+    - inventory_hostname == groups['kube-master']|first
+    - kubeadm_already_run.stat.exists
+    - apiserver_sans_check.changed or force_certificate_regeneration
+
+- name: kubeadm | Initialize first master
+  command: >-
+    timeout -k 300s 300s
+    {{ bin_dir }}/kubeadm init
+    --config={{ kube_config_dir }}/kubeadm-config.yaml
+    --ignore-preflight-errors=all
+    --skip-phases={{ kubeadm_init_phases_skip | join(',') }}
+    --upload-certs
+  register: kubeadm_init
+  # Retry is because upload config sometimes fails
+  retries: 3
+  until: kubeadm_init is succeeded or "field is immutable" in kubeadm_init.stderr
+  when: inventory_hostname == groups['kube-master']|first and not kubeadm_already_run.stat.exists
+  failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
+  notify: Master | restart kubelet
+
+- name: set kubeadm certificate key
+  set_fact:
+    kubeadm_certificate_key: "{{ item | regex_search('--certificate-key ([^ ]+)','\\1') | first }}"
+  with_items: "{{ hostvars[groups['kube-master'][0]]['kubeadm_init'].stdout_lines | default([]) }}"
+  when:
+    - kubeadm_certificate_key is not defined
+    - (item | trim) is match('.*--certificate-key.*')
+
+- name: Create hardcoded kubeadm token for joining nodes with 24h expiration (if defined)
+  shell: >-
+    {{ bin_dir }}/kubeadm --kubeconfig /etc/kubernetes/admin.conf token delete {{ kubeadm_token }} || :;
+    {{ bin_dir }}/kubeadm --kubeconfig /etc/kubernetes/admin.conf token create {{ kubeadm_token }}
+  changed_when: false
+  when:
+    - inventory_hostname == groups['kube-master']|first
+    - kubeadm_token is defined
+    - kubeadm_refresh_token
+  tags:
+    - kubeadm_token
+
+- name: Create kubeadm token for joining nodes with 24h expiration (default)
+  command: "{{ bin_dir }}/kubeadm --kubeconfig /etc/kubernetes/admin.conf token create"
+  changed_when: false
+  register: temp_token
+  retries: 5
+  delay: 5
+  until: temp_token is succeeded
+  delegate_to: "{{ groups['kube-master'] | first }}"
+  when: kubeadm_token is not defined
+  tags:
+    - kubeadm_token
+
+- name: Set kubeadm_token
+  set_fact:
+    kubeadm_token: "{{ temp_token.stdout }}"
+  when: temp_token.stdout is defined
+  tags:
+    - kubeadm_token
+
+- name: kubeadm | Join other masters
+  include_tasks: kubeadm-secondary.yml
+
+- name: kubeadm | upgrade kubernetes cluster
+  include_tasks: kubeadm-upgrade.yml
+  when:
+    - upgrade_cluster_setup
+    - kubeadm_already_run.stat.exists
+
+- name: kubeadm | Check serviceaccount key again
+  stat:
+    path: "{{ kube_cert_dir }}/sa.key"
+  register: sa_key_after
+  run_once: true
+
+- name: kubeadm | Set secret_changed if service account key was updated
+  command: /bin/true
+  notify: Master | set secret_changed
+  when: sa_key_before.stat.checksum|default("") != sa_key_after.stat.checksum
+
+- name: kubeadm | cleanup old certs if necessary
+  import_tasks: kubeadm-cleanup-old-certs.yml
+  when:
+    - old_apiserver_cert.stat.exists
+
+# FIXME(mattymo): from docs: If you don't want to taint your control-plane node, set this field to an empty slice, i.e. `taints: {}` in the YAML file.
+- name: kubeadm | Remove taint for master with node role
+  command: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf taint node {{ inventory_hostname }} node-role.kubernetes.io/master:NoSchedule- node-role.kubernetes.io/control-plane:NoSchedule-"
+  delegate_to: "{{ groups['kube-master'] | first }}"
+  when: inventory_hostname in groups['kube-node']
+  failed_when: false

roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml

@@ -0,0 +1,74 @@
+---
+- name: kubeadm | Check api is up
+  uri:
+    url: "https://{{ ip | default(fallback_ips[inventory_hostname]) }}:{{ kube_apiserver_port }}/healthz"
+    validate_certs: false
+  when: inventory_hostname == groups['kube-master']
+  register: _result
+  retries: 60
+  delay: 5
+  until: _result.status == 200
+
+- name: kubeadm | Upgrade first master
+  command: >-
+    timeout -k 600s 600s
+    {{ bin_dir }}/kubeadm
+    upgrade apply -y {{ kube_version }}
+    --config={{ kube_config_dir }}/kubeadm-config.yaml
+    --ignore-preflight-errors=all
+    --allow-experimental-upgrades
+    --etcd-upgrade={{ etcd_kubeadm_enabled | bool | lower }}
+    --force
+  register: kubeadm_upgrade
+  # Retry is because upload config sometimes fails
+  retries: 3
+  until: kubeadm_upgrade.rc == 0
+  when: inventory_hostname == groups['kube-master']|first
+  failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
+  notify: Master | restart kubelet
+
+- name: kubeadm | Upgrade other masters
+  command: >-
+    timeout -k 600s 600s
+    {{ bin_dir }}/kubeadm
+    upgrade apply -y {{ kube_version }}
+    --config={{ kube_config_dir }}/kubeadm-config.yaml
+    --ignore-preflight-errors=all
+    --allow-experimental-upgrades
+    --etcd-upgrade={{ etcd_kubeadm_enabled | bool | lower }}
+    --force
+  register: kubeadm_upgrade
+  when: inventory_hostname != groups['kube-master']|first
+  failed_when:
+    - kubeadm_upgrade.rc != 0
+    - '"field is immutable" not in kubeadm_upgrade.stderr'
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
+  notify: Master | restart kubelet
+
+- name: kubeadm | clean kubectl cache to refresh api types
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+    - /root/.kube/cache
+    - /root/.kube/http-cache
+
+# FIXME: https://github.com/kubernetes/kubeadm/issues/1318
+- name: kubeadm | scale down coredns replicas to 0 if not using coredns dns_mode
+  command: >-
+    {{ bin_dir }}/kubectl
+    --kubeconfig /etc/kubernetes/admin.conf
+    -n kube-system
+    scale deployment/coredns --replicas 0
+  register: scale_down_coredns
+  retries: 6
+  delay: 5
+  until: scale_down_coredns is succeeded
+  run_once: yes
+  when:
+    - kubeadm_scale_down_coredns_enabled
+    - dns_mode not in ['coredns', 'coredns_dual']
+  changed_when: false

roles/kubernetes/control-plane/tasks/kubeadm-version.yml

@@ -0,0 +1,15 @@
+---
+- name: Get the kubeadm version
+  command: "{{ bin_dir }}/kubeadm version -o short"
+  register: kubeadm_output
+  changed_when: false
+
+- name: Set kubeadm api version to v1beta2
+  set_fact:
+    kubeadmConfig_api_version: v1beta2
+
+- name: kubeadm | Create kubeadm config
+  template:
+    src: "kubeadm-config.{{ kubeadmConfig_api_version }}.yaml.j2"
+    dest: "{{ kube_config_dir }}/kubeadm-config.yaml"
+    mode: 0640

roles/kubernetes/control-plane/tasks/main.yml

@@ -0,0 +1,64 @@
+---
+- import_tasks: pre-upgrade.yml
+  tags:
+    - k8s-pre-upgrade
+
+- name: Create webhook token auth config
+  template:
+    src: webhook-token-auth-config.yaml.j2
+    dest: "{{ kube_config_dir }}/webhook-token-auth-config.yaml"
+  when: kube_webhook_token_auth|default(false)
+
+- name: Create webhook authorization config
+  template:
+    src: webhook-authorization-config.yaml.j2
+    dest: "{{ kube_config_dir }}/webhook-authorization-config.yaml"
+  when: kube_webhook_authorization|default(false)
+
+- import_tasks: encrypt-at-rest.yml
+  when:
+    - kube_encrypt_secret_data
+
+- name: Install | Copy kubectl binary from download dir
+  copy:
+    src: "{{ local_release_dir }}/kubectl-{{ kube_version }}-{{ image_arch }}"
+    dest: "{{ bin_dir }}/kubectl"
+    mode: 0755
+    remote_src: true
+  tags:
+    - kubectl
+    - upgrade
+
+- name: Install kubectl bash completion
+  shell: "{{ bin_dir }}/kubectl completion bash >/etc/bash_completion.d/kubectl.sh"
+  when: ansible_os_family in ["Debian","RedHat"]
+  tags:
+    - kubectl
+  ignore_errors: True
+
+- name: Set kubectl bash completion file permissions
+  file:
+    path: /etc/bash_completion.d/kubectl.sh
+    owner: root
+    group: root
+    mode: 0755
+  when: ansible_os_family in ["Debian","RedHat"]
+  tags:
+    - kubectl
+    - upgrade
+  ignore_errors: True
+
+- name: Disable SecurityContextDeny admission-controller and enable PodSecurityPolicy
+  set_fact:
+    kube_apiserver_enable_admission_plugins: "{{ kube_apiserver_enable_admission_plugins | difference(['SecurityContextDeny']) | union(['PodSecurityPolicy']) | unique }}"
+  when: podsecuritypolicy_enabled
+
+- name: Include kubeadm setup
+  import_tasks: kubeadm-setup.yml
+
+- name: Include kubeadm etcd extra tasks
+  include_tasks: kubeadm-etcd.yml
+  when: etcd_kubeadm_enabled
+
+- name: Include kubeadm secondary server apiserver fixes
+  include_tasks: kubeadm-fix-apiserver.yml

roles/kubernetes/control-plane/tasks/pre-upgrade.yml

@@ -0,0 +1,21 @@
+---
+- name: "Pre-upgrade | Delete master manifests if etcd secrets changed"
+  file:
+    path: "/etc/kubernetes/manifests/{{ item }}.manifest"
+    state: absent
+  with_items:
+    - ["kube-apiserver", "kube-controller-manager", "kube-scheduler"]
+  register: kube_apiserver_manifest_replaced
+  when: etcd_secret_changed|default(false)
+
+- name: "Pre-upgrade | Delete master containers forcefully"  # noqa 503
+  shell: "set -o pipefail && docker ps -af name=k8s_{{ item }}* -q | xargs --no-run-if-empty docker rm -f"
+  args:
+    executable: /bin/bash
+  with_items:
+    - ["kube-apiserver", "kube-controller-manager", "kube-scheduler"]
+  when: kube_apiserver_manifest_replaced.changed
+  register: remove_master_container
+  retries: 10
+  until: remove_master_container.rc == 0
+  delay: 1

roles/kubernetes/control-plane/templates/apiserver-audit-policy.yaml.j2

@@ -0,0 +1,129 @@
+apiVersion: audit.k8s.io/v1
+kind: Policy
+rules:
+{% if audit_policy_custom_rules is defined and audit_policy_custom_rules != "" %}
+{{ audit_policy_custom_rules | indent(2, true) }}
+{% else %}
+  # The following requests were manually identified as high-volume and low-risk,
+  # so drop them.
+  - level: None
+    users: ["system:kube-proxy"]
+    verbs: ["watch"]
+    resources:
+      - group: "" # core
+        resources: ["endpoints", "services", "services/status"]
+  - level: None
+    # Ingress controller reads `configmaps/ingress-uid` through the unsecured port.
+    # TODO(#46983): Change this to the ingress controller service account.
+    users: ["system:unsecured"]
+    namespaces: ["kube-system"]
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["configmaps"]
+  - level: None
+    users: ["kubelet"] # legacy kubelet identity
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["nodes", "nodes/status"]
+  - level: None
+    userGroups: ["system:nodes"]
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["nodes", "nodes/status"]
+  - level: None
+    users:
+      - system:kube-controller-manager
+      - system:kube-scheduler
+      - system:serviceaccount:kube-system:endpoint-controller
+    verbs: ["get", "update"]
+    namespaces: ["kube-system"]
+    resources:
+      - group: "" # core
+        resources: ["endpoints"]
+  - level: None
+    users: ["system:apiserver"]
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["namespaces", "namespaces/status", "namespaces/finalize"]
+  # Don't log HPA fetching metrics.
+  - level: None
+    users:
+      - system:kube-controller-manager
+    verbs: ["get", "list"]
+    resources:
+      - group: "metrics.k8s.io"
+  # Don't log these read-only URLs.
+  - level: None
+    nonResourceURLs:
+      - /healthz*
+      - /version
+      - /swagger*
+  # Don't log events requests.
+  - level: None
+    resources:
+      - group: "" # core
+        resources: ["events"]
+  # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data,
+  # so only log at the Metadata level.
+  - level: Metadata
+    resources:
+      - group: "" # core
+        resources: ["secrets", "configmaps"]
+      - group: authentication.k8s.io
+        resources: ["tokenreviews"]
+    omitStages:
+      - "RequestReceived"
+  # Get responses can be large; skip them.
+  - level: Request
+    verbs: ["get", "list", "watch"]
+    resources:
+      - group: "" # core
+      - group: "admissionregistration.k8s.io"
+      - group: "apiextensions.k8s.io"
+      - group: "apiregistration.k8s.io"
+      - group: "apps"
+      - group: "authentication.k8s.io"
+      - group: "authorization.k8s.io"
+      - group: "autoscaling"
+      - group: "batch"
+      - group: "certificates.k8s.io"
+      - group: "extensions"
+      - group: "metrics.k8s.io"
+      - group: "networking.k8s.io"
+      - group: "policy"
+      - group: "rbac.authorization.k8s.io"
+      - group: "settings.k8s.io"
+      - group: "storage.k8s.io"
+    omitStages:
+      - "RequestReceived"
+  # Default level for known APIs
+  - level: RequestResponse
+    resources:
+      - group: "" # core
+      - group: "admissionregistration.k8s.io"
+      - group: "apiextensions.k8s.io"
+      - group: "apiregistration.k8s.io"
+      - group: "apps"
+      - group: "authentication.k8s.io"
+      - group: "authorization.k8s.io"
+      - group: "autoscaling"
+      - group: "batch"
+      - group: "certificates.k8s.io"
+      - group: "extensions"
+      - group: "metrics.k8s.io"
+      - group: "networking.k8s.io"
+      - group: "policy"
+      - group: "rbac.authorization.k8s.io"
+      - group: "settings.k8s.io"
+      - group: "storage.k8s.io"
+    omitStages:
+      - "RequestReceived"
+  # Default level for all other requests.
+  - level: Metadata
+    omitStages:
+      - "RequestReceived"
+{% endif %}

roles/kubernetes/control-plane/templates/apiserver-audit-webhook-config.yaml.j2

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    server: {{ audit_webhook_server_url }}
+{% for key in audit_webhook_server_extra_args %}
+    {{ key }}: "{{ audit_webhook_server_extra_args[key] }}"
+{% endfor %}
+  name: auditsink
+contexts:
+- context:
+    cluster: auditsink
+    user: ""
+  name: default-context
+current-context: default-context
+preferences: {}
+users: []

roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2

@@ -0,0 +1,406 @@
+apiVersion: kubeadm.k8s.io/v1beta2
+kind: InitConfiguration
+{% if kubeadm_token is defined %}
+bootstrapTokens:
+- token: "{{ kubeadm_token }}"
+  description: "kubespray kubeadm bootstrap token"
+  ttl: "24h"
+{% endif %}
+localAPIEndpoint:
+  advertiseAddress: {{ ip | default(fallback_ips[inventory_hostname]) }}
+  bindPort: {{ kube_apiserver_port }}
+{% if kubeadm_certificate_key is defined %}
+certificateKey: {{ kubeadm_certificate_key }}
+{% endif %}
+nodeRegistration:
+{% if kube_override_hostname|default('') %}
+  name: {{ kube_override_hostname }}
+{% endif %}
+{% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %}
+  taints:
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/master
+{% else %}
+  taints: []
+{% endif %}
+  criSocket: {{ cri_socket }}
+{% if cloud_provider is defined and cloud_provider in ["external"] %}
+  kubeletExtraArgs:
+    cloud-provider: external
+{% endif %}
+---
+apiVersion: kubeadm.k8s.io/v1beta2
+kind: ClusterConfiguration
+clusterName: {{ cluster_name }}
+etcd:
+{% if not etcd_kubeadm_enabled %}
+  external:
+      endpoints:
+{% for endpoint in etcd_access_addresses.split(',') %}
+      - {{ endpoint }}
+{% endfor %}
+      caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }}
+      certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }}
+      keyFile: {{ etcd_cert_dir }}/{{ kube_etcd_key_file }}
+{% elif etcd_kubeadm_enabled %}
+  local:
+    imageRepository: "{{ etcd_image_repo | regex_replace("/etcd$","") }}"
+    imageTag: "{{ etcd_image_tag }}"
+    dataDir: "{{ etcd_data_dir }}"
+    extraArgs:
+      metrics: {{ etcd_metrics }}
+      election-timeout: "{{ etcd_election_timeout }}"
+      heartbeat-interval: "{{ etcd_heartbeat_interval }}"
+      auto-compaction-retention: "{{ etcd_compaction_retention }}"
+{% if etcd_snapshot_count is defined %}
+      snapshot-count: "{{ etcd_snapshot_count }}"
+{% endif %}
+{% if etcd_quota_backend_bytes is defined %}
+      quota-backend-bytes: "{{ etcd_quota_backend_bytes }}"
+{% endif %}
+{% if etcd_log_package_levels is defined %}
+      log-package_levels: "{{ etcd_log_package_levels }}"
+{% endif %}
+{% for key, value in etcd_extra_vars.items() %}
+      {{ key }}: "{{ value }}"
+{% endfor %}
+{% if host_architecture != "amd64" -%}
+      etcd-unsupported-arch: {{host_architecture}}
+{% endif %}
+    serverCertSANs:
+{% for san in etcd_cert_alt_names %}
+      - {{ san }}
+{% endfor %}
+{% for san in etcd_cert_alt_ips %}
+      - {{ san }}
+{% endfor %}
+    peerCertSANs:
+{% for san in etcd_cert_alt_names %}
+      - {{ san }}
+{% endfor %}
+{% for san in etcd_cert_alt_ips %}
+      - {{ san }}
+{% endfor %}
+{% endif %}
+dns:
+  type: CoreDNS
+  imageRepository: {{ coredns_image_repo | regex_replace('/coredns$','') }}
+  imageTag: {{ coredns_image_tag }}
+networking:
+  dnsDomain: {{ dns_domain }}
+  serviceSubnet: {{ kube_service_addresses }}
+  podSubnet: {{ kube_pods_subnet }}
+kubernetesVersion: {{ kube_version }}
+{% if kubeadm_config_api_fqdn is defined %}
+controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
+{% else %}
+controlPlaneEndpoint: {{ ip | default(fallback_ips[inventory_hostname]) }}:{{ kube_apiserver_port }}
+{% endif %}
+certificatesDir: {{ kube_cert_dir }}
+imageRepository: {{ kube_image_repo }}
+apiServer:
+  extraArgs:
+{% if kube_apiserver_pod_eviction_not_ready_timeout_seconds is defined %}
+    default-not-ready-toleration-seconds: "{{ kube_apiserver_pod_eviction_not_ready_timeout_seconds }}"
+{% endif %}
+{% if kube_apiserver_pod_eviction_unreachable_timeout_seconds is defined %}
+    default-unreachable-toleration-seconds: "{{ kube_apiserver_pod_eviction_unreachable_timeout_seconds }}"
+{% endif %}
+{% if kube_api_anonymous_auth is defined %}
+    anonymous-auth: "{{ kube_api_anonymous_auth }}"
+{% endif %}
+    authorization-mode: {{ authorization_modes | join(',') }}
+    bind-address: {{ kube_apiserver_bind_address }}
+{% if kube_apiserver_insecure_port|string != "0" %}
+    insecure-bind-address: {{ kube_apiserver_insecure_bind_address }}
+{% endif %}
+    insecure-port: "{{ kube_apiserver_insecure_port }}"
+{% if kube_apiserver_enable_admission_plugins|length > 0 %}
+    enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }}
+{% endif %}
+{% if kube_apiserver_disable_admission_plugins|length > 0 %}
+    disable-admission-plugins: {{ kube_apiserver_disable_admission_plugins | join(',') }}
+{% endif %}
+    apiserver-count: "{{ kube_apiserver_count }}"
+    endpoint-reconciler-type: lease
+{% if etcd_events_cluster_enabled %}
+    etcd-servers-overrides: "/events#{{ etcd_events_access_addresses_semicolon }}"
+{% endif %}
+    service-node-port-range: {{ kube_apiserver_node_port_range }}
+    kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}"
+    profiling: "{{ kube_profiling }}"
+    request-timeout: "{{ kube_apiserver_request_timeout }}"
+    enable-aggregator-routing: "{{ kube_api_aggregator_routing }}"
+{% if kube_token_auth|default(true) %}
+    token-auth-file: {{ kube_token_dir }}/known_tokens.csv
+{% endif %}
+{% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %}
+    oidc-issuer-url: "{{ kube_oidc_url }}"
+    oidc-client-id: "{{ kube_oidc_client_id }}"
+{%   if kube_oidc_ca_file is defined %}
+    oidc-ca-file: "{{ kube_oidc_ca_file }}"
+{%   endif %}
+{%   if kube_oidc_username_claim is defined %}
+    oidc-username-claim: "{{ kube_oidc_username_claim }}"
+{%   endif %}
+{%   if kube_oidc_groups_claim is defined %}
+    oidc-groups-claim: "{{ kube_oidc_groups_claim }}"
+{%   endif %}
+{%   if kube_oidc_username_prefix is defined %}
+    oidc-username-prefix: "{{ kube_oidc_username_prefix }}"
+{%   endif %}
+{%   if kube_oidc_groups_prefix is defined %}
+    oidc-groups-prefix: "{{ kube_oidc_groups_prefix }}"
+{%   endif %}
+{% endif %}
+{% if kube_webhook_token_auth|default(false) %}
+    authentication-token-webhook-config-file: {{ kube_config_dir }}/webhook-token-auth-config.yaml
+{% endif %}
+{% if kube_webhook_authorization|default(false) %}
+    authorization-webhook-config-file: {{ kube_config_dir }}/webhook-authorization-config.yaml
+{% endif %}
+{% if kube_encrypt_secret_data %}
+    encryption-provider-config: {{ kube_cert_dir }}/secrets_encryption.yaml
+{% endif %}
+    storage-backend: {{ kube_apiserver_storage_backend }}
+{% if kube_api_runtime_config|length > 0 %}
+    runtime-config: {{ kube_api_runtime_config | join(',') }}
+{% endif %}
+    allow-privileged: "true"
+{% if kubernetes_audit or kubernetes_audit_webhook %}
+    audit-policy-file: {{ audit_policy_file }}
+{% endif %}
+{% if kubernetes_audit %}
+    audit-log-path: "{{ audit_log_path }}"
+    audit-log-maxage: "{{ audit_log_maxage }}"
+    audit-log-maxbackup: "{{ audit_log_maxbackups }}"
+    audit-log-maxsize: "{{ audit_log_maxsize }}"
+{% endif %}
+{% if kubernetes_audit_webhook %}
+    audit-webhook-config-file: {{ audit_webhook_config_file }}
+    audit-webhook-mode: {{ audit_webhook_mode }}
+    audit-webhook-batch-max-size: "{{ audit_webhook_batch_max_size }}"
+    audit-webhook-batch-max-wait: "{{ audit_webhook_batch_max_wait }}"
+{% endif %}
+{% for key in kube_kubeadm_apiserver_extra_args %}
+    {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}"
+{% endfor %}
+{% if kube_feature_gates %}
+    feature-gates: {{ kube_feature_gates|join(',') }}
+{% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
+    cloud-provider: {{ cloud_provider }}
+    cloud-config: {{ kube_config_dir }}/cloud_config
+{% endif %}
+{% if tls_min_version is defined %}
+    tls-min-version: {{ tls_min_version }}
+{% endif %}
+{% if tls_cipher_suites is defined %}
+    tls-cipher-suites: {% for tls in tls_cipher_suites %}{{ tls }}{{ "," if not loop.last else "" }}{% endfor %}
+
+{% endif %}
+{% if event_ttl_duration is defined %}
+    event-ttl: {{ event_ttl_duration }}
+{% endif %}
+{% if kubelet_rotate_server_certificates %}
+    kubelet-certificate-authority: {{ kube_cert_dir }}/ca.crt
+{% endif %}
+{% if kubernetes_audit or kube_token_auth|default(true) or kube_webhook_token_auth|default(false) or ( cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] ) or apiserver_extra_volumes or ssl_ca_dirs|length %}
+  extraVolumes:
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
+  - name: cloud-config
+    hostPath: {{ kube_config_dir }}/cloud_config
+    mountPath: {{ kube_config_dir }}/cloud_config
+{% endif %}
+{% if kube_token_auth|default(true) %}
+  - name: token-auth-config
+    hostPath: {{ kube_token_dir }}
+    mountPath: {{ kube_token_dir }}
+{% endif %}
+{% if kube_webhook_token_auth|default(false) %}
+  - name: webhook-token-auth-config
+    hostPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml
+    mountPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml
+{% endif %}
+{% if kube_webhook_authorization|default(false) %}
+  - name: webhook-authorization-config
+    hostPath: {{ kube_config_dir }}/webhook-authorization-config.yaml
+    mountPath: {{ kube_config_dir }}/webhook-authorization-config.yaml
+{% endif %}
+{% if kubernetes_audit or kubernetes_audit_webhook %}
+  - name: {{ audit_policy_name }}
+    hostPath: {{ audit_policy_hostpath }}
+    mountPath: {{ audit_policy_mountpath }}
+{% if audit_log_path != "-" %}
+  - name: {{ audit_log_name }}
+    hostPath: {{ audit_log_hostpath }}
+    mountPath: {{ audit_log_mountpath }}
+    readOnly: false
+{% endif %}
+{% endif %}
+{% for volume in apiserver_extra_volumes %}
+  - name: {{ volume.name }}
+    hostPath: {{ volume.hostPath }}
+    mountPath: {{ volume.mountPath }}
+    readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }}
+{% endfor %}
+{% if ssl_ca_dirs|length %}
+{% for dir in ssl_ca_dirs %}
+  - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
+    hostPath: {{ dir }}
+    mountPath: {{ dir }}
+    readOnly: true
+{% endfor %}
+{% endif %}
+{% endif %}
+  certSANs:
+{% for san in apiserver_sans %}
+  - {{ san }}
+{% endfor %}
+  timeoutForControlPlane: 5m0s
+controllerManager:
+  extraArgs:
+    node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
+    node-monitor-period: {{ kube_controller_node_monitor_period }}
+    node-cidr-mask-size: "{{ kube_network_node_prefix }}"
+    profiling: "{{ kube_profiling }}"
+    terminated-pod-gc-threshold: "{{ kube_controller_terminated_pod_gc_threshold }}"
+    bind-address: {{ kube_controller_manager_bind_address }}
+    leader-elect-lease-duration: {{ kube_controller_manager_leader_elect_lease_duration }}
+    leader-elect-renew-deadline: {{ kube_controller_manager_leader_elect_renew_deadline }}
+{% if kube_feature_gates %}
+    feature-gates: {{ kube_feature_gates|join(',') }}
+{% endif %}
+{% for key in kube_kubeadm_controller_extra_args %}
+    {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
+{% endfor %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
+    cloud-provider: {{ cloud_provider }}
+    cloud-config: {{ kube_config_dir }}/cloud_config
+{% endif %}
+{% if kube_network_plugin is defined and kube_network_plugin not in ["cloud"] %}
+    configure-cloud-routes: "false"
+{% endif %}
+{% if kubelet_flexvolumes_plugins_dir is defined %}
+    flex-volume-plugin-dir: {{kubelet_flexvolumes_plugins_dir}}
+{% endif %}
+{% if tls_min_version is defined %}
+    tls-min-version: {{ tls_min_version }}
+{% endif %}
+{% if tls_cipher_suites is defined %}
+    tls-cipher-suites: {% for tls in tls_cipher_suites %}{{ tls }}{{ "," if not loop.last else "" }}{% endfor %}
+
+{% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] or controller_manager_extra_volumes %}
+  extraVolumes:
+{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %}
+  - name: openstackcacert
+    hostPath: "{{ kube_config_dir }}/openstack-cacert.pem"
+    mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
+{% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
+  - name: cloud-config
+    hostPath: {{ kube_config_dir }}/cloud_config
+    mountPath: {{ kube_config_dir }}/cloud_config
+{% endif %}
+{% for volume in controller_manager_extra_volumes %}
+  - name: {{ volume.name }}
+    hostPath: {{ volume.hostPath }}
+    mountPath: {{ volume.mountPath }}
+    readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }}
+{% endfor %}
+{% endif %}
+scheduler:
+  extraArgs:
+    bind-address: {{ kube_scheduler_bind_address }}
+    leader-elect-lease-duration: {{ kube_scheduler_leader_elect_lease_duration }}
+    leader-elect-renew-deadline: {{ kube_scheduler_leader_elect_renew_deadline }}
+{% if kube_feature_gates %}
+    feature-gates: {{ kube_feature_gates|join(',') }}
+{% endif %}
+{% if kube_kubeadm_scheduler_extra_args|length > 0 %}
+{% for key in kube_kubeadm_scheduler_extra_args %}
+    {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}"
+{% endfor %}
+{% endif %}
+{% if tls_min_version is defined %}
+    tls-min-version: {{ tls_min_version }}
+{% endif %}
+{% if tls_cipher_suites is defined %}
+    tls-cipher-suites: {% for tls in tls_cipher_suites %}{{ tls }}{{ "," if not loop.last else "" }}{% endfor %}
+
+{% endif %}
+{% if scheduler_extra_volumes %}
+  extraVolumes:
+{% for volume in scheduler_extra_volumes %}
+  - name: {{ volume.name }}
+    hostPath: {{ volume.hostPath }}
+    mountPath: {{ volume.mountPath }}
+    readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }}
+{% endfor %}
+{% endif %}
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+bindAddress: {{ kube_proxy_bind_address }}
+clientConnection:
+  acceptContentTypes: {{ kube_proxy_client_accept_content_types }}
+  burst: {{ kube_proxy_client_burst }}
+  contentType: {{ kube_proxy_client_content_type }}
+  kubeconfig: {{ kube_proxy_client_kubeconfig }}
+  qps: {{ kube_proxy_client_qps }}
+clusterCIDR: {{ kube_pods_subnet }}
+configSyncPeriod: {{ kube_proxy_config_sync_period }}
+conntrack:
+  maxPerCore: {{ kube_proxy_conntrack_max_per_core }}
+  min: {{ kube_proxy_conntrack_min }}
+  tcpCloseWaitTimeout: {{ kube_proxy_conntrack_tcp_close_wait_timeout }}
+  tcpEstablishedTimeout: {{ kube_proxy_conntrack_tcp_established_timeout }}
+enableProfiling: {{ kube_proxy_enable_profiling }}
+healthzBindAddress: {{ kube_proxy_healthz_bind_address }}
+hostnameOverride: {{ kube_override_hostname }}
+iptables:
+  masqueradeAll: {{ kube_proxy_masquerade_all }}
+  masqueradeBit: {{ kube_proxy_masquerade_bit }}
+  minSyncPeriod: {{ kube_proxy_min_sync_period }}
+  syncPeriod: {{ kube_proxy_sync_period }}
+ipvs:
+  excludeCIDRs: {{ kube_proxy_exclude_cidrs }}
+  minSyncPeriod: {{ kube_proxy_min_sync_period }}
+  scheduler: {{ kube_proxy_scheduler }}
+  syncPeriod: {{ kube_proxy_sync_period }}
+  strictARP: {{ kube_proxy_strict_arp }}
+  tcpTimeout: {{ kube_proxy_tcp_timeout }}
+  tcpFinTimeout: {{ kube_proxy_tcp_fin_timeout }}
+  udpTimeout: {{ kube_proxy_udp_timeout }}
+metricsBindAddress: {{ kube_proxy_metrics_bind_address }}
+mode: {{ kube_proxy_mode }}
+nodePortAddresses: {{ kube_proxy_nodeport_addresses }}
+oomScoreAdj: {{ kube_proxy_oom_score_adj }}
+portRange: {{ kube_proxy_port_range }}
+udpIdleTimeout: {{ kube_proxy_udp_idle_timeout }}
+{% if kube_feature_gates %}
+featureGates:
+{% for feature in kube_feature_gates %}
+  {{ feature|replace("=", ": ") }}
+{% endfor %}
+{% endif %}
+{# DNS settings for kubelet #}
+{% if enable_nodelocaldns %}
+{% set kubelet_cluster_dns = [nodelocaldns_ip] %}
+{% elif dns_mode in ['coredns'] %}
+{% set kubelet_cluster_dns = [skydns_server] %}
+{% elif dns_mode == 'coredns_dual' %}
+{% set kubelet_cluster_dns = [skydns_server,skydns_server_secondary] %}
+{% elif dns_mode == 'manual' %}
+{% set kubelet_cluster_dns = [manual_dns_server] %}
+{% else %}
+{% set kubelet_cluster_dns = [] %}
+{% endif %}
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+clusterDNS:
+{% for dns_address in kubelet_cluster_dns %}
+- {{ dns_address }}
+{% endfor %}

roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta2.yaml.j2

@@ -0,0 +1,21 @@
+apiVersion: kubeadm.k8s.io/v1beta2
+kind: JoinConfiguration
+discovery:
+  bootstrapToken:
+{% if kubeadm_config_api_fqdn is defined %}
+    apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
+{% else %}
+    apiServerEndpoint: {{ kubeadm_discovery_address }}
+{% endif %}
+    token: {{ kubeadm_token }}
+    unsafeSkipCAVerification: true
+  timeout: {{ discovery_timeout }}
+  tlsBootstrapToken: {{ kubeadm_token }}
+controlPlane:
+  localAPIEndpoint:
+    advertiseAddress: {{ kube_apiserver_address }}
+    bindPort: {{ kube_apiserver_port }}
+  certificateKey: {{ kubeadm_certificate_key }}
+nodeRegistration:
+  name: {{ kube_override_hostname|default(inventory_hostname) }}
+  criSocket: {{ cri_socket }}

roles/kubernetes/control-plane/templates/secrets_encryption.yaml.j2

@@ -0,0 +1,11 @@
+kind: EncryptionConfig
+apiVersion: v1
+resources:
+  - resources:
+{{ kube_encryption_resources|to_nice_yaml|indent(4, True) }}
+    providers:
+    - {{ kube_encryption_algorithm }}:
+        keys:
+        - name: key
+          secret: {{ kube_encrypt_token | b64encode }}
+    - identity: {}

roles/kubernetes/control-plane/templates/webhook-authorization-config.yaml.j2

@@ -0,0 +1,18 @@
+# clusters refers to the remote service.
+clusters:
+- name: webhook-token-authz-cluster
+  cluster:
+    server: {{ kube_webhook_authorization_url }}
+    insecure-skip-tls-verify: {{ kube_webhook_authorization_url_skip_tls_verify }}
+
+# users refers to the API server's webhook configuration.
+users:
+- name: webhook-token-authz-user
+
+# kubeconfig files require a context. Provide one for the API server.
+current-context: webhook-token-authz
+contexts:
+- context:
+    cluster: webhook-token-authz-cluster
+    user: webhook-token-authz-user
+  name: webhook-token-authz

roles/kubernetes/control-plane/templates/webhook-token-auth-config.yaml.j2

@@ -0,0 +1,18 @@
+# clusters refers to the remote service.
+clusters:
+- name: webhook-token-auth-cluster
+  cluster:
+    server: {{ kube_webhook_token_auth_url }}
+    insecure-skip-tls-verify: {{ kube_webhook_token_auth_url_skip_tls_verify }}
+
+# users refers to the API server's webhook configuration.
+users:
+- name: webhook-token-auth-user
+
+# kubeconfig files require a context. Provide one for the API server.
+current-context: webhook-token-auth
+contexts:
+- context:
+    cluster: webhook-token-auth-cluster
+    user: webhook-token-auth-user
+  name: webhook-token-auth