Remove access to cluster from anonymous users (#11016)

* feat: add user facing variable with default

* feat: remove rolebinding to anonymous users after init and upgrade

* feat: use file discovery for secondary control plane nodes

* feat: use file discovery for nodes

* fix: do not fail if rolebinding does not exist

* docs: add warning about kube_api_anonymous_auth

* style: improve readability of delegate_to parameter

* refactor: rename discovery kubeconfig file

* test: enable new variable in hardening and upgrade test cases

* docs: add option to config parameters

* test: multiple instances and upgrade

roles/kubernetes/control-plane/defaults/main/main.yml

@@ -240,3 +240,6 @@ kubeadm_upgrade_auto_cert_renewal: true
 kube_apiserver_tracing: false
 kube_apiserver_tracing_endpoint: 0.0.0.0:4317
 kube_apiserver_tracing_sampling_rate_per_million: 100
+
+# Enable kubeadm file discovery if anonymous access has been removed
+kubeadm_use_file_discovery: "{{ remove_anonymous_access }}"

roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml

@@ -63,6 +63,26 @@
     - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists
     - not kube_external_ca_mode
 
+- name: Get kubeconfig for join discovery process
+  command: "{{ kubectl }} -n kube-public get cm cluster-info -o jsonpath='{.data.kubeconfig}'"
+  register: kubeconfig_file_discovery
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'] | first }}"
+  when:
+    - kubeadm_use_file_discovery
+    - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists
+
+- name: Copy discovery kubeconfig
+  copy:
+    dest: "{{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml"
+    content: "{{ kubeconfig_file_discovery.stdout }}"
+    owner: "root"
+    mode: 0644
+  when:
+    - inventory_hostname != first_kube_control_plane
+    - kubeadm_use_file_discovery
+    - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists
+
 - name: Joining control plane node to the cluster.
   command: >-
     {{ bin_dir }}/kubeadm join

roles/kubernetes/control-plane/tasks/kubeadm-setup.yml

@@ -221,12 +221,16 @@
     {{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token create {{ kubeadm_token }}
   changed_when: false
   when:
-    - inventory_hostname ==  first_kube_control_plane
+    - inventory_hostname == first_kube_control_plane
     - kubeadm_token is defined
     - kubeadm_refresh_token
   tags:
     - kubeadm_token
 
+- name: Remove binding to anonymous user
+  command: "{{ kubectl }} -n kube-public delete rolebinding kubeadm:bootstrap-signer-clusterinfo --ignore-not-found"
+  when: inventory_hostname == first_kube_control_plane and remove_anonymous_access
+
 - name: Create kubeadm token for joining nodes with 24h expiration (default)
   command: "{{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token create"
   changed_when: false

roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml

@@ -53,6 +53,10 @@
     PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
   notify: Master | restart kubelet
 
+- name: Kubeadm | Remove binding to anonymous user
+  command: "{{ kubectl }} -n kube-public delete rolebinding kubeadm:bootstrap-signer-clusterinfo --ignore-not-found"
+  when: remove_anonymous_access
+
 - name: Kubeadm | clean kubectl cache to refresh api types
   file:
     path: "{{ item }}"

roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2

@@ -1,6 +1,10 @@
 apiVersion: kubeadm.k8s.io/v1beta3
 kind: JoinConfiguration
 discovery:
+{% if kubeadm_use_file_discovery %}
+  file:
+    kubeConfigPath: {{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml
+{% else %}
   bootstrapToken:
 {% if kubeadm_config_api_fqdn is defined %}
     apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
@@ -9,6 +13,7 @@ discovery:
 {% endif %}
     token: {{ kubeadm_token }}
     unsafeSkipCAVerification: true
+{% endif %}
   timeout: {{ discovery_timeout }}
   tlsBootstrapToken: {{ kubeadm_token }}
 controlPlane:

roles/kubernetes/kubeadm/defaults/main.yml

@@ -4,6 +4,9 @@
 discovery_timeout: 60s
 kubeadm_join_timeout: 120s
 
+# Enable kubeadm file discovery if anonymous access has been removed
+kubeadm_use_file_discovery: "{{ remove_anonymous_access }}"
+
 # If non-empty, will use this string as identification instead of the actual hostname
 kube_override_hostname: >-
   {%- if cloud_provider is defined and cloud_provider in ['aws'] -%}

roles/kubernetes/kubeadm/tasks/main.yml

@@ -57,6 +57,24 @@
   set_fact:
     kubeadmConfig_api_version: v1beta3
 
+- name: Get kubeconfig for join discovery process
+  command: "{{ kubectl }} -n kube-public get cm cluster-info -o jsonpath='{.data.kubeconfig}'"
+  register: kubeconfig_file_discovery
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'] | first }}"
+  when: kubeadm_use_file_discovery
+
+- name: Copy discovery kubeconfig
+  copy:
+    dest: "{{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml"
+    content: "{{ kubeconfig_file_discovery.stdout }}"
+    owner: "root"
+    mode: 0644
+  when:
+    - not is_kube_master
+    - not kubelet_conf.stat.exists
+    - kubeadm_use_file_discovery
+
 - name: Create kubeadm client config
   template:
     src: "kubeadm-client.conf.{{ kubeadmConfig_api_version }}.j2"

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2

@@ -2,6 +2,10 @@
 apiVersion: kubeadm.k8s.io/v1beta3
 kind: JoinConfiguration
 discovery:
+{% if kubeadm_use_file_discovery %}
+  file:
+    kubeConfigPath: {{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml
+{% else %}
   bootstrapToken:
 {% if kubeadm_config_api_fqdn is defined %}
     apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
@@ -14,6 +18,7 @@ discovery:
     - sha256:{{ kubeadm_ca_hash.stdout }}
 {% else %}
     unsafeSkipCAVerification: true
+{% endif %}
 {% endif %}
   timeout: {{ discovery_timeout }}
   tlsBootstrapToken: {{ kubeadm_token }}

roles/kubespray-defaults/defaults/main/main.yml

@@ -6,6 +6,8 @@ ansible_ssh_common_args: "{% if 'bastion' in groups['all'] %} -o ProxyCommand='s
 # selinux state
 preinstall_selinux_state: permissive
 
+# Setting this value to false will fail
+# For details, read this comment https://github.com/kubernetes-sigs/kubespray/pull/11016#issuecomment-2004985001
 kube_api_anonymous_auth: true
 
 # Default value, but will be set to true automatically if detected
@@ -50,6 +52,9 @@ kubeadm_join_phases_skip_default: []
 kubeadm_join_phases_skip: >-
  {{ kubeadm_join_phases_skip_default }}
 
+# Set to true to remove the role binding to anonymous users created by kubeadm
+remove_anonymous_access: false
+
 # A string slice of values which specify the addresses to use for NodePorts.
 # Values may be valid IP blocks (e.g. 1.2.3.0/24, 1.2.3.4/32).
 # The default empty string slice ([]) means to use all local addresses.