diff --git a/contrib/azurerm/roles/generate-inventory_2/templates/loadbalancer_vars.j2 b/contrib/azurerm/roles/generate-inventory_2/templates/loadbalancer_vars.j2 index 95a62f327..8a9edad2f 100644 --- a/contrib/azurerm/roles/generate-inventory_2/templates/loadbalancer_vars.j2 +++ b/contrib/azurerm/roles/generate-inventory_2/templates/loadbalancer_vars.j2 @@ -1,8 +1,6 @@ ## External LB example config -apiserver_loadbalancer_domain_name: {{ lb_pubip.dnsSettings.fqdn }} -loadbalancer_apiserver: - address: {{ lb_pubip.ipAddress }} - port: 6443 +kube_apiserver_endpoint: "https://{{ lb_pubip.dnsSettings.fqdn }}:6443" +kube_apiserver_cluster_internal_endpoint: "https://{{ lb_pubip.dnsSettings.fqdn }}:6443" ## Internal loadbalancers for apiservers loadbalancer_apiserver_localhost: false diff --git a/contrib/terraform/aws/README.md b/contrib/terraform/aws/README.md index 28a9f08c4..1e27e8fa3 100644 --- a/contrib/terraform/aws/README.md +++ b/contrib/terraform/aws/README.md @@ -86,8 +86,8 @@ You can use the following set of commands to get the kubeconfig file from your n CONTROLLER_HOST_NAME=$(cat ./inventory/hosts | grep "\[kube_control_plane\]" -A 1 | tail -n 1) CONTROLLER_IP=$(cat ./inventory/hosts | grep $CONTROLLER_HOST_NAME | grep ansible_host | cut -d'=' -f2) -# Get the hostname of the load balancer. -LB_HOST=$(cat inventory/hosts | grep apiserver_loadbalancer_domain_name | cut -d'"' -f2) +# Get the API endpoint of the load balancer. +LB_ENDPOINT=$(cat inventory/hosts | grep kube_apiserver_endpoint | cut -d'"' -f2) # Get the controller's SSH fingerprint. ssh-keygen -R $CONTROLLER_IP > /dev/null 2>&1 @@ -97,7 +97,7 @@ ssh-keyscan -H $CONTROLLER_IP >> ~/.ssh/known_hosts 2>/dev/null mkdir -p ~/.kube ssh -F ssh-bastion.conf centos@$CONTROLLER_IP "sudo chmod 644 /etc/kubernetes/admin.conf" scp -F ssh-bastion.conf centos@$CONTROLLER_IP:/etc/kubernetes/admin.conf ~/.kube/config -sed -i "s^server:.*^server: https://$LB_HOST:6443^" ~/.kube/config +sed -i "s^server:.*^server: $LB_ENDPOINT^" ~/.kube/config kubectl get nodes ``` diff --git a/contrib/terraform/aws/create-infrastructure.tf b/contrib/terraform/aws/create-infrastructure.tf index 810bd16f6..6ea3ad4dd 100644 --- a/contrib/terraform/aws/create-infrastructure.tf +++ b/contrib/terraform/aws/create-infrastructure.tf @@ -170,7 +170,7 @@ data "template_file" "inventory" { list_node = join("\n", aws_instance.k8s-worker.*.private_dns) connection_strings_etcd = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-etcd.*.private_dns, aws_instance.k8s-etcd.*.private_ip)) list_etcd = join("\n", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_dns) : (aws_instance.k8s-master.*.private_dns))) - nlb_api_fqdn = "apiserver_loadbalancer_domain_name=\"${module.aws-nlb.aws_nlb_api_fqdn}\"" + nlb_api_fqdn = "kube_apiserver_endpoint=\"https://${module.aws-nlb.aws_nlb_api_fqdn}:${var.aws_nlb_api_port}\"\nkube_apiserver_cluster_internal_endpoint=\"https://${module.aws-nlb.aws_nlb_api_fqdn}:${var.aws_nlb_api_port}\"" } } diff --git a/docs/CNI/calico.md b/docs/CNI/calico.md index 32d048866..26ebeb9e5 100644 --- a/docs/CNI/calico.md +++ b/docs/CNI/calico.md @@ -379,7 +379,7 @@ To clean up any ipvs leftovers: Calico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.tigera.io/calico/latest/operations/ebpf/enabling-ebpf) for guidelines on how to do this. -Kubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `loadbalancer_apiserver` inventory variable documented in [HA Mode](/docs/operations/ha-mode.md). +Kubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `kube_apiserver_cluster_internal_endpoint` inventory variable documented in [HA Mode](/docs/operations/ha-mode.md). If no external loadbalancer is used, Calico eBPF can also use the localhost loadbalancer option. We are able to do so only if you use the same port for the localhost apiserver loadbalancer and the kube-apiserver. In this case Calico Automatic Host Endpoints need to be enabled to allow services like `coredns` and `metrics-server` to communicate with the kubernetes host endpoint. See [this blog post](https://www.projectcalico.org/securing-kubernetes-nodes-with-calico-automatic-host-endpoints/) on enabling automatic host endpoints. diff --git a/docs/ansible/vars.md b/docs/ansible/vars.md index bd808b1c5..f1e9b03c5 100644 --- a/docs/ansible/vars.md +++ b/docs/ansible/vars.md @@ -47,13 +47,15 @@ Some variables of note include: * *access_ip6* - similarly ``access_ip`` but IPv6 * *ansible_default_ipv6.address* - Not Kubespray-specific, but it is used if ip6 and access_ip6 are undefined -* *loadbalancer_apiserver* - If defined, all hosts will connect to this - address instead of localhost for kube_control_planes and kube_control_plane[0] for - kube_nodes. See more details in the - [HA guide](/docs/operations/ha-mode.md). -* *loadbalancer_apiserver_localhost* - makes all hosts to connect to - the apiserver internally load balanced endpoint. Mutual exclusive to the - `loadbalancer_apiserver`. See more details in the +* *kube_apiserver_endpoint* - Primary API endpoint for external access (e.g., kubectl). + If you have a load balancer or VIP, set this in your inventory (e.g. `https://lb.example.com:6443`). + See more details in the [HA guide](/docs/operations/ha-mode.md). +* *kube_apiserver_cluster_internal_endpoint* - Internal endpoint used for in-cluster + traffic, including nodes and pods. Defaults to localhost LB proxy or `kube_apiserver_endpoint`. + See more details in the [HA guide](/docs/operations/ha-mode.md). +* *loadbalancer_apiserver_localhost* - makes all hosts connect to + the apiserver internally via the local load balanced proxy (nginx/haproxy). Defaults to `true` + when no external or internal endpoint is defined. See more details in the [HA guide](/docs/operations/ha-mode.md). ## Special network variables diff --git a/docs/calico_peer_example/new-york.yml b/docs/calico_peer_example/new-york.yml index 3f44e5593..daa02c9c8 100644 --- a/docs/calico_peer_example/new-york.yml +++ b/docs/calico_peer_example/new-york.yml @@ -11,6 +11,5 @@ # numallowedlocalasnumbers: 0 # sourceaddress: "None" -# loadbalancer_apiserver: -# address: "10.99.0.44" -# port: "8383" +# kube_apiserver_endpoint: "https://10.99.0.44:8383" +# kube_apiserver_cluster_internal_endpoint: "https://10.99.0.44:8383" diff --git a/docs/calico_peer_example/paris.yml b/docs/calico_peer_example/paris.yml index 746a7ddc7..75dd256d7 100644 --- a/docs/calico_peer_example/paris.yml +++ b/docs/calico_peer_example/paris.yml @@ -11,6 +11,5 @@ # numallowedlocalasnumbers: 0 # sourceaddress: "None" -# loadbalancer_apiserver: -# address: "10.99.0.21" -# port: "8383" +# kube_apiserver_endpoint: "https://10.99.0.21:8383" +# kube_apiserver_cluster_internal_endpoint: "https://10.99.0.21:8383" diff --git a/docs/ingress/kube-vip.md b/docs/ingress/kube-vip.md index 2ca975825..8806e4a07 100644 --- a/docs/ingress/kube-vip.md +++ b/docs/ingress/kube-vip.md @@ -25,9 +25,8 @@ You also need to enable # HA for control-plane, requires a VIP kube_vip_controlplane_enabled: true kube_vip_address: 10.42.42.42 -loadbalancer_apiserver: - address: "{{ kube_vip_address }}" - port: 6443 +kube_apiserver_endpoint: "https://{{ kube_vip_address }}:6443" +kube_apiserver_cluster_internal_endpoint: "https://{{ kube_vip_address }}:6443" # kube_vip_interface: ens160 # LoadBalancer for services diff --git a/docs/operations/ha-mode.md b/docs/operations/ha-mode.md index 64e8c025b..29a9f54e1 100644 --- a/docs/operations/ha-mode.md +++ b/docs/operations/ha-mode.md @@ -21,7 +21,7 @@ is less efficient than a dedicated load balancer because it creates extra health checks on the Kubernetes apiserver, but is more practical for scenarios where an external LB or virtual IP management is inconvenient. This option is configured by the variable `loadbalancer_apiserver_localhost` (defaults to -`True`. Or `False`, if there is an external `loadbalancer_apiserver` defined). +`True`. Or `False`, if `kube_apiserver_endpoint` or `kube_apiserver_cluster_internal_endpoint` is defined). You may also define the port the local internal loadbalancer uses by changing, `loadbalancer_apiserver_port`. This defaults to the value of `kube_apiserver_port`. It is also important to note that Kubespray will only @@ -29,6 +29,22 @@ configure kubelet and kube-proxy on non-master nodes to use the local internal loadbalancer. If you wish to control the name of the loadbalancer container, you can set the variable `loadbalancer_apiserver_pod_name`. +## API Server Endpoint Configuration + +Kubespray uses two primary variables to configure API server endpoints: + +* `kube_apiserver_endpoint` - Used to access the API from outside the cluster (kubectl, CI/CD, etc.) +* `kube_apiserver_cluster_internal_endpoint` - Used to access the API from inside the cluster (nodes, pods with hostNetwork). Optional, defaults to `kube_apiserver_endpoint` or localhost LB. + +These variables automatically resolve to appropriate values based on your +loadbalancer configuration, providing sane defaults for HA, non-HA, and +localhost LB scenarios. Legacy variables (`loadbalancer_apiserver`, +`apiserver_loadbalancer_domain_name`, `kubeadm_config_api_fqdn`) were removed +in v2.33.0 and should be replaced with the endpoint variables above. + +> [!NOTE] +> If your load balancer has both a domain name and an IP address (VIP), set `kube_apiserver_endpoint` to the domain name and add the VIP to `supplementary_addresses_in_ssl_keys` so that both are included in the API server certificate SANs. + If you choose to NOT use the local internal loadbalancer, you will need to use the [kube-vip](/docs/ingress/kube-vip.md) ansible role or configure your own loadbalancer to achieve HA. By default, it only configures a non-HA endpoint, which points to the `access_ip` or IP address of the first server node in the `kube_control_plane` group. @@ -61,10 +77,10 @@ And the corresponding example global vars for such a "cluster-aware" external LB with the cluster API access modes configured in Kubespray: ```yml -apiserver_loadbalancer_domain_name: "my-apiserver-lb.example.com" -loadbalancer_apiserver: - address: - port: 8383 +## Configure endpoints directly +kube_apiserver_endpoint: "https://:8383" +# Optional: if different from external endpoint +kube_apiserver_cluster_internal_endpoint: "https://:8383" ``` Note: The default kubernetes apiserver configuration binds to all interfaces, @@ -73,12 +89,6 @@ loadbalancer_apiserver: listens on a specific interface (to avoid conflict with haproxy binding the port on the VIP address) -This domain name, or default "lb-apiserver.kubernetes.local", will be inserted -into the `/etc/hosts` file of all servers in the `k8s_cluster` group and wired -into the generated self-signed TLS/SSL certificates as well. Note that -the HAProxy service should as well be HA and requires a VIP management, which -is out of scope of this doc. - There is a special case for an internal and an externally configured (not with Kubespray) LB used simultaneously. Keep in mind that the cluster is not aware of such an external LB and you need no to specify any configuration variables @@ -90,7 +100,7 @@ for it. `supplementary_addresses_in_ssl_keys` list. Then, kubespray will add them into the generated cluster certificates as well. -Aside of that specific case, the `loadbalancer_apiserver` considered mutually +Aside of that specific case, defining `kube_apiserver_endpoint` or `kube_apiserver_cluster_internal_endpoint` is considered mutually exclusive to `loadbalancer_apiserver_localhost`. Access API endpoints are evaluated automatically, as the following: @@ -106,17 +116,20 @@ Access API endpoints are evaluated automatically, as the following: Where: * `m[0]` - the first node in the `kube_control_plane` group; -* `lb` - LB FQDN, `apiserver_loadbalancer_domain_name`; +* `lb` - External loadbalancer address from `kube_apiserver_endpoint`; * `ext` - Externally load balanced VIP:port and FQDN, not managed by Kubespray; * `lc` - localhost; * `cbip` - a custom bind IP, `kube_apiserver_bind_address`; * `dbip` - localhost for the default bind IP '0.0.0.0'; * `nsp` - nginx secure port, `loadbalancer_apiserver_port`, defers to `sp`; * `sp` - secure port, `kube_apiserver_port`; -* `lp` - LB port, `loadbalancer_apiserver.port`, defers to the secure port; +* `lp` - External loadbalancer port from `kube_apiserver_endpoint`; * `ip` - the node IP, defers to the ansible IP; * `aip` - `access_ip`, defers to the ip. +The `kube_apiserver_endpoint` and `kube_apiserver_cluster_internal_endpoint` variables +automatically resolve to the appropriate values based on your loadbalancer configuration. + A second and a third column represent internal cluster access modes. The last column illustrates an example URI to access the cluster APIs externally. Kubespray has nothing to do with it, this is informational only. diff --git a/inventory/sample/group_vars/all/all.yml b/inventory/sample/group_vars/all/all.yml index 8a595e77b..9a85592cc 100644 --- a/inventory/sample/group_vars/all/all.yml +++ b/inventory/sample/group_vars/all/all.yml @@ -10,11 +10,10 @@ bin_dir: /usr/local/bin # access_ip: 1.1.1.1 -## External LB example config -## apiserver_loadbalancer_domain_name: "elb.some.domain" -# loadbalancer_apiserver: -# address: 1.2.3.4 -# port: 1234 +## API server endpoints +## Users should set this to their LB/VIP endpoint for external access: +# kube_apiserver_endpoint: "https://elb.some.domain:6443" +# kube_apiserver_cluster_internal_endpoint: "https://elb.some.domain:6443" ## Internal loadbalancers for apiservers # loadbalancer_apiserver_localhost: true diff --git a/inventory/sample/group_vars/k8s_cluster/addons.yml b/inventory/sample/group_vars/k8s_cluster/addons.yml index d558b16a4..bfecf5d84 100644 --- a/inventory/sample/group_vars/k8s_cluster/addons.yml +++ b/inventory/sample/group_vars/k8s_cluster/addons.yml @@ -189,9 +189,8 @@ kube_vip_enabled: false # kube_vip_arp_enabled: true # kube_vip_controlplane_enabled: true # kube_vip_address: 192.168.56.120 -# loadbalancer_apiserver: -# address: "{{ kube_vip_address }}" -# port: 6443 +# kube_apiserver_endpoint: "https://{{ kube_vip_address }}:6443" +# kube_apiserver_cluster_internal_endpoint: "https://{{ kube_vip_address }}:6443" # kube_vip_interface: eth0 # kube_vip_services_enabled: false # kube_vip_dns_mode: first diff --git a/roles/kubernetes-apps/ansible/tasks/main.yml b/roles/kubernetes-apps/ansible/tasks/main.yml index 53d6f0d32..2d747343f 100644 --- a/roles/kubernetes-apps/ansible/tasks/main.yml +++ b/roles/kubernetes-apps/ansible/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Kubernetes Apps | Wait for kube-apiserver uri: - url: "{{ kube_apiserver_endpoint }}/healthz" + url: "{{ _kube_apiserver_local_endpoint }}/healthz" validate_certs: false client_cert: "{{ kube_apiserver_client_cert }}" client_key: "{{ kube_apiserver_client_key }}" diff --git a/roles/kubernetes-apps/cluster_roles/tasks/main.yml b/roles/kubernetes-apps/cluster_roles/tasks/main.yml index bbb53a399..5c68e12a8 100644 --- a/roles/kubernetes-apps/cluster_roles/tasks/main.yml +++ b/roles/kubernetes-apps/cluster_roles/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Kubernetes Apps | Wait for kube-apiserver uri: - url: "{{ kube_apiserver_endpoint }}/healthz" + url: "{{ _kube_apiserver_local_endpoint }}/healthz" validate_certs: false client_cert: "{{ kube_apiserver_client_cert }}" client_key: "{{ kube_apiserver_client_key }}" diff --git a/roles/kubernetes/client/tasks/main.yml b/roles/kubernetes/client/tasks/main.yml index 25b5312ce..ea5f65db0 100644 --- a/roles/kubernetes/client/tasks/main.yml +++ b/roles/kubernetes/client/tasks/main.yml @@ -1,25 +1,4 @@ --- -- name: Set external kube-apiserver endpoint - set_fact: - # noqa: jinja[spacing] - external_apiserver_address: >- - {%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined -%} - {{ loadbalancer_apiserver.address }} - {%- elif kubeconfig_localhost_ansible_host is defined and kubeconfig_localhost_ansible_host -%} - {{ hostvars[groups['kube_control_plane'][0]].ansible_host }} - {%- else -%} - {{ kube_apiserver_access_address }} - {%- endif -%} - # noqa: jinja[spacing] - external_apiserver_port: >- - {%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined and loadbalancer_apiserver.port is defined -%} - {{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} - {%- else -%} - {{ kube_apiserver_port }} - {%- endif -%} - tags: - - facts - - name: Create kube config dir for current/ansible become user file: path: "{{ ansible_env.HOME | default('/root') }}/.kube" @@ -47,8 +26,8 @@ - name: Wait for k8s apiserver wait_for: - host: "{{ kube_apiserver_access_address }}" - port: "{{ kube_apiserver_port }}" + host: "{{ _kube_apiserver_local_endpoint | urlsplit('hostname') }}" + port: "{{ _kube_apiserver_local_endpoint | urlsplit('port') | default(kube_apiserver_port, true) }}" timeout: 180 - name: Create kubeconfig localhost artifacts @@ -73,9 +52,13 @@ vars: admin_kubeconfig: "{{ kubeadm_admin_kubeconfig.stdout | from_yaml }}" context: "kubernetes-admin-{{ cluster_name }}@{{ cluster_name }}" + client_kubeconfig_endpoint: >- + {{ ('https://' ~ (hostvars[groups['kube_control_plane'][0]].ansible_host | ansible.utils.ipwrap) ~ ':' ~ kube_apiserver_port) + if (kubeconfig_localhost_ansible_host | default(false) and kube_apiserver_endpoint is not defined) + else _kube_apiserver_endpoint }} override: clusters: - - "{{ admin_kubeconfig['clusters'][0] | combine({'name': cluster_name, 'cluster': admin_kubeconfig['clusters'][0]['cluster'] | combine({'server': 'https://' + (external_apiserver_address | string | ansible.utils.ipwrap) + ':' + (external_apiserver_port | string)})}, recursive=true) }}" + - "{{ admin_kubeconfig['clusters'][0] | combine({'name': cluster_name, 'cluster': admin_kubeconfig['clusters'][0]['cluster'] | combine({'server': client_kubeconfig_endpoint})}, recursive=true) }}" contexts: - "{{ admin_kubeconfig['contexts'][0] | combine({'name': context, 'context': admin_kubeconfig['contexts'][0]['context'] | combine({'cluster': cluster_name})}, recursive=true) }}" current-context: "{{ context }}" diff --git a/roles/kubernetes/control-plane/handlers/main.yml b/roles/kubernetes/control-plane/handlers/main.yml index 215521ef0..87d860c3e 100644 --- a/roles/kubernetes/control-plane/handlers/main.yml +++ b/roles/kubernetes/control-plane/handlers/main.yml @@ -106,7 +106,7 @@ - name: Control plane | wait for the apiserver to be running uri: - url: "{{ kube_apiserver_endpoint }}/healthz" + url: "{{ _kube_apiserver_local_endpoint }}/healthz" validate_certs: false register: result until: result.status == 200 diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml b/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml index f42029d63..be65ed51e 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml @@ -1,16 +1,4 @@ --- -- name: Set kubeadm_discovery_address - set_fact: - # noqa: jinja[spacing] - kubeadm_discovery_address: >- - {%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%} - {{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }} - {%- else -%} - {{ kube_apiserver_endpoint | regex_replace('https://', '') }} - {%- endif %} - tags: - - facts - - name: Obtain kubeadm certificate key for joining control planes nodes when: - not kube_external_ca_mode @@ -31,8 +19,8 @@ - name: Wait for k8s apiserver wait_for: - host: "{{ kubeadm_discovery_address | regex_replace('\\]?:\\d+$', '') | regex_replace('^\\[', '') }}" - port: "{{ kubeadm_discovery_address.split(':')[-1] }}" + host: "{{ _kube_apiserver_endpoint | urlsplit('hostname') }}" + port: "{{ _kube_apiserver_endpoint | urlsplit('port') | default(kube_apiserver_port, true) }}" timeout: 180 diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml index 7c431fd17..d907daeb6 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml @@ -36,8 +36,8 @@ - "localhost" - "127.0.0.1" - "::1" - - "{{ apiserver_loadbalancer_domain_name | d('') }}" - - "{{ loadbalancer_apiserver.address | d('') }}" + - "{{ _kube_apiserver_endpoint | urlsplit('hostname') }}" + - "{{ _kube_apiserver_cluster_internal_endpoint | urlsplit('hostname') }}" - "{{ supplementary_addresses_in_ssl_keys }}" - "{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_access_ip') }}" - "{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_ip') }}" @@ -87,12 +87,6 @@ when: kube_apiserver_tracing notify: Control plane | Restart apiserver -# Nginx LB(default), If kubeadm_config_api_fqdn is defined, use other LB by kubeadm controlPlaneEndpoint. -- name: Set kubeadm_config_api_fqdn define - set_fact: - kubeadm_config_api_fqdn: "{{ apiserver_loadbalancer_domain_name }}" - when: loadbalancer_apiserver is defined - - name: Kubeadm | Create kubeadm config template: src: "kubeadm-config.v1beta4.yaml.j2" diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 index c21b5c4b1..4fb9d0924 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2 @@ -116,8 +116,8 @@ featureGates: {% endfor %} {% endif %} kubernetesVersion: v{{ kube_version }} -{% if kubeadm_config_api_fqdn is defined %} -controlPlaneEndpoint: "{{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}" +{% if kube_apiserver_endpoint is defined %} +controlPlaneEndpoint: "{{ _kube_apiserver_endpoint_netloc }}" {% else %} controlPlaneEndpoint: "{{ main_ip | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}" {% endif %} diff --git a/roles/kubernetes/control-plane/templates/kubeadm-controlplane.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-controlplane.yaml.j2 index 6ce1654bc..eee330796 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-controlplane.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-controlplane.yaml.j2 @@ -6,11 +6,7 @@ discovery: kubeConfigPath: {{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml {% else %} bootstrapToken: -{% if kubeadm_config_api_fqdn is defined %} - apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} -{% else %} - apiServerEndpoint: "{{ kubeadm_discovery_address }}" -{% endif %} + apiServerEndpoint: "{{ _kube_apiserver_endpoint_netloc }}" token: {{ kubeadm_token }} unsafeSkipCAVerification: true {% endif %} diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml index 9851b73d5..fe9f589f7 100644 --- a/roles/kubernetes/kubeadm/tasks/main.yml +++ b/roles/kubernetes/kubeadm/tasks/main.yml @@ -1,16 +1,4 @@ --- -- name: Set kubeadm_discovery_address - set_fact: - # noqa: jinja[spacing] - kubeadm_discovery_address: >- - {%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%} - {{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }} - {%- else -%} - {{ kube_apiserver_endpoint | replace("https://", "") }} - {%- endif %} - tags: - - facts - - name: Check if kubelet.conf exists stat: path: "{{ kube_config_dir }}/kubelet.conf" @@ -104,25 +92,11 @@ lineinfile: dest: "{{ kube_config_dir }}/kubelet.conf" regexp: '^(\s+)server: https.*' - line: '\1server: {{ kube_apiserver_endpoint }}' - backrefs: true - backup: true - when: - - kubeadm_config_api_fqdn is not defined - - ('kube_control_plane' not in group_names) - - kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "") - notify: Kubeadm | restart kubelet - -- name: Update server field in kubelet kubeconfig - external lb - lineinfile: - dest: "{{ kube_config_dir }}/kubelet.conf" - regexp: '^(\s+)server: https.*' - line: '\1server: {{ kube_apiserver_endpoint }}' + line: '\1server: {{ _kube_apiserver_cluster_internal_endpoint }}' backrefs: true backup: true when: - ('kube_control_plane' not in group_names) - - loadbalancer_apiserver is defined notify: Kubeadm | restart kubelet - name: Get current resourceVersion of kube-proxy configmap @@ -136,31 +110,14 @@ tags: - kube-proxy -# FIXME(mattymo): Need to point to localhost, otherwise control plane nodes will all point -# incorrectly to first control plane node, creating SPoF. - name: Update server field in kube-proxy kubeconfig shell: >- - set -o pipefail && {{ kubectl }} get configmap kube-proxy -n kube-system -o yaml - | sed 's#server:.*#server: https://127.0.0.1:{{ kube_apiserver_port }}#g' - | {{ kubectl }} replace -f - - args: - executable: /bin/bash - run_once: true - delegate_to: "{{ groups['kube_control_plane'] | first }}" - delegate_facts: false - when: - - kubeadm_config_api_fqdn is not defined - - kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "") - - kube_proxy_deployed - - loadbalancer_apiserver_localhost - tags: - - kube-proxy - -- name: Update server field in kube-proxy kubeconfig - external lb - shell: >- - set -o pipefail && {{ kubectl }} get configmap kube-proxy -n kube-system -o yaml - | sed 's#server:.*#server: {{kube_apiserver_endpoint}}#g' - | {{ kubectl }} replace -f - + set -o pipefail && + if ! {{ kubectl }} get configmap kube-proxy -n kube-system -o yaml | grep -Fq "server: {{ _kube_apiserver_cluster_internal_endpoint }}"; then + {{ kubectl }} get configmap kube-proxy -n kube-system -oyaml | + sed 's#server:.*#server: {{ _kube_apiserver_cluster_internal_endpoint }}#g' | + {{ kubectl }} replace -f - + fi args: executable: /bin/bash run_once: true @@ -168,7 +125,6 @@ delegate_facts: false when: - kube_proxy_deployed - - loadbalancer_apiserver is defined tags: - kube-proxy @@ -196,8 +152,6 @@ delegate_to: "{{ groups['kube_control_plane'] | first }}" delegate_facts: false when: - - kubeadm_config_api_fqdn is not defined or loadbalancer_apiserver is defined - - kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "") or loadbalancer_apiserver is defined - kube_proxy_deployed - original_configmap_resource_version.stdout != new_configmap_resource_version.stdout tags: diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2 index 96157460e..09c2c7649 100644 --- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2 +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2 @@ -7,11 +7,7 @@ discovery: kubeConfigPath: {{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml {% else %} bootstrapToken: -{% if kubeadm_config_api_fqdn is defined %} - apiServerEndpoint: "{{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}" -{% else %} - apiServerEndpoint: "{{ kubeadm_discovery_address }}" -{% endif %} + apiServerEndpoint: "{{ _kube_apiserver_endpoint_netloc }}" token: {{ kubeadm_token }} {% if ca_cert_content is defined %} caCertHashes: diff --git a/roles/kubernetes/node-label/tasks/main.yml b/roles/kubernetes/node-label/tasks/main.yml index 3ebb64594..aa74e938c 100644 --- a/roles/kubernetes/node-label/tasks/main.yml +++ b/roles/kubernetes/node-label/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Kubernetes Apps | Wait for kube-apiserver uri: - url: "{{ kube_apiserver_endpoint }}/healthz" + url: "{{ _kube_apiserver_local_endpoint }}/healthz" validate_certs: false client_cert: "{{ kube_apiserver_client_cert }}" client_key: "{{ kube_apiserver_client_key }}" diff --git a/roles/kubernetes/preinstall/handlers/main.yml b/roles/kubernetes/preinstall/handlers/main.yml index 21bc90f9c..9bfec95aa 100644 --- a/roles/kubernetes/preinstall/handlers/main.yml +++ b/roles/kubernetes/preinstall/handlers/main.yml @@ -107,7 +107,7 @@ # When running this as the last phase ensure we wait for kube-apiserver to come up - name: Preinstall | wait for the apiserver to be running uri: - url: "{{ kube_apiserver_endpoint }}/healthz" + url: "{{ _kube_apiserver_local_endpoint }}/healthz" validate_certs: false register: result until: result.status == 200 diff --git a/roles/kubespray_defaults/defaults/main/main.yml b/roles/kubespray_defaults/defaults/main/main.yml index b38d6ae38..e04e79eea 100644 --- a/roles/kubespray_defaults/defaults/main/main.yml +++ b/roles/kubespray_defaults/defaults/main/main.yml @@ -655,28 +655,20 @@ first_kube_control_plane: "{{ groups['kube_control_plane'] | first }}" kube_apiserver_address: "{{ hostvars[inventory_hostname]['main_ip'] }}" kube_apiserver_access_address: "{{ hostvars[inventory_hostname]['main_access_ip'] }}" first_kube_control_plane_address: "{{ hostvars[groups['kube_control_plane'][0]]['main_access_ip'] }}" -loadbalancer_apiserver_localhost: "{{ loadbalancer_apiserver is not defined }}" +# Loadbalancer configuration +loadbalancer_apiserver_localhost: "{{ kube_apiserver_endpoint is not defined and kube_apiserver_cluster_internal_endpoint is not defined }}" loadbalancer_apiserver_type: "nginx" -# applied if only external loadbalancer_apiserver is defined, otherwise ignored -apiserver_loadbalancer_domain_name: "{{ 'localhost' if loadbalancer_apiserver_localhost else (loadbalancer_apiserver.address | d(undef())) }}" -kube_apiserver_global_endpoint: |- - {% if loadbalancer_apiserver is defined -%} - https://{{ apiserver_loadbalancer_domain_name | ansible.utils.ipwrap }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} - {%- elif loadbalancer_apiserver_localhost -%} - https://localhost:{{ loadbalancer_apiserver_port | default(kube_apiserver_port) }} - {%- else -%} - https://{{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }} - {%- endif %} -kube_apiserver_endpoint: |- - {% if loadbalancer_apiserver is defined -%} - https://{{ apiserver_loadbalancer_domain_name | ansible.utils.ipwrap }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} - {%- elif ('kube_control_plane' not in group_names) and loadbalancer_apiserver_localhost -%} - https://localhost:{{ loadbalancer_apiserver_port | default(kube_apiserver_port) }} - {%- elif 'kube_control_plane' in group_names -%} - https://{{ kube_apiserver_bind_address | regex_replace('::', '127.0.0.1') | ansible.utils.ipwrap }}:{{ kube_apiserver_port }} - {%- else -%} - https://{{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }} - {%- endif %} +loadbalancer_apiserver_port: "{{ kube_apiserver_port }}" + +# Primary API endpoint for external access (e.g., kubectl). +# User should set this in their inventory if they have a Load Balancer. +# If not provided, resolved internally in vars. +kube_apiserver_endpoint: "{{ undef() }}" + +# Internal endpoint for in-cluster traffic, including nodes and pods. +# Optional override; resolved internally in vars if not provided. +kube_apiserver_cluster_internal_endpoint: "{{ undef() }}" + kube_apiserver_client_cert: "{{ kube_cert_dir }}/ca.crt" kube_apiserver_client_key: "{{ kube_cert_dir }}/ca.key" diff --git a/roles/kubespray_defaults/vars/main/main.yml b/roles/kubespray_defaults/vars/main/main.yml index 893e91845..6f3124909 100644 --- a/roles/kubespray_defaults/vars/main/main.yml +++ b/roles/kubespray_defaults/vars/main/main.yml @@ -81,3 +81,40 @@ ssl_ca_dirs_per_os_family: RedHat: - /etc/pki/tls - /etc/pki/ca-trust + +# Internal API server endpoints (cannot be overridden by inventory). +# +# _kube_apiserver_endpoint: +# External API server endpoint used for external access from outside the cluster +# (e.g., in admin kubeconfig written for localhost). +# Resolves to kube_apiserver_endpoint if defined; otherwise defaults to +# https://:. +# +# _kube_apiserver_cluster_internal_endpoint: +# Internal endpoint used for in-cluster communication to the API server by +# cluster components, nodes (kubelet, kube-proxy), and pods (CNI plugins). +# If loadbalancer_apiserver_localhost is enabled, it points to the local load balancer +# proxy (https://127.0.0.1:), otherwise falls back +# to _kube_apiserver_endpoint. +# +# _kube_apiserver_local_endpoint: +# Direct local endpoint to the kube-apiserver process running on the control plane node itself, +# bypassing any load balancer or proxy. +# Used on control plane nodes for local readiness checks, health checks (/healthz), +# and restart handlers. +# +# _kube_apiserver_endpoint_netloc: +# The : representation of _kube_apiserver_endpoint without the scheme. +# Used where kubeadm requires a host:port string (e.g. controlPlaneEndpoint, apiServerEndpoint). +_kube_apiserver_endpoint: >- + {{ kube_apiserver_endpoint if kube_apiserver_endpoint is defined + else 'https://' ~ (first_kube_control_plane_address | ansible.utils.ipwrap) ~ ':' ~ kube_apiserver_port }} + +_kube_apiserver_cluster_internal_endpoint: >- + {{ kube_apiserver_cluster_internal_endpoint if kube_apiserver_cluster_internal_endpoint is defined + else (('https://127.0.0.1:' ~ loadbalancer_apiserver_port) + if loadbalancer_apiserver_localhost else _kube_apiserver_endpoint) }} + +_kube_apiserver_local_endpoint: "https://{{ kube_apiserver_bind_address | regex_replace('^0\\.0\\.0\\.0$', '127.0.0.1') | regex_replace('^::$', '127.0.0.1') | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}" + +_kube_apiserver_endpoint_netloc: "{{ _kube_apiserver_endpoint | urlsplit('netloc') }}" diff --git a/roles/network_facts/tasks/main.yaml b/roles/network_facts/tasks/main.yaml index 27aa6180b..58b9ea196 100644 --- a/roles/network_facts/tasks/main.yaml +++ b/roles/network_facts/tasks/main.yaml @@ -46,8 +46,8 @@ | select('defined'))) | unique }}" no_proxy_prepare: - - "{{ apiserver_loadbalancer_domain_name | d('') }}" - - "{{ loadbalancer_apiserver.address if loadbalancer_apiserver is defined else '' }}" + - "{{ _kube_apiserver_endpoint | urlsplit('hostname') }}" + - "{{ _kube_apiserver_cluster_internal_endpoint | urlsplit('hostname') }}" - "{{ hosts_with_no_proxy | map('extract', hostvars, morekeys=['main_access_ip']) }}" - "{{ _hostnames }}" - "{{ _hostnames | map('regex_replace', '$', '.' + dns_domain ) }}" diff --git a/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2 b/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2 index 36c548cc3..78eac2d0d 100644 --- a/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2 +++ b/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2 @@ -6,6 +6,6 @@ metadata: name: kubernetes-services-endpoint data: {% if calico_bpf_enabled or loadbalancer_apiserver_localhost %} - KUBERNETES_SERVICE_HOST: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}" - KUBERNETES_SERVICE_PORT: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}" + KUBERNETES_SERVICE_HOST: "{{ _kube_apiserver_cluster_internal_endpoint | urlsplit('hostname') }}" + KUBERNETES_SERVICE_PORT: "{{ _kube_apiserver_cluster_internal_endpoint | urlsplit('port') }}" {% endif %} diff --git a/roles/network_plugin/cilium/templates/values.yaml.j2 b/roles/network_plugin/cilium/templates/values.yaml.j2 index 39179fb8c..d0427c6b1 100644 --- a/roles/network_plugin/cilium/templates/values.yaml.j2 +++ b/roles/network_plugin/cilium/templates/values.yaml.j2 @@ -7,8 +7,8 @@ image: repository: {{ cilium_image_repo }} tag: {{ cilium_image_tag }} -k8sServiceHost: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}" -k8sServicePort: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}" +k8sServiceHost: "{{ _kube_apiserver_cluster_internal_endpoint | urlsplit('hostname') }}" +k8sServicePort: "{{ _kube_apiserver_cluster_internal_endpoint | urlsplit('port') }}" ipv4: enabled: {{ cilium_enable_ipv4 | to_json }} diff --git a/roles/network_plugin/kube-router/templates/kubeconfig.yml.j2 b/roles/network_plugin/kube-router/templates/kubeconfig.yml.j2 index 470885111..c4ac06e3a 100644 --- a/roles/network_plugin/kube-router/templates/kubeconfig.yml.j2 +++ b/roles/network_plugin/kube-router/templates/kubeconfig.yml.j2 @@ -5,7 +5,7 @@ clusters: - name: cluster cluster: certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - server: {{ kube_apiserver_endpoint }} + server: {{ _kube_apiserver_cluster_internal_endpoint }} users: - name: kube-router user: diff --git a/roles/validate_inventory/tasks/main.yml b/roles/validate_inventory/tasks/main.yml index 981b79326..6746c4c02 100644 --- a/roles/validate_inventory/tasks/main.yml +++ b/roles/validate_inventory/tasks/main.yml @@ -9,13 +9,43 @@ - name: Fail if removed variables are used vars: # Always remove items from this list after the release in comments - removed_vars: [] + removed_vars: + - apiserver_loadbalancer_domain_name # 2.33.0 + - kube_apiserver_global_endpoint # 2.33.0 + - kubeadm_config_api_fqdn # 2.33.0 + - loadbalancer_apiserver # 2.33.0 removed_vars_found: "{{ query('varnames', '^(' + (removed_vars | join('|')) + ')$') }}" assert: that: removed_vars_found | length == 0 fail_msg: "Removed variables present: {{ removed_vars_found | join(', ') }}" run_once: true +- name: Check kube_apiserver_endpoint format + assert: + that: + - kube_apiserver_endpoint | urlsplit('scheme') == 'https' + - kube_apiserver_endpoint | urlsplit('hostname') is not none + - kube_apiserver_endpoint | urlsplit('hostname') | length > 0 + - kube_apiserver_endpoint | urlsplit('port') is not none + msg: "kube_apiserver_endpoint must be in the format 'https://:' (e.g., 'https://lb.example.com:6443')" + run_once: true + when: + - kube_apiserver_endpoint is defined + - not ignore_assert_errors + +- name: Check kube_apiserver_cluster_internal_endpoint format + assert: + that: + - kube_apiserver_cluster_internal_endpoint | urlsplit('scheme') == 'https' + - kube_apiserver_cluster_internal_endpoint | urlsplit('hostname') is not none + - kube_apiserver_cluster_internal_endpoint | urlsplit('hostname') | length > 0 + - kube_apiserver_cluster_internal_endpoint | urlsplit('port') is not none + msg: "kube_apiserver_cluster_internal_endpoint must be in the format 'https://:' (e.g., 'https://lb.internal:6443')" + run_once: true + when: + - kube_apiserver_cluster_internal_endpoint is defined + - not ignore_assert_errors + - name: Stop if kube_control_plane group is empty assert: that: groups.get( 'kube_control_plane' ) != []