refactor: unify inconsistent variables in apiserver endpoint (#13392)

This commit is contained in:
Dasmat Hansda
2026-10-12 06:25:22 +00:00
committed by GitHub
parent 7c57bc015d
commit 38f82cdce9
30 changed files with 170 additions and 192 deletions
@@ -1,8 +1,6 @@
## External LB example config ## External LB example config
apiserver_loadbalancer_domain_name: {{ lb_pubip.dnsSettings.fqdn }} kube_apiserver_endpoint: "https://{{ lb_pubip.dnsSettings.fqdn }}:6443"
loadbalancer_apiserver: kube_apiserver_cluster_internal_endpoint: "https://{{ lb_pubip.dnsSettings.fqdn }}:6443"
address: {{ lb_pubip.ipAddress }}
port: 6443
## Internal loadbalancers for apiservers ## Internal loadbalancers for apiservers
loadbalancer_apiserver_localhost: false loadbalancer_apiserver_localhost: false
+3 -3
View File
@@ -86,8 +86,8 @@ You can use the following set of commands to get the kubeconfig file from your n
CONTROLLER_HOST_NAME=$(cat ./inventory/hosts | grep "\[kube_control_plane\]" -A 1 | tail -n 1) CONTROLLER_HOST_NAME=$(cat ./inventory/hosts | grep "\[kube_control_plane\]" -A 1 | tail -n 1)
CONTROLLER_IP=$(cat ./inventory/hosts | grep $CONTROLLER_HOST_NAME | grep ansible_host | cut -d'=' -f2) CONTROLLER_IP=$(cat ./inventory/hosts | grep $CONTROLLER_HOST_NAME | grep ansible_host | cut -d'=' -f2)
# Get the hostname of the load balancer. # Get the API endpoint of the load balancer.
LB_HOST=$(cat inventory/hosts | grep apiserver_loadbalancer_domain_name | cut -d'"' -f2) LB_ENDPOINT=$(cat inventory/hosts | grep kube_apiserver_endpoint | cut -d'"' -f2)
# Get the controller's SSH fingerprint. # Get the controller's SSH fingerprint.
ssh-keygen -R $CONTROLLER_IP > /dev/null 2>&1 ssh-keygen -R $CONTROLLER_IP > /dev/null 2>&1
@@ -97,7 +97,7 @@ ssh-keyscan -H $CONTROLLER_IP >> ~/.ssh/known_hosts 2>/dev/null
mkdir -p ~/.kube mkdir -p ~/.kube
ssh -F ssh-bastion.conf centos@$CONTROLLER_IP "sudo chmod 644 /etc/kubernetes/admin.conf" ssh -F ssh-bastion.conf centos@$CONTROLLER_IP "sudo chmod 644 /etc/kubernetes/admin.conf"
scp -F ssh-bastion.conf centos@$CONTROLLER_IP:/etc/kubernetes/admin.conf ~/.kube/config scp -F ssh-bastion.conf centos@$CONTROLLER_IP:/etc/kubernetes/admin.conf ~/.kube/config
sed -i "s^server:.*^server: https://$LB_HOST:6443^" ~/.kube/config sed -i "s^server:.*^server: $LB_ENDPOINT^" ~/.kube/config
kubectl get nodes kubectl get nodes
``` ```
@@ -170,7 +170,7 @@ data "template_file" "inventory" {
list_node = join("\n", aws_instance.k8s-worker.*.private_dns) list_node = join("\n", aws_instance.k8s-worker.*.private_dns)
connection_strings_etcd = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-etcd.*.private_dns, aws_instance.k8s-etcd.*.private_ip)) connection_strings_etcd = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-etcd.*.private_dns, aws_instance.k8s-etcd.*.private_ip))
list_etcd = join("\n", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_dns) : (aws_instance.k8s-master.*.private_dns))) list_etcd = join("\n", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_dns) : (aws_instance.k8s-master.*.private_dns)))
nlb_api_fqdn = "apiserver_loadbalancer_domain_name=\"${module.aws-nlb.aws_nlb_api_fqdn}\"" nlb_api_fqdn = "kube_apiserver_endpoint=\"https://${module.aws-nlb.aws_nlb_api_fqdn}:${var.aws_nlb_api_port}\"\nkube_apiserver_cluster_internal_endpoint=\"https://${module.aws-nlb.aws_nlb_api_fqdn}:${var.aws_nlb_api_port}\""
} }
} }
+1 -1
View File
@@ -379,7 +379,7 @@ To clean up any ipvs leftovers:
Calico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.tigera.io/calico/latest/operations/ebpf/enabling-ebpf) for guidelines on how to do this. Calico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.tigera.io/calico/latest/operations/ebpf/enabling-ebpf) for guidelines on how to do this.
Kubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `loadbalancer_apiserver` inventory variable documented in [HA Mode](/docs/operations/ha-mode.md). Kubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `kube_apiserver_cluster_internal_endpoint` inventory variable documented in [HA Mode](/docs/operations/ha-mode.md).
If no external loadbalancer is used, Calico eBPF can also use the localhost loadbalancer option. We are able to do so only if you use the same port for the localhost apiserver loadbalancer and the kube-apiserver. In this case Calico Automatic Host Endpoints need to be enabled to allow services like `coredns` and `metrics-server` to communicate with the kubernetes host endpoint. See [this blog post](https://www.projectcalico.org/securing-kubernetes-nodes-with-calico-automatic-host-endpoints/) on enabling automatic host endpoints. If no external loadbalancer is used, Calico eBPF can also use the localhost loadbalancer option. We are able to do so only if you use the same port for the localhost apiserver loadbalancer and the kube-apiserver. In this case Calico Automatic Host Endpoints need to be enabled to allow services like `coredns` and `metrics-server` to communicate with the kubernetes host endpoint. See [this blog post](https://www.projectcalico.org/securing-kubernetes-nodes-with-calico-automatic-host-endpoints/) on enabling automatic host endpoints.
+9 -7
View File
@@ -47,13 +47,15 @@ Some variables of note include:
* *access_ip6* - similarly ``access_ip`` but IPv6 * *access_ip6* - similarly ``access_ip`` but IPv6
* *ansible_default_ipv6.address* - Not Kubespray-specific, but it is used if ip6 * *ansible_default_ipv6.address* - Not Kubespray-specific, but it is used if ip6
and access_ip6 are undefined and access_ip6 are undefined
* *loadbalancer_apiserver* - If defined, all hosts will connect to this * *kube_apiserver_endpoint* - Primary API endpoint for external access (e.g., kubectl).
address instead of localhost for kube_control_planes and kube_control_plane[0] for If you have a load balancer or VIP, set this in your inventory (e.g. `https://lb.example.com:6443`).
kube_nodes. See more details in the See more details in the [HA guide](/docs/operations/ha-mode.md).
[HA guide](/docs/operations/ha-mode.md). * *kube_apiserver_cluster_internal_endpoint* - Internal endpoint used for in-cluster
* *loadbalancer_apiserver_localhost* - makes all hosts to connect to traffic, including nodes and pods. Defaults to localhost LB proxy or `kube_apiserver_endpoint`.
the apiserver internally load balanced endpoint. Mutual exclusive to the See more details in the [HA guide](/docs/operations/ha-mode.md).
`loadbalancer_apiserver`. See more details in the * *loadbalancer_apiserver_localhost* - makes all hosts connect to
the apiserver internally via the local load balanced proxy (nginx/haproxy). Defaults to `true`
when no external or internal endpoint is defined. See more details in the
[HA guide](/docs/operations/ha-mode.md). [HA guide](/docs/operations/ha-mode.md).
## Special network variables ## Special network variables
+2 -3
View File
@@ -11,6 +11,5 @@
# numallowedlocalasnumbers: 0 # numallowedlocalasnumbers: 0
# sourceaddress: "None" # sourceaddress: "None"
# loadbalancer_apiserver: # kube_apiserver_endpoint: "https://10.99.0.44:8383"
# address: "10.99.0.44" # kube_apiserver_cluster_internal_endpoint: "https://10.99.0.44:8383"
# port: "8383"
+2 -3
View File
@@ -11,6 +11,5 @@
# numallowedlocalasnumbers: 0 # numallowedlocalasnumbers: 0
# sourceaddress: "None" # sourceaddress: "None"
# loadbalancer_apiserver: # kube_apiserver_endpoint: "https://10.99.0.21:8383"
# address: "10.99.0.21" # kube_apiserver_cluster_internal_endpoint: "https://10.99.0.21:8383"
# port: "8383"
+2 -3
View File
@@ -25,9 +25,8 @@ You also need to enable
# HA for control-plane, requires a VIP # HA for control-plane, requires a VIP
kube_vip_controlplane_enabled: true kube_vip_controlplane_enabled: true
kube_vip_address: 10.42.42.42 kube_vip_address: 10.42.42.42
loadbalancer_apiserver: kube_apiserver_endpoint: "https://{{ kube_vip_address }}:6443"
address: "{{ kube_vip_address }}" kube_apiserver_cluster_internal_endpoint: "https://{{ kube_vip_address }}:6443"
port: 6443
# kube_vip_interface: ens160 # kube_vip_interface: ens160
# LoadBalancer for services # LoadBalancer for services
+27 -14
View File
@@ -21,7 +21,7 @@ is less efficient than a dedicated load balancer because it creates extra
health checks on the Kubernetes apiserver, but is more practical for scenarios health checks on the Kubernetes apiserver, but is more practical for scenarios
where an external LB or virtual IP management is inconvenient. This option is where an external LB or virtual IP management is inconvenient. This option is
configured by the variable `loadbalancer_apiserver_localhost` (defaults to configured by the variable `loadbalancer_apiserver_localhost` (defaults to
`True`. Or `False`, if there is an external `loadbalancer_apiserver` defined). `True`. Or `False`, if `kube_apiserver_endpoint` or `kube_apiserver_cluster_internal_endpoint` is defined).
You may also define the port the local internal loadbalancer uses by changing, You may also define the port the local internal loadbalancer uses by changing,
`loadbalancer_apiserver_port`. This defaults to the value of `loadbalancer_apiserver_port`. This defaults to the value of
`kube_apiserver_port`. It is also important to note that Kubespray will only `kube_apiserver_port`. It is also important to note that Kubespray will only
@@ -29,6 +29,22 @@ configure kubelet and kube-proxy on non-master nodes to use the local internal
loadbalancer. If you wish to control the name of the loadbalancer container, loadbalancer. If you wish to control the name of the loadbalancer container,
you can set the variable `loadbalancer_apiserver_pod_name`. you can set the variable `loadbalancer_apiserver_pod_name`.
## API Server Endpoint Configuration
Kubespray uses two primary variables to configure API server endpoints:
* `kube_apiserver_endpoint` - Used to access the API from outside the cluster (kubectl, CI/CD, etc.)
* `kube_apiserver_cluster_internal_endpoint` - Used to access the API from inside the cluster (nodes, pods with hostNetwork). Optional, defaults to `kube_apiserver_endpoint` or localhost LB.
These variables automatically resolve to appropriate values based on your
loadbalancer configuration, providing sane defaults for HA, non-HA, and
localhost LB scenarios. Legacy variables (`loadbalancer_apiserver`,
`apiserver_loadbalancer_domain_name`, `kubeadm_config_api_fqdn`) were removed
in v2.33.0 and should be replaced with the endpoint variables above.
> [!NOTE]
> If your load balancer has both a domain name and an IP address (VIP), set `kube_apiserver_endpoint` to the domain name and add the VIP to `supplementary_addresses_in_ssl_keys` so that both are included in the API server certificate SANs.
If you choose to NOT use the local internal loadbalancer, you will need to If you choose to NOT use the local internal loadbalancer, you will need to
use the [kube-vip](/docs/ingress/kube-vip.md) ansible role or configure your own loadbalancer to achieve HA. By default, it only configures a non-HA endpoint, which points to the use the [kube-vip](/docs/ingress/kube-vip.md) ansible role or configure your own loadbalancer to achieve HA. By default, it only configures a non-HA endpoint, which points to the
`access_ip` or IP address of the first server node in the `kube_control_plane` group. `access_ip` or IP address of the first server node in the `kube_control_plane` group.
@@ -61,10 +77,10 @@ And the corresponding example global vars for such a "cluster-aware"
external LB with the cluster API access modes configured in Kubespray: external LB with the cluster API access modes configured in Kubespray:
```yml ```yml
apiserver_loadbalancer_domain_name: "my-apiserver-lb.example.com" ## Configure endpoints directly
loadbalancer_apiserver: kube_apiserver_endpoint: "https://<VIP>:8383"
address: <VIP> # Optional: if different from external endpoint
port: 8383 kube_apiserver_cluster_internal_endpoint: "https://<VIP>:8383"
``` ```
Note: The default kubernetes apiserver configuration binds to all interfaces, Note: The default kubernetes apiserver configuration binds to all interfaces,
@@ -73,12 +89,6 @@ loadbalancer_apiserver:
listens on a specific interface (to avoid conflict with haproxy binding the listens on a specific interface (to avoid conflict with haproxy binding the
port on the VIP address) port on the VIP address)
This domain name, or default "lb-apiserver.kubernetes.local", will be inserted
into the `/etc/hosts` file of all servers in the `k8s_cluster` group and wired
into the generated self-signed TLS/SSL certificates as well. Note that
the HAProxy service should as well be HA and requires a VIP management, which
is out of scope of this doc.
There is a special case for an internal and an externally configured (not with There is a special case for an internal and an externally configured (not with
Kubespray) LB used simultaneously. Keep in mind that the cluster is not aware Kubespray) LB used simultaneously. Keep in mind that the cluster is not aware
of such an external LB and you need no to specify any configuration variables of such an external LB and you need no to specify any configuration variables
@@ -90,7 +100,7 @@ for it.
`supplementary_addresses_in_ssl_keys` list. Then, kubespray will add them into `supplementary_addresses_in_ssl_keys` list. Then, kubespray will add them into
the generated cluster certificates as well. the generated cluster certificates as well.
Aside of that specific case, the `loadbalancer_apiserver` considered mutually Aside of that specific case, defining `kube_apiserver_endpoint` or `kube_apiserver_cluster_internal_endpoint` is considered mutually
exclusive to `loadbalancer_apiserver_localhost`. exclusive to `loadbalancer_apiserver_localhost`.
Access API endpoints are evaluated automatically, as the following: Access API endpoints are evaluated automatically, as the following:
@@ -106,17 +116,20 @@ Access API endpoints are evaluated automatically, as the following:
Where: Where:
* `m[0]` - the first node in the `kube_control_plane` group; * `m[0]` - the first node in the `kube_control_plane` group;
* `lb` - LB FQDN, `apiserver_loadbalancer_domain_name`; * `lb` - External loadbalancer address from `kube_apiserver_endpoint`;
* `ext` - Externally load balanced VIP:port and FQDN, not managed by Kubespray; * `ext` - Externally load balanced VIP:port and FQDN, not managed by Kubespray;
* `lc` - localhost; * `lc` - localhost;
* `cbip` - a custom bind IP, `kube_apiserver_bind_address`; * `cbip` - a custom bind IP, `kube_apiserver_bind_address`;
* `dbip` - localhost for the default bind IP '0.0.0.0'; * `dbip` - localhost for the default bind IP '0.0.0.0';
* `nsp` - nginx secure port, `loadbalancer_apiserver_port`, defers to `sp`; * `nsp` - nginx secure port, `loadbalancer_apiserver_port`, defers to `sp`;
* `sp` - secure port, `kube_apiserver_port`; * `sp` - secure port, `kube_apiserver_port`;
* `lp` - LB port, `loadbalancer_apiserver.port`, defers to the secure port; * `lp` - External loadbalancer port from `kube_apiserver_endpoint`;
* `ip` - the node IP, defers to the ansible IP; * `ip` - the node IP, defers to the ansible IP;
* `aip` - `access_ip`, defers to the ip. * `aip` - `access_ip`, defers to the ip.
The `kube_apiserver_endpoint` and `kube_apiserver_cluster_internal_endpoint` variables
automatically resolve to the appropriate values based on your loadbalancer configuration.
A second and a third column represent internal cluster access modes. The last A second and a third column represent internal cluster access modes. The last
column illustrates an example URI to access the cluster APIs externally. column illustrates an example URI to access the cluster APIs externally.
Kubespray has nothing to do with it, this is informational only. Kubespray has nothing to do with it, this is informational only.
+4 -5
View File
@@ -10,11 +10,10 @@ bin_dir: /usr/local/bin
# access_ip: 1.1.1.1 # access_ip: 1.1.1.1
## External LB example config ## API server endpoints
## apiserver_loadbalancer_domain_name: "elb.some.domain" ## Users should set this to their LB/VIP endpoint for external access:
# loadbalancer_apiserver: # kube_apiserver_endpoint: "https://elb.some.domain:6443"
# address: 1.2.3.4 # kube_apiserver_cluster_internal_endpoint: "https://elb.some.domain:6443"
# port: 1234
## Internal loadbalancers for apiservers ## Internal loadbalancers for apiservers
# loadbalancer_apiserver_localhost: true # loadbalancer_apiserver_localhost: true
@@ -189,9 +189,8 @@ kube_vip_enabled: false
# kube_vip_arp_enabled: true # kube_vip_arp_enabled: true
# kube_vip_controlplane_enabled: true # kube_vip_controlplane_enabled: true
# kube_vip_address: 192.168.56.120 # kube_vip_address: 192.168.56.120
# loadbalancer_apiserver: # kube_apiserver_endpoint: "https://{{ kube_vip_address }}:6443"
# address: "{{ kube_vip_address }}" # kube_apiserver_cluster_internal_endpoint: "https://{{ kube_vip_address }}:6443"
# port: 6443
# kube_vip_interface: eth0 # kube_vip_interface: eth0
# kube_vip_services_enabled: false # kube_vip_services_enabled: false
# kube_vip_dns_mode: first # kube_vip_dns_mode: first
+1 -1
View File
@@ -1,7 +1,7 @@
--- ---
- name: Kubernetes Apps | Wait for kube-apiserver - name: Kubernetes Apps | Wait for kube-apiserver
uri: uri:
url: "{{ kube_apiserver_endpoint }}/healthz" url: "{{ _kube_apiserver_local_endpoint }}/healthz"
validate_certs: false validate_certs: false
client_cert: "{{ kube_apiserver_client_cert }}" client_cert: "{{ kube_apiserver_client_cert }}"
client_key: "{{ kube_apiserver_client_key }}" client_key: "{{ kube_apiserver_client_key }}"
@@ -1,7 +1,7 @@
--- ---
- name: Kubernetes Apps | Wait for kube-apiserver - name: Kubernetes Apps | Wait for kube-apiserver
uri: uri:
url: "{{ kube_apiserver_endpoint }}/healthz" url: "{{ _kube_apiserver_local_endpoint }}/healthz"
validate_certs: false validate_certs: false
client_cert: "{{ kube_apiserver_client_cert }}" client_cert: "{{ kube_apiserver_client_cert }}"
client_key: "{{ kube_apiserver_client_key }}" client_key: "{{ kube_apiserver_client_key }}"
+7 -24
View File
@@ -1,25 +1,4 @@
--- ---
- name: Set external kube-apiserver endpoint
set_fact:
# noqa: jinja[spacing]
external_apiserver_address: >-
{%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined -%}
{{ loadbalancer_apiserver.address }}
{%- elif kubeconfig_localhost_ansible_host is defined and kubeconfig_localhost_ansible_host -%}
{{ hostvars[groups['kube_control_plane'][0]].ansible_host }}
{%- else -%}
{{ kube_apiserver_access_address }}
{%- endif -%}
# noqa: jinja[spacing]
external_apiserver_port: >-
{%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined and loadbalancer_apiserver.port is defined -%}
{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
{%- else -%}
{{ kube_apiserver_port }}
{%- endif -%}
tags:
- facts
- name: Create kube config dir for current/ansible become user - name: Create kube config dir for current/ansible become user
file: file:
path: "{{ ansible_env.HOME | default('/root') }}/.kube" path: "{{ ansible_env.HOME | default('/root') }}/.kube"
@@ -47,8 +26,8 @@
- name: Wait for k8s apiserver - name: Wait for k8s apiserver
wait_for: wait_for:
host: "{{ kube_apiserver_access_address }}" host: "{{ _kube_apiserver_local_endpoint | urlsplit('hostname') }}"
port: "{{ kube_apiserver_port }}" port: "{{ _kube_apiserver_local_endpoint | urlsplit('port') | default(kube_apiserver_port, true) }}"
timeout: 180 timeout: 180
- name: Create kubeconfig localhost artifacts - name: Create kubeconfig localhost artifacts
@@ -73,9 +52,13 @@
vars: vars:
admin_kubeconfig: "{{ kubeadm_admin_kubeconfig.stdout | from_yaml }}" admin_kubeconfig: "{{ kubeadm_admin_kubeconfig.stdout | from_yaml }}"
context: "kubernetes-admin-{{ cluster_name }}@{{ cluster_name }}" context: "kubernetes-admin-{{ cluster_name }}@{{ cluster_name }}"
client_kubeconfig_endpoint: >-
{{ ('https://' ~ (hostvars[groups['kube_control_plane'][0]].ansible_host | ansible.utils.ipwrap) ~ ':' ~ kube_apiserver_port)
if (kubeconfig_localhost_ansible_host | default(false) and kube_apiserver_endpoint is not defined)
else _kube_apiserver_endpoint }}
override: override:
clusters: clusters:
- "{{ admin_kubeconfig['clusters'][0] | combine({'name': cluster_name, 'cluster': admin_kubeconfig['clusters'][0]['cluster'] | combine({'server': 'https://' + (external_apiserver_address | string | ansible.utils.ipwrap) + ':' + (external_apiserver_port | string)})}, recursive=true) }}" - "{{ admin_kubeconfig['clusters'][0] | combine({'name': cluster_name, 'cluster': admin_kubeconfig['clusters'][0]['cluster'] | combine({'server': client_kubeconfig_endpoint})}, recursive=true) }}"
contexts: contexts:
- "{{ admin_kubeconfig['contexts'][0] | combine({'name': context, 'context': admin_kubeconfig['contexts'][0]['context'] | combine({'cluster': cluster_name})}, recursive=true) }}" - "{{ admin_kubeconfig['contexts'][0] | combine({'name': context, 'context': admin_kubeconfig['contexts'][0]['context'] | combine({'cluster': cluster_name})}, recursive=true) }}"
current-context: "{{ context }}" current-context: "{{ context }}"
@@ -106,7 +106,7 @@
- name: Control plane | wait for the apiserver to be running - name: Control plane | wait for the apiserver to be running
uri: uri:
url: "{{ kube_apiserver_endpoint }}/healthz" url: "{{ _kube_apiserver_local_endpoint }}/healthz"
validate_certs: false validate_certs: false
register: result register: result
until: result.status == 200 until: result.status == 200
@@ -1,16 +1,4 @@
--- ---
- name: Set kubeadm_discovery_address
set_fact:
# noqa: jinja[spacing]
kubeadm_discovery_address: >-
{%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%}
{{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}
{%- else -%}
{{ kube_apiserver_endpoint | regex_replace('https://', '') }}
{%- endif %}
tags:
- facts
- name: Obtain kubeadm certificate key for joining control planes nodes - name: Obtain kubeadm certificate key for joining control planes nodes
when: when:
- not kube_external_ca_mode - not kube_external_ca_mode
@@ -31,8 +19,8 @@
- name: Wait for k8s apiserver - name: Wait for k8s apiserver
wait_for: wait_for:
host: "{{ kubeadm_discovery_address | regex_replace('\\]?:\\d+$', '') | regex_replace('^\\[', '') }}" host: "{{ _kube_apiserver_endpoint | urlsplit('hostname') }}"
port: "{{ kubeadm_discovery_address.split(':')[-1] }}" port: "{{ _kube_apiserver_endpoint | urlsplit('port') | default(kube_apiserver_port, true) }}"
timeout: 180 timeout: 180
@@ -36,8 +36,8 @@
- "localhost" - "localhost"
- "127.0.0.1" - "127.0.0.1"
- "::1" - "::1"
- "{{ apiserver_loadbalancer_domain_name | d('') }}" - "{{ _kube_apiserver_endpoint | urlsplit('hostname') }}"
- "{{ loadbalancer_apiserver.address | d('') }}" - "{{ _kube_apiserver_cluster_internal_endpoint | urlsplit('hostname') }}"
- "{{ supplementary_addresses_in_ssl_keys }}" - "{{ supplementary_addresses_in_ssl_keys }}"
- "{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_access_ip') }}" - "{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_access_ip') }}"
- "{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_ip') }}" - "{{ groups['kube_control_plane'] | map('extract', hostvars, 'main_ip') }}"
@@ -87,12 +87,6 @@
when: kube_apiserver_tracing when: kube_apiserver_tracing
notify: Control plane | Restart apiserver notify: Control plane | Restart apiserver
# Nginx LB(default), If kubeadm_config_api_fqdn is defined, use other LB by kubeadm controlPlaneEndpoint.
- name: Set kubeadm_config_api_fqdn define
set_fact:
kubeadm_config_api_fqdn: "{{ apiserver_loadbalancer_domain_name }}"
when: loadbalancer_apiserver is defined
- name: Kubeadm | Create kubeadm config - name: Kubeadm | Create kubeadm config
template: template:
src: "kubeadm-config.v1beta4.yaml.j2" src: "kubeadm-config.v1beta4.yaml.j2"
@@ -116,8 +116,8 @@ featureGates:
{% endfor %} {% endfor %}
{% endif %} {% endif %}
kubernetesVersion: v{{ kube_version }} kubernetesVersion: v{{ kube_version }}
{% if kubeadm_config_api_fqdn is defined %} {% if kube_apiserver_endpoint is defined %}
controlPlaneEndpoint: "{{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}" controlPlaneEndpoint: "{{ _kube_apiserver_endpoint_netloc }}"
{% else %} {% else %}
controlPlaneEndpoint: "{{ main_ip | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}" controlPlaneEndpoint: "{{ main_ip | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}"
{% endif %} {% endif %}
@@ -6,11 +6,7 @@ discovery:
kubeConfigPath: {{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml kubeConfigPath: {{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml
{% else %} {% else %}
bootstrapToken: bootstrapToken:
{% if kubeadm_config_api_fqdn is defined %} apiServerEndpoint: "{{ _kube_apiserver_endpoint_netloc }}"
apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
{% else %}
apiServerEndpoint: "{{ kubeadm_discovery_address }}"
{% endif %}
token: {{ kubeadm_token }} token: {{ kubeadm_token }}
unsafeSkipCAVerification: true unsafeSkipCAVerification: true
{% endif %} {% endif %}
+7 -53
View File
@@ -1,16 +1,4 @@
--- ---
- name: Set kubeadm_discovery_address
set_fact:
# noqa: jinja[spacing]
kubeadm_discovery_address: >-
{%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%}
{{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}
{%- else -%}
{{ kube_apiserver_endpoint | replace("https://", "") }}
{%- endif %}
tags:
- facts
- name: Check if kubelet.conf exists - name: Check if kubelet.conf exists
stat: stat:
path: "{{ kube_config_dir }}/kubelet.conf" path: "{{ kube_config_dir }}/kubelet.conf"
@@ -104,25 +92,11 @@
lineinfile: lineinfile:
dest: "{{ kube_config_dir }}/kubelet.conf" dest: "{{ kube_config_dir }}/kubelet.conf"
regexp: '^(\s+)server: https.*' regexp: '^(\s+)server: https.*'
line: '\1server: {{ kube_apiserver_endpoint }}' line: '\1server: {{ _kube_apiserver_cluster_internal_endpoint }}'
backrefs: true
backup: true
when:
- kubeadm_config_api_fqdn is not defined
- ('kube_control_plane' not in group_names)
- kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "")
notify: Kubeadm | restart kubelet
- name: Update server field in kubelet kubeconfig - external lb
lineinfile:
dest: "{{ kube_config_dir }}/kubelet.conf"
regexp: '^(\s+)server: https.*'
line: '\1server: {{ kube_apiserver_endpoint }}'
backrefs: true backrefs: true
backup: true backup: true
when: when:
- ('kube_control_plane' not in group_names) - ('kube_control_plane' not in group_names)
- loadbalancer_apiserver is defined
notify: Kubeadm | restart kubelet notify: Kubeadm | restart kubelet
- name: Get current resourceVersion of kube-proxy configmap - name: Get current resourceVersion of kube-proxy configmap
@@ -136,31 +110,14 @@
tags: tags:
- kube-proxy - kube-proxy
# FIXME(mattymo): Need to point to localhost, otherwise control plane nodes will all point
# incorrectly to first control plane node, creating SPoF.
- name: Update server field in kube-proxy kubeconfig - name: Update server field in kube-proxy kubeconfig
shell: >- shell: >-
set -o pipefail && {{ kubectl }} get configmap kube-proxy -n kube-system -o yaml set -o pipefail &&
| sed 's#server:.*#server: https://127.0.0.1:{{ kube_apiserver_port }}#g' if ! {{ kubectl }} get configmap kube-proxy -n kube-system -o yaml | grep -Fq "server: {{ _kube_apiserver_cluster_internal_endpoint }}"; then
| {{ kubectl }} replace -f - {{ kubectl }} get configmap kube-proxy -n kube-system -oyaml |
args: sed 's#server:.*#server: {{ _kube_apiserver_cluster_internal_endpoint }}#g' |
executable: /bin/bash {{ kubectl }} replace -f -
run_once: true fi
delegate_to: "{{ groups['kube_control_plane'] | first }}"
delegate_facts: false
when:
- kubeadm_config_api_fqdn is not defined
- kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "")
- kube_proxy_deployed
- loadbalancer_apiserver_localhost
tags:
- kube-proxy
- name: Update server field in kube-proxy kubeconfig - external lb
shell: >-
set -o pipefail && {{ kubectl }} get configmap kube-proxy -n kube-system -o yaml
| sed 's#server:.*#server: {{kube_apiserver_endpoint}}#g'
| {{ kubectl }} replace -f -
args: args:
executable: /bin/bash executable: /bin/bash
run_once: true run_once: true
@@ -168,7 +125,6 @@
delegate_facts: false delegate_facts: false
when: when:
- kube_proxy_deployed - kube_proxy_deployed
- loadbalancer_apiserver is defined
tags: tags:
- kube-proxy - kube-proxy
@@ -196,8 +152,6 @@
delegate_to: "{{ groups['kube_control_plane'] | first }}" delegate_to: "{{ groups['kube_control_plane'] | first }}"
delegate_facts: false delegate_facts: false
when: when:
- kubeadm_config_api_fqdn is not defined or loadbalancer_apiserver is defined
- kubeadm_discovery_address != kube_apiserver_endpoint | replace("https://", "") or loadbalancer_apiserver is defined
- kube_proxy_deployed - kube_proxy_deployed
- original_configmap_resource_version.stdout != new_configmap_resource_version.stdout - original_configmap_resource_version.stdout != new_configmap_resource_version.stdout
tags: tags:
@@ -7,11 +7,7 @@ discovery:
kubeConfigPath: {{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml kubeConfigPath: {{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml
{% else %} {% else %}
bootstrapToken: bootstrapToken:
{% if kubeadm_config_api_fqdn is defined %} apiServerEndpoint: "{{ _kube_apiserver_endpoint_netloc }}"
apiServerEndpoint: "{{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}"
{% else %}
apiServerEndpoint: "{{ kubeadm_discovery_address }}"
{% endif %}
token: {{ kubeadm_token }} token: {{ kubeadm_token }}
{% if ca_cert_content is defined %} {% if ca_cert_content is defined %}
caCertHashes: caCertHashes:
+1 -1
View File
@@ -1,7 +1,7 @@
--- ---
- name: Kubernetes Apps | Wait for kube-apiserver - name: Kubernetes Apps | Wait for kube-apiserver
uri: uri:
url: "{{ kube_apiserver_endpoint }}/healthz" url: "{{ _kube_apiserver_local_endpoint }}/healthz"
validate_certs: false validate_certs: false
client_cert: "{{ kube_apiserver_client_cert }}" client_cert: "{{ kube_apiserver_client_cert }}"
client_key: "{{ kube_apiserver_client_key }}" client_key: "{{ kube_apiserver_client_key }}"
@@ -107,7 +107,7 @@
# When running this as the last phase ensure we wait for kube-apiserver to come up # When running this as the last phase ensure we wait for kube-apiserver to come up
- name: Preinstall | wait for the apiserver to be running - name: Preinstall | wait for the apiserver to be running
uri: uri:
url: "{{ kube_apiserver_endpoint }}/healthz" url: "{{ _kube_apiserver_local_endpoint }}/healthz"
validate_certs: false validate_certs: false
register: result register: result
until: result.status == 200 until: result.status == 200
+13 -21
View File
@@ -655,28 +655,20 @@ first_kube_control_plane: "{{ groups['kube_control_plane'] | first }}"
kube_apiserver_address: "{{ hostvars[inventory_hostname]['main_ip'] }}" kube_apiserver_address: "{{ hostvars[inventory_hostname]['main_ip'] }}"
kube_apiserver_access_address: "{{ hostvars[inventory_hostname]['main_access_ip'] }}" kube_apiserver_access_address: "{{ hostvars[inventory_hostname]['main_access_ip'] }}"
first_kube_control_plane_address: "{{ hostvars[groups['kube_control_plane'][0]]['main_access_ip'] }}" first_kube_control_plane_address: "{{ hostvars[groups['kube_control_plane'][0]]['main_access_ip'] }}"
loadbalancer_apiserver_localhost: "{{ loadbalancer_apiserver is not defined }}" # Loadbalancer configuration
loadbalancer_apiserver_localhost: "{{ kube_apiserver_endpoint is not defined and kube_apiserver_cluster_internal_endpoint is not defined }}"
loadbalancer_apiserver_type: "nginx" loadbalancer_apiserver_type: "nginx"
# applied if only external loadbalancer_apiserver is defined, otherwise ignored loadbalancer_apiserver_port: "{{ kube_apiserver_port }}"
apiserver_loadbalancer_domain_name: "{{ 'localhost' if loadbalancer_apiserver_localhost else (loadbalancer_apiserver.address | d(undef())) }}"
kube_apiserver_global_endpoint: |- # Primary API endpoint for external access (e.g., kubectl).
{% if loadbalancer_apiserver is defined -%} # User should set this in their inventory if they have a Load Balancer.
https://{{ apiserver_loadbalancer_domain_name | ansible.utils.ipwrap }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} # If not provided, resolved internally in vars.
{%- elif loadbalancer_apiserver_localhost -%} kube_apiserver_endpoint: "{{ undef() }}"
https://localhost:{{ loadbalancer_apiserver_port | default(kube_apiserver_port) }}
{%- else -%} # Internal endpoint for in-cluster traffic, including nodes and pods.
https://{{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }} # Optional override; resolved internally in vars if not provided.
{%- endif %} kube_apiserver_cluster_internal_endpoint: "{{ undef() }}"
kube_apiserver_endpoint: |-
{% if loadbalancer_apiserver is defined -%}
https://{{ apiserver_loadbalancer_domain_name | ansible.utils.ipwrap }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
{%- elif ('kube_control_plane' not in group_names) and loadbalancer_apiserver_localhost -%}
https://localhost:{{ loadbalancer_apiserver_port | default(kube_apiserver_port) }}
{%- elif 'kube_control_plane' in group_names -%}
https://{{ kube_apiserver_bind_address | regex_replace('::', '127.0.0.1') | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}
{%- else -%}
https://{{ first_kube_control_plane_address | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}
{%- endif %}
kube_apiserver_client_cert: "{{ kube_cert_dir }}/ca.crt" kube_apiserver_client_cert: "{{ kube_cert_dir }}/ca.crt"
kube_apiserver_client_key: "{{ kube_cert_dir }}/ca.key" kube_apiserver_client_key: "{{ kube_cert_dir }}/ca.key"
@@ -81,3 +81,40 @@ ssl_ca_dirs_per_os_family:
RedHat: RedHat:
- /etc/pki/tls - /etc/pki/tls
- /etc/pki/ca-trust - /etc/pki/ca-trust
# Internal API server endpoints (cannot be overridden by inventory).
#
# _kube_apiserver_endpoint:
# External API server endpoint used for external access from outside the cluster
# (e.g., in admin kubeconfig written for localhost).
# Resolves to kube_apiserver_endpoint if defined; otherwise defaults to
# https://<first_control_plane_address>:<kube_apiserver_port>.
#
# _kube_apiserver_cluster_internal_endpoint:
# Internal endpoint used for in-cluster communication to the API server by
# cluster components, nodes (kubelet, kube-proxy), and pods (CNI plugins).
# If loadbalancer_apiserver_localhost is enabled, it points to the local load balancer
# proxy (https://127.0.0.1:<loadbalancer_apiserver_port>), otherwise falls back
# to _kube_apiserver_endpoint.
#
# _kube_apiserver_local_endpoint:
# Direct local endpoint to the kube-apiserver process running on the control plane node itself,
# bypassing any load balancer or proxy.
# Used on control plane nodes for local readiness checks, health checks (/healthz),
# and restart handlers.
#
# _kube_apiserver_endpoint_netloc:
# The <host>:<port> representation of _kube_apiserver_endpoint without the scheme.
# Used where kubeadm requires a host:port string (e.g. controlPlaneEndpoint, apiServerEndpoint).
_kube_apiserver_endpoint: >-
{{ kube_apiserver_endpoint if kube_apiserver_endpoint is defined
else 'https://' ~ (first_kube_control_plane_address | ansible.utils.ipwrap) ~ ':' ~ kube_apiserver_port }}
_kube_apiserver_cluster_internal_endpoint: >-
{{ kube_apiserver_cluster_internal_endpoint if kube_apiserver_cluster_internal_endpoint is defined
else (('https://127.0.0.1:' ~ loadbalancer_apiserver_port)
if loadbalancer_apiserver_localhost else _kube_apiserver_endpoint) }}
_kube_apiserver_local_endpoint: "https://{{ kube_apiserver_bind_address | regex_replace('^0\\.0\\.0\\.0$', '127.0.0.1') | regex_replace('^::$', '127.0.0.1') | ansible.utils.ipwrap }}:{{ kube_apiserver_port }}"
_kube_apiserver_endpoint_netloc: "{{ _kube_apiserver_endpoint | urlsplit('netloc') }}"
+2 -2
View File
@@ -46,8 +46,8 @@
| select('defined'))) | select('defined')))
| unique }}" | unique }}"
no_proxy_prepare: no_proxy_prepare:
- "{{ apiserver_loadbalancer_domain_name | d('') }}" - "{{ _kube_apiserver_endpoint | urlsplit('hostname') }}"
- "{{ loadbalancer_apiserver.address if loadbalancer_apiserver is defined else '' }}" - "{{ _kube_apiserver_cluster_internal_endpoint | urlsplit('hostname') }}"
- "{{ hosts_with_no_proxy | map('extract', hostvars, morekeys=['main_access_ip']) }}" - "{{ hosts_with_no_proxy | map('extract', hostvars, morekeys=['main_access_ip']) }}"
- "{{ _hostnames }}" - "{{ _hostnames }}"
- "{{ _hostnames | map('regex_replace', '$', '.' + dns_domain ) }}" - "{{ _hostnames | map('regex_replace', '$', '.' + dns_domain ) }}"
@@ -6,6 +6,6 @@ metadata:
name: kubernetes-services-endpoint name: kubernetes-services-endpoint
data: data:
{% if calico_bpf_enabled or loadbalancer_apiserver_localhost %} {% if calico_bpf_enabled or loadbalancer_apiserver_localhost %}
KUBERNETES_SERVICE_HOST: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}" KUBERNETES_SERVICE_HOST: "{{ _kube_apiserver_cluster_internal_endpoint | urlsplit('hostname') }}"
KUBERNETES_SERVICE_PORT: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}" KUBERNETES_SERVICE_PORT: "{{ _kube_apiserver_cluster_internal_endpoint | urlsplit('port') }}"
{% endif %} {% endif %}
@@ -7,8 +7,8 @@ image:
repository: {{ cilium_image_repo }} repository: {{ cilium_image_repo }}
tag: {{ cilium_image_tag }} tag: {{ cilium_image_tag }}
k8sServiceHost: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}" k8sServiceHost: "{{ _kube_apiserver_cluster_internal_endpoint | urlsplit('hostname') }}"
k8sServicePort: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}" k8sServicePort: "{{ _kube_apiserver_cluster_internal_endpoint | urlsplit('port') }}"
ipv4: ipv4:
enabled: {{ cilium_enable_ipv4 | to_json }} enabled: {{ cilium_enable_ipv4 | to_json }}
@@ -5,7 +5,7 @@ clusters:
- name: cluster - name: cluster
cluster: cluster:
certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
server: {{ kube_apiserver_endpoint }} server: {{ _kube_apiserver_cluster_internal_endpoint }}
users: users:
- name: kube-router - name: kube-router
user: user:
+31 -1
View File
@@ -9,13 +9,43 @@
- name: Fail if removed variables are used - name: Fail if removed variables are used
vars: vars:
# Always remove items from this list after the release in comments # Always remove items from this list after the release in comments
removed_vars: [] removed_vars:
- apiserver_loadbalancer_domain_name # 2.33.0
- kube_apiserver_global_endpoint # 2.33.0
- kubeadm_config_api_fqdn # 2.33.0
- loadbalancer_apiserver # 2.33.0
removed_vars_found: "{{ query('varnames', '^(' + (removed_vars | join('|')) + ')$') }}" removed_vars_found: "{{ query('varnames', '^(' + (removed_vars | join('|')) + ')$') }}"
assert: assert:
that: removed_vars_found | length == 0 that: removed_vars_found | length == 0
fail_msg: "Removed variables present: {{ removed_vars_found | join(', ') }}" fail_msg: "Removed variables present: {{ removed_vars_found | join(', ') }}"
run_once: true run_once: true
- name: Check kube_apiserver_endpoint format
assert:
that:
- kube_apiserver_endpoint | urlsplit('scheme') == 'https'
- kube_apiserver_endpoint | urlsplit('hostname') is not none
- kube_apiserver_endpoint | urlsplit('hostname') | length > 0
- kube_apiserver_endpoint | urlsplit('port') is not none
msg: "kube_apiserver_endpoint must be in the format 'https://<host>:<port>' (e.g., 'https://lb.example.com:6443')"
run_once: true
when:
- kube_apiserver_endpoint is defined
- not ignore_assert_errors
- name: Check kube_apiserver_cluster_internal_endpoint format
assert:
that:
- kube_apiserver_cluster_internal_endpoint | urlsplit('scheme') == 'https'
- kube_apiserver_cluster_internal_endpoint | urlsplit('hostname') is not none
- kube_apiserver_cluster_internal_endpoint | urlsplit('hostname') | length > 0
- kube_apiserver_cluster_internal_endpoint | urlsplit('port') is not none
msg: "kube_apiserver_cluster_internal_endpoint must be in the format 'https://<host>:<port>' (e.g., 'https://lb.internal:6443')"
run_once: true
when:
- kube_apiserver_cluster_internal_endpoint is defined
- not ignore_assert_errors
- name: Stop if kube_control_plane group is empty - name: Stop if kube_control_plane group is empty
assert: assert:
that: groups.get( 'kube_control_plane' ) != [] that: groups.get( 'kube_control_plane' ) != []