diff --git a/docs/advanced/dns-stack.md b/docs/advanced/dns-stack.md index 9524d2f85..ae521e7cc 100644 --- a/docs/advanced/dns-stack.md +++ b/docs/advanced/dns-stack.md @@ -206,10 +206,11 @@ There are three modes available: This activates the classic Kubespray behavior that modifies the hosts ``/etc/resolv.conf`` file and dhclient configuration to point to the cluster dns server (either coredns or coredns_dual, depending on dns_mode). -As cluster DNS is not available on early deployment stage, this mode is split into 2 stages. In the first -stage (``dns_early: true``), ``/etc/resolv.conf`` is configured to use the DNS servers found in ``upstream_dns_servers`` -and ``nameservers``. Later, ``/etc/resolv.conf`` is reconfigured to use the cluster DNS server first, leaving -the other nameservers as backups. +As cluster DNS is not available during early deployment, the ``kubernetes/dns_config`` role automatically +detects whether kubelet has been configured. Before kubelet is set up, ``/etc/resolv.conf`` is configured +to use the DNS servers found in ``upstream_dns_servers`` and ``nameservers``. After the cluster DNS is +running, the ``kubernetes/dns_config`` role is invoked again and reconfigures ``/etc/resolv.conf`` to use +the cluster DNS server first, leaving the other nameservers as backups. Also note, existing records will be purged from the `/etc/resolv.conf`, including resolvconf's base/head/cloud-init config files and those that come from dhclient. diff --git a/extra_playbooks/upgrade-only-k8s.yml b/extra_playbooks/upgrade-only-k8s.yml index bc86efc9a..7c3b1a854 100644 --- a/extra_playbooks/upgrade-only-k8s.yml +++ b/extra_playbooks/upgrade-only-k8s.yml @@ -35,6 +35,7 @@ roles: - { role: kubespray_defaults} - { role: kubernetes/preinstall, tags: preinstall } + - { role: kubernetes/dns_config, tags: [preinstall, resolvconf] } - name: Handle upgrades to control plane components first to maintain backwards compat. hosts: kube_control_plane diff --git a/inventory/sample/group_vars/all/all.yml b/inventory/sample/group_vars/all/all.yml index 9a85592cc..c50a1451c 100644 --- a/inventory/sample/group_vars/all/all.yml +++ b/inventory/sample/group_vars/all/all.yml @@ -30,7 +30,7 @@ loadbalancer_apiserver_healthcheck_port: 8081 ### OTHER OPTIONAL VARIABLES ## By default, Kubespray collects nameservers on the host. It then adds the previously collected nameservers in nameserverentries. -## If true, Kubespray does not include host nameservers in nameserverentries in dns_late stage. However, It uses the nameserver to make sure cluster installed safely in dns_early stage. +## If true, Kubespray does not include host nameservers in nameserverentries after the cluster DNS is running. However, it uses the host nameservers during initial setup to ensure the cluster installs safely. ## Use this option with caution, you may need to define your dns servers. Otherwise, the outbound queries such as www.google.com may fail. # disable_host_nameservers: false diff --git a/playbooks/cluster.yml b/playbooks/cluster.yml index 1bdba4556..57069a9ab 100644 --- a/playbooks/cluster.yml +++ b/playbooks/cluster.yml @@ -13,6 +13,7 @@ roles: - { role: kubespray_defaults } - { role: kubernetes/preinstall, tags: preinstall } + - { role: kubernetes/dns_config, tags: [preinstall, resolvconf] } - { role: "container-engine", tags: "container-engine", when: deploy_container_engine } - { role: download, tags: download, when: "not skip_downloads" } @@ -93,4 +94,4 @@ environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true } + - { role: kubernetes/dns_config, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true } diff --git a/playbooks/reset.yml b/playbooks/reset.yml index 4e0ecb362..35cf902da 100644 --- a/playbooks/reset.yml +++ b/playbooks/reset.yml @@ -31,5 +31,5 @@ environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults} - - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_early: true } + - { role: kubernetes/dns_config, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_early: true } - { role: reset, tags: reset } diff --git a/playbooks/scale.yml b/playbooks/scale.yml index 501268632..9f1d9813c 100644 --- a/playbooks/scale.yml +++ b/playbooks/scale.yml @@ -19,6 +19,7 @@ roles: - { role: kubespray_defaults, when: "not skip_downloads and download_run_once and not download_localhost" } - { role: kubernetes/preinstall, tags: preinstall, when: "not skip_downloads and download_run_once and not download_localhost" } + - { role: kubernetes/dns_config, tags: [preinstall, resolvconf], when: "not skip_downloads and download_run_once and not download_localhost" } - { role: download, tags: download, when: "not skip_downloads and download_run_once and not download_localhost" } - name: Target only workers to get kubelet installed and checking in on any new nodes(engine) @@ -29,6 +30,7 @@ roles: - { role: kubespray_defaults } - { role: kubernetes/preinstall, tags: preinstall } + - { role: kubernetes/dns_config, tags: [preinstall, resolvconf] } - { role: container-engine, tags: "container-engine", when: deploy_container_engine } - { role: download, tags: download, when: "not skip_downloads" } - role: etcd @@ -90,4 +92,4 @@ environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true } + - { role: kubernetes/dns_config, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true } diff --git a/playbooks/upgrade_cluster.yml b/playbooks/upgrade_cluster.yml index 0562bbe2c..887cbf2a2 100644 --- a/playbooks/upgrade_cluster.yml +++ b/playbooks/upgrade_cluster.yml @@ -13,6 +13,7 @@ roles: - { role: kubespray_defaults, when: "not skip_downloads and download_run_once and not download_localhost"} - { role: kubernetes/preinstall, tags: preinstall, when: "not skip_downloads and download_run_once and not download_localhost" } + - { role: kubernetes/dns_config, tags: [preinstall, resolvconf], when: "not skip_downloads and download_run_once and not download_localhost" } - { role: download, tags: download, when: "not skip_downloads and download_run_once and not download_localhost" } - name: Prepare nodes for upgrade @@ -23,6 +24,7 @@ roles: - { role: kubespray_defaults } - { role: kubernetes/preinstall, tags: preinstall } + - { role: kubernetes/dns_config, tags: [preinstall, resolvconf] } - { role: download, tags: download, when: "not skip_downloads" } - name: Upgrade container engine on non-cluster nodes @@ -129,4 +131,4 @@ environment: "{{ proxy_disable_env }}" roles: - { role: kubespray_defaults } - - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true } + - { role: kubernetes/dns_config, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true } diff --git a/roles/kubernetes/dns_config/defaults/main.yml b/roles/kubernetes/dns_config/defaults/main.yml new file mode 100644 index 000000000..00625624d --- /dev/null +++ b/roles/kubernetes/dns_config/defaults/main.yml @@ -0,0 +1,22 @@ +--- +nameservers: [] +cloud_resolver: [] +disable_host_nameservers: false +# Kubespray sets this to true after clusterDNS is running to apply changes to the host resolv.conf +dns_late: false +# DNS resolver option timeout and retry attempts. +dns_timeout: 2 +dns_attempts: 2 + +# Set to false to disable the backup parameter, set to true to accumulate backups of config files. +leave_etc_backup_files: true + +# Remove default cluster search domains (``default.svc.{{ dns_domain }}, svc.{{ dns_domain }}``). +remove_default_searchdomains: false + +# Flatcar Container Linux by Kinvolk cloud init config file to define /etc/resolv.conf content +# for hostnet pods and infra needs +resolveconf_cloud_init_conf: /etc/resolveconf_cloud_init.conf + +# Sets DNSStubListener=no, useful if you get "0.0.0.0:53: bind: address already in use" +systemd_resolved_disable_stub_listener: "{{ ansible_facts['os_family'] == 'Flatcar' }}" diff --git a/roles/kubernetes/dns_config/handlers/main.yml b/roles/kubernetes/dns_config/handlers/main.yml new file mode 100644 index 000000000..8bf717d7d --- /dev/null +++ b/roles/kubernetes/dns_config/handlers/main.yml @@ -0,0 +1,126 @@ +--- +- name: DNS config | apply resolvconf cloud-init + command: /usr/bin/coreos-cloudinit --from-file {{ resolveconf_cloud_init_conf }} + when: ansible_facts['os_family'] == "Flatcar" + listen: DNS config | propagate resolvconf to k8s components + +- name: DNS config | reload NetworkManager + service: + name: NetworkManager.service + state: restarted + listen: DNS config | update resolvconf for networkmanager + +- name: DNS config | reload kubelet + service: + name: kubelet + state: restarted + notify: + - DNS config | kube-controller configured + - DNS config | kube-apiserver configured + - DNS config | restart kube-controller-manager docker + - DNS config | restart kube-controller-manager crio/containerd + - DNS config | restart kube-apiserver docker + - DNS config | restart kube-apiserver crio/containerd + when: not dns_early | bool + listen: + - DNS config | propagate resolvconf to k8s components + - DNS config | update resolvconf for networkmanager + +# FIXME(mattymo): Also restart for kubeadm mode +- name: DNS config | kube-apiserver configured + stat: + path: "{{ kube_manifest_dir }}/kube-apiserver.yaml" + get_attributes: false + get_checksum: false + get_mime: false + register: kube_apiserver_set + when: ('kube_control_plane' in group_names) and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' + listen: DNS config | propagate resolvconf to k8s components + +# FIXME(mattymo): Also restart for kubeadm mode +- name: DNS config | kube-controller configured + stat: + path: "{{ kube_manifest_dir }}/kube-controller-manager.yaml" + get_attributes: false + get_checksum: false + get_mime: false + register: kube_controller_set + when: ('kube_control_plane' in group_names) and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' + listen: DNS config | propagate resolvconf to k8s components + +- name: DNS config | restart kube-controller-manager docker + shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -f name=k8s_POD_kube-controller-manager* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f" + args: + executable: /bin/bash + when: + - container_manager == "docker" + - ('kube_control_plane' in group_names) + - dns_mode != 'none' + - resolvconf_mode == 'host_resolvconf' + - kube_controller_set.stat.exists + listen: DNS config | propagate resolvconf to k8s components + +- name: DNS config | restart kube-controller-manager crio/containerd + shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-controller-manager* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'" + args: + executable: /bin/bash + register: dns_config_restart_controller_manager + retries: 10 + delay: 1 + until: dns_config_restart_controller_manager.rc == 0 + when: + - container_manager in ['crio', 'containerd'] + - ('kube_control_plane' in group_names) + - dns_mode != 'none' + - resolvconf_mode == 'host_resolvconf' + - kube_controller_set.stat.exists + listen: DNS config | propagate resolvconf to k8s components + +- name: DNS config | restart kube-apiserver docker + shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -f name=k8s_POD_kube-apiserver* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f" + args: + executable: /bin/bash + when: + - container_manager == "docker" + - ('kube_control_plane' in group_names) + - dns_mode != 'none' + - resolvconf_mode == 'host_resolvconf' + - kube_apiserver_set.stat.exists + listen: DNS config | propagate resolvconf to k8s components + +- name: DNS config | restart kube-apiserver crio/containerd + shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-apiserver* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'" + args: + executable: /bin/bash + register: dns_config_restart_apiserver + retries: 10 + until: dns_config_restart_apiserver.rc == 0 + delay: 1 + when: + - container_manager in ['crio', 'containerd'] + - ('kube_control_plane' in group_names) + - dns_mode != 'none' + - resolvconf_mode == 'host_resolvconf' + - kube_apiserver_set.stat.exists + listen: DNS config | propagate resolvconf to k8s components + +# When running this as the last phase ensure we wait for kube-apiserver to come up +- name: DNS config | wait for the apiserver to be running + uri: + url: "{{ _kube_apiserver_local_endpoint }}/healthz" + validate_certs: false + register: result + until: result.status == 200 + retries: 60 + delay: 1 + when: + - dns_late + - ('kube_control_plane' in group_names) + - dns_mode != 'none' + - resolvconf_mode == 'host_resolvconf' + listen: DNS config | propagate resolvconf to k8s components + +- name: DNS config | Restart systemd-resolved + service: + name: systemd-resolved + state: restarted diff --git a/roles/kubernetes/preinstall/tasks/0110-dhclient-hooks-undo.yml b/roles/kubernetes/dns_config/tasks/dhclient-hooks-undo.yml similarity index 79% rename from roles/kubernetes/preinstall/tasks/0110-dhclient-hooks-undo.yml rename to roles/kubernetes/dns_config/tasks/dhclient-hooks-undo.yml index 0bc23c1be..1272299cd 100644 --- a/roles/kubernetes/preinstall/tasks/0110-dhclient-hooks-undo.yml +++ b/roles/kubernetes/dns_config/tasks/dhclient-hooks-undo.yml @@ -9,10 +9,10 @@ state: absent backup: "{{ leave_etc_backup_files }}" marker: "# Ansible entries {mark}" - notify: Preinstall | propagate resolvconf to k8s components + notify: DNS config | propagate resolvconf to k8s components - name: Remove kubespray specific dhclient hook file: path: "{{ dhclienthookfile }}" state: absent - notify: Preinstall | propagate resolvconf to k8s components + notify: DNS config | propagate resolvconf to k8s components diff --git a/roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml b/roles/kubernetes/dns_config/tasks/dhclient-hooks.yml similarity index 85% rename from roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml rename to roles/kubernetes/dns_config/tasks/dhclient-hooks.yml index 7076389e5..32197f618 100644 --- a/roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml +++ b/roles/kubernetes/dns_config/tasks/dhclient-hooks.yml @@ -17,7 +17,7 @@ backup: "{{ leave_etc_backup_files }}" marker: "# Ansible entries {mark}" mode: "0644" - notify: Preinstall | propagate resolvconf to k8s components + notify: DNS config | propagate resolvconf to k8s components - name: Configure dhclient hooks for resolv.conf (non-RH) template: @@ -25,7 +25,7 @@ dest: "{{ dhclienthookfile }}" owner: root mode: "0755" - notify: Preinstall | propagate resolvconf to k8s components + notify: DNS config | propagate resolvconf to k8s components when: ansible_facts['os_family'] not in [ "RedHat", "Suse" ] - name: Configure dhclient hooks for resolv.conf (RH-only) @@ -34,5 +34,5 @@ dest: "{{ dhclienthookfile }}" owner: root mode: "0755" - notify: Preinstall | propagate resolvconf to k8s components + notify: DNS config | propagate resolvconf to k8s components when: ansible_facts['os_family'] == "RedHat" diff --git a/roles/kubernetes/dns_config/tasks/main.yml b/roles/kubernetes/dns_config/tasks/main.yml new file mode 100644 index 000000000..4e23b14e0 --- /dev/null +++ b/roles/kubernetes/dns_config/tasks/main.yml @@ -0,0 +1,75 @@ +--- +- name: Set DNS facts + import_tasks: set_facts.yml + tags: + - resolvconf + - facts + +- name: Stop if /etc/resolv.conf has no configured nameservers + assert: + that: configured_nameservers | length > 0 + fail_msg: "nameserver should not be empty in /etc/resolv.conf" + when: + - upstream_dns_servers | length == 0 + - not disable_host_nameservers + - dns_mode in ['coredns', 'coredns_dual'] + tags: + - asserts + +- name: Apply resolvconf settings + import_tasks: resolvconf.yml + when: + - dns_mode != 'none' + - resolvconf_mode == 'host_resolvconf' + - systemd_resolved_enabled.rc != 0 + - networkmanager_enabled.rc != 0 + tags: + - bootstrap_os + - resolvconf + +- name: Apply systemd-resolved settings + import_tasks: systemd-resolved.yml + when: + - dns_mode != 'none' + - resolvconf_mode == 'host_resolvconf' + - systemd_resolved_enabled.rc == 0 + tags: + - bootstrap_os + - resolvconf + +- name: Apply networkmanager DNS settings + import_tasks: networkmanager-dns.yml + when: + - dns_mode != 'none' + - resolvconf_mode == 'host_resolvconf' + - networkmanager_enabled.rc == 0 + tags: + - bootstrap_os + - resolvconf + +- name: Configure dhclient + import_tasks: dhclient-hooks.yml + when: + - dns_mode != 'none' + - resolvconf_mode == 'host_resolvconf' + - dhclientconffile is defined + - ansible_facts['os_family'] != "Flatcar" + tags: + - bootstrap_os + - resolvconf + +- name: Configure dhclient dhclient hooks + import_tasks: dhclient-hooks-undo.yml + when: + - dns_mode != 'none' + - resolvconf_mode != 'host_resolvconf' + - dhclientconffile is defined + - ansible_facts['os_family'] != "Flatcar" + tags: + - bootstrap_os + - resolvconf + +# We need to make sure the network is restarted early enough so that docker can later pick up the correct system +# nameservers and search domains +- name: Flush handlers + meta: flush_handlers diff --git a/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml b/roles/kubernetes/dns_config/tasks/networkmanager-dns.yml similarity index 88% rename from roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml rename to roles/kubernetes/dns_config/tasks/networkmanager-dns.yml index d31908424..e3f776bbb 100644 --- a/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml +++ b/roles/kubernetes/dns_config/tasks/networkmanager-dns.yml @@ -10,7 +10,7 @@ when: - ('127.0.0.53' not in nameserverentries or systemd_resolved_enabled.rc != 0) - notify: Preinstall | update resolvconf for networkmanager + notify: DNS config | update resolvconf for networkmanager - name: Set default dns if remove_default_searchdomains is false set_fact: @@ -25,7 +25,7 @@ value: "{{ (default_searchdomains | default([]) + searchdomains) | join(',') }}" mode: '0600' backup: "{{ leave_etc_backup_files }}" - notify: Preinstall | update resolvconf for networkmanager + notify: DNS config | update resolvconf for networkmanager - name: NetworkManager | Add DNS options to NM configuration community.general.ini_file: @@ -35,4 +35,4 @@ value: "ndots:{{ ndots }},timeout:{{ dns_timeout }},attempts:{{ dns_attempts }}" mode: '0600' backup: "{{ leave_etc_backup_files }}" - notify: Preinstall | update resolvconf for networkmanager + notify: DNS config | update resolvconf for networkmanager diff --git a/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml b/roles/kubernetes/dns_config/tasks/resolvconf.yml similarity index 88% rename from roles/kubernetes/preinstall/tasks/0060-resolvconf.yml rename to roles/kubernetes/dns_config/tasks/resolvconf.yml index 2d485dab7..fc67bb1a7 100644 --- a/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml +++ b/roles/kubernetes/dns_config/tasks/resolvconf.yml @@ -19,7 +19,7 @@ backup: "{{ not resolvconf_stat.stat.islnk }}" marker: "# Ansible entries {mark}" mode: "0644" - notify: Preinstall | propagate resolvconf to k8s components + notify: DNS config | propagate resolvconf to k8s components - name: Remove search/domain/nameserver options before block replace: @@ -29,7 +29,7 @@ with_nested: - "{{ [resolvconffile, base | default(''), head | default('')] | select | list }}" - [ 'search\s', 'nameserver\s', 'domain\s', 'options\s' ] - notify: Preinstall | propagate resolvconf to k8s components + notify: DNS config | propagate resolvconf to k8s components - name: Remove search/domain/nameserver options after block replace: @@ -40,7 +40,7 @@ with_nested: - "{{ [resolvconffile, base | default(''), head | default('')] | select | list }}" - [ 'search\s', 'nameserver\s', 'domain\s', 'options\s' ] - notify: Preinstall | propagate resolvconf to k8s components + notify: DNS config | propagate resolvconf to k8s components - name: Get temporary resolveconf cloud init file content command: cat {{ resolvconffile }} @@ -53,5 +53,5 @@ src: resolvconf.j2 owner: root mode: "0644" - notify: Preinstall | propagate resolvconf to k8s components + notify: DNS config | propagate resolvconf to k8s components when: ansible_facts['os_family'] == "Flatcar" diff --git a/roles/kubernetes/dns_config/tasks/set_facts.yml b/roles/kubernetes/dns_config/tasks/set_facts.yml new file mode 100644 index 000000000..b336605a0 --- /dev/null +++ b/roles/kubernetes/dns_config/tasks/set_facts.yml @@ -0,0 +1,122 @@ +--- +- name: Check resolvconf + command: which resolvconf + register: resolvconf + failed_when: false + changed_when: false + check_mode: false + +- name: Check existence of /etc/resolvconf/resolv.conf.d + stat: + path: /etc/resolvconf/resolv.conf.d + get_attributes: false + get_checksum: false + get_mime: false + failed_when: false + register: resolvconfd_path + +- name: Check status of /etc/resolv.conf + stat: + path: /etc/resolv.conf + follow: false + get_attributes: false + get_checksum: false + get_mime: false + failed_when: false + register: resolvconf_stat + + # Used in vars/ +- name: Fetch resolv.conf + when: resolvconf_stat.stat.exists + slurp: + src: /etc/resolv.conf + register: resolvconf_slurp + +- name: NetworkManager | Check if host has NetworkManager + # noqa command-instead-of-module - Should we use service_facts for this? + command: systemctl is-active --quiet NetworkManager.service + register: networkmanager_enabled + failed_when: false + changed_when: false + check_mode: false + +- name: Check systemd-resolved + # noqa command-instead-of-module - Should we use service_facts for this? + command: systemctl is-active systemd-resolved + register: systemd_resolved_enabled + failed_when: false + changed_when: false + check_mode: false + +- name: Set default dns if remove_default_searchdomains is false + set_fact: + default_searchdomains: ["default.svc.{{ dns_domain }}", "svc.{{ dns_domain }}"] + when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | length == 0) + +- name: Set dns facts + set_fact: + resolvconf: >- + {%- if resolvconf.rc == 0 and resolvconfd_path.stat.isdir is defined and resolvconfd_path.stat.isdir -%}true{%- else -%}false{%- endif -%} + +- name: Check if kubelet is configured + stat: + path: "{{ kube_config_dir }}/kubelet.env" + get_attributes: false + get_checksum: false + get_mime: false + register: kubelet_configured + changed_when: false + +- name: Check if early DNS configuration stage + set_fact: + dns_early: "{{ not kubelet_configured.stat.exists }}" + +- name: Target resolv.conf files + set_fact: + resolvconffile: /etc/resolv.conf + base: >- + {%- if resolvconf | bool -%}/etc/resolvconf/resolv.conf.d/base{%- endif -%} + head: >- + {%- if resolvconf | bool -%}/etc/resolvconf/resolv.conf.d/head{%- endif -%} + when: ansible_facts['os_family'] != "Flatcar" and not is_fedora_coreos + +- name: Target temporary resolvconf cloud init file (Flatcar Container Linux by Kinvolk / Fedora CoreOS) + set_fact: + resolvconffile: /tmp/resolveconf_cloud_init_conf + when: ansible_facts['os_family'] == "Flatcar" or is_fedora_coreos + +- name: Check if /etc/dhclient.conf exists + stat: + path: /etc/dhclient.conf + get_attributes: false + get_checksum: false + get_mime: false + register: dhclient_stat + +- name: Target dhclient conf file for /etc/dhclient.conf + set_fact: + dhclientconffile: /etc/dhclient.conf + when: dhclient_stat.stat.exists + +- name: Check if /etc/dhcp/dhclient.conf exists + stat: + path: /etc/dhcp/dhclient.conf + get_attributes: false + get_checksum: false + get_mime: false + register: dhcp_dhclient_stat + +- name: Target dhclient conf file for /etc/dhcp/dhclient.conf + set_fact: + dhclientconffile: /etc/dhcp/dhclient.conf + when: dhcp_dhclient_stat.stat.exists + +- name: Target dhclient hook file for Red Hat family + set_fact: + dhclienthookfile: /etc/dhcp/dhclient.d/zdnsupdate.sh + when: ansible_facts['os_family'] == "RedHat" + +- name: Target dhclient hook file for Debian family + set_fact: + dhclienthookfile: /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate + when: ansible_facts['os_family'] == "Debian" diff --git a/roles/kubernetes/preinstall/tasks/0061-systemd-resolved.yml b/roles/kubernetes/dns_config/tasks/systemd-resolved.yml similarity index 87% rename from roles/kubernetes/preinstall/tasks/0061-systemd-resolved.yml rename to roles/kubernetes/dns_config/tasks/systemd-resolved.yml index f1aa8f5c3..5cdba5696 100644 --- a/roles/kubernetes/preinstall/tasks/0061-systemd-resolved.yml +++ b/roles/kubernetes/dns_config/tasks/systemd-resolved.yml @@ -12,4 +12,4 @@ owner: root group: root mode: "0644" - notify: Preinstall | Restart systemd-resolved + notify: DNS config | Restart systemd-resolved diff --git a/roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2 b/roles/kubernetes/dns_config/templates/dhclient_dnsupdate.sh.j2 similarity index 100% rename from roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2 rename to roles/kubernetes/dns_config/templates/dhclient_dnsupdate.sh.j2 diff --git a/roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2 b/roles/kubernetes/dns_config/templates/dhclient_dnsupdate_rh.sh.j2 similarity index 100% rename from roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2 rename to roles/kubernetes/dns_config/templates/dhclient_dnsupdate_rh.sh.j2 diff --git a/roles/kubernetes/preinstall/templates/resolvconf.j2 b/roles/kubernetes/dns_config/templates/resolvconf.j2 similarity index 100% rename from roles/kubernetes/preinstall/templates/resolvconf.j2 rename to roles/kubernetes/dns_config/templates/resolvconf.j2 diff --git a/roles/kubernetes/preinstall/templates/resolved.conf.j2 b/roles/kubernetes/dns_config/templates/resolved.conf.j2 similarity index 100% rename from roles/kubernetes/preinstall/templates/resolved.conf.j2 rename to roles/kubernetes/dns_config/templates/resolved.conf.j2 diff --git a/roles/kubernetes/preinstall/vars/main.yml b/roles/kubernetes/dns_config/vars/main.yml similarity index 100% rename from roles/kubernetes/preinstall/vars/main.yml rename to roles/kubernetes/dns_config/vars/main.yml diff --git a/roles/kubernetes/preinstall/defaults/main.yml b/roles/kubernetes/preinstall/defaults/main.yml index f7e22edd0..499612d62 100644 --- a/roles/kubernetes/preinstall/defaults/main.yml +++ b/roles/kubernetes/preinstall/defaults/main.yml @@ -3,23 +3,11 @@ ignore_assert_errors: false # Set to false to disable the backup parameter, set to true to accumulate backups of config files. leave_etc_backup_files: true -nameservers: [] -cloud_resolver: [] -disable_host_nameservers: false -# Kubespray sets this to true after clusterDNS is running to apply changes to the host resolv.conf -dns_late: false -# DNS resolver option timeout and retry attempts. -dns_timeout: 2 -dns_attempts: 2 - # Set to true if your network does not support IPv6 # This may be necessary for pulling Docker images from # GCE docker repository disable_ipv6_dns: false -# Remove default cluster search domains (``default.svc.{{ dns_domain }}, svc.{{ dns_domain }}``). -remove_default_searchdomains: false - kube_owner: kube kube_cert_group: kube-cert kube_config_dir: /etc/kubernetes @@ -27,10 +15,6 @@ kube_cert_dir: "{{ kube_config_dir }}/ssl" kube_cert_compat_dir: /etc/kubernetes/pki kubelet_flexvolumes_plugins_dir: /usr/libexec/kubernetes/kubelet-plugins/volume/exec -# Flatcar Container Linux by Kinvolk cloud init config file to define /etc/resolv.conf content -# for hostnet pods and infra needs -resolveconf_cloud_init_conf: /etc/resolveconf_cloud_init.conf - # sysctl_file_path to add sysctl conf to sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf" @@ -106,9 +90,6 @@ redhat_os_family_extensions: - "UnionTech" - "UniontechOS" -# Sets DNSStubListener=no, useful if you get "0.0.0.0:53: bind: address already in use" -systemd_resolved_disable_stub_listener: "{{ ansible_facts['os_family'] == 'Flatcar' }}" - # Used to disable File Access Policy Daemon service. # If service is enabled, the CNI plugin installation will fail disable_fapolicyd: true diff --git a/roles/kubernetes/preinstall/handlers/main.yml b/roles/kubernetes/preinstall/handlers/main.yml index 9bfec95aa..16b3daf81 100644 --- a/roles/kubernetes/preinstall/handlers/main.yml +++ b/roles/kubernetes/preinstall/handlers/main.yml @@ -1,129 +1,8 @@ --- -- name: Preinstall | apply resolvconf cloud-init - command: /usr/bin/coreos-cloudinit --from-file {{ resolveconf_cloud_init_conf }} - when: ansible_facts['os_family'] == "Flatcar" - listen: Preinstall | propagate resolvconf to k8s components - - name: Preinstall | reload NetworkManager service: name: NetworkManager.service state: restarted - listen: Preinstall | update resolvconf for networkmanager - -- name: Preinstall | reload kubelet - service: - name: kubelet - state: restarted - notify: - - Preinstall | kube-controller configured - - Preinstall | kube-apiserver configured - - Preinstall | restart kube-controller-manager docker - - Preinstall | restart kube-controller-manager crio/containerd - - Preinstall | restart kube-apiserver docker - - Preinstall | restart kube-apiserver crio/containerd - when: not dns_early | bool - listen: - - Preinstall | propagate resolvconf to k8s components - - Preinstall | update resolvconf for networkmanager - -# FIXME(mattymo): Also restart for kubeadm mode -- name: Preinstall | kube-apiserver configured - stat: - path: "{{ kube_manifest_dir }}/kube-apiserver.yaml" - get_attributes: false - get_checksum: false - get_mime: false - register: kube_apiserver_set - when: ('kube_control_plane' in group_names) and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' - listen: Preinstall | propagate resolvconf to k8s components - -# FIXME(mattymo): Also restart for kubeadm mode -- name: Preinstall | kube-controller configured - stat: - path: "{{ kube_manifest_dir }}/kube-controller-manager.yaml" - get_attributes: false - get_checksum: false - get_mime: false - register: kube_controller_set - when: ('kube_control_plane' in group_names) and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' - listen: Preinstall | propagate resolvconf to k8s components - -- name: Preinstall | restart kube-controller-manager docker - shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -f name=k8s_POD_kube-controller-manager* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f" - args: - executable: /bin/bash - when: - - container_manager == "docker" - - ('kube_control_plane' in group_names) - - dns_mode != 'none' - - resolvconf_mode == 'host_resolvconf' - - kube_controller_set.stat.exists - listen: Preinstall | propagate resolvconf to k8s components - -- name: Preinstall | restart kube-controller-manager crio/containerd - shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-controller-manager* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'" - args: - executable: /bin/bash - register: preinstall_restart_controller_manager - retries: 10 - delay: 1 - until: preinstall_restart_controller_manager.rc == 0 - when: - - container_manager in ['crio', 'containerd'] - - ('kube_control_plane' in group_names) - - dns_mode != 'none' - - resolvconf_mode == 'host_resolvconf' - - kube_controller_set.stat.exists - listen: Preinstall | propagate resolvconf to k8s components - -- name: Preinstall | restart kube-apiserver docker - shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -f name=k8s_POD_kube-apiserver* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f" - args: - executable: /bin/bash - when: - - container_manager == "docker" - - ('kube_control_plane' in group_names) - - dns_mode != 'none' - - resolvconf_mode == 'host_resolvconf' - - kube_apiserver_set.stat.exists - listen: Preinstall | propagate resolvconf to k8s components - -- name: Preinstall | restart kube-apiserver crio/containerd - shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-apiserver* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'" - args: - executable: /bin/bash - register: preinstall_restart_apiserver - retries: 10 - until: preinstall_restart_apiserver.rc == 0 - delay: 1 - when: - - container_manager in ['crio', 'containerd'] - - ('kube_control_plane' in group_names) - - dns_mode != 'none' - - resolvconf_mode == 'host_resolvconf' - - kube_apiserver_set.stat.exists - listen: Preinstall | propagate resolvconf to k8s components - -# When running this as the last phase ensure we wait for kube-apiserver to come up -- name: Preinstall | wait for the apiserver to be running - uri: - url: "{{ _kube_apiserver_local_endpoint }}/healthz" - validate_certs: false - register: result - until: result.status == 200 - retries: 60 - delay: 1 - when: - - dns_late - - ('kube_control_plane' in group_names) - - dns_mode != 'none' - - resolvconf_mode == 'host_resolvconf' - listen: Preinstall | propagate resolvconf to k8s components - -- name: Preinstall | Restart systemd-resolved - service: - name: systemd-resolved - state: restarted - name: Preinstall | restart ntp service: diff --git a/roles/kubernetes/preinstall/tasks/0020-set_facts.yml b/roles/kubernetes/preinstall/tasks/0020-set_facts.yml index 474981eba..66b23677a 100644 --- a/roles/kubernetes/preinstall/tasks/0020-set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/0020-set_facts.yml @@ -18,39 +18,6 @@ tags: - facts -- name: Check resolvconf - command: which resolvconf - register: resolvconf - failed_when: false - changed_when: false - check_mode: false - -- name: Check existence of /etc/resolvconf/resolv.conf.d - stat: - path: /etc/resolvconf/resolv.conf.d - get_attributes: false - get_checksum: false - get_mime: false - failed_when: false - register: resolvconfd_path - -- name: Check status of /etc/resolv.conf - stat: - path: /etc/resolv.conf - follow: false - get_attributes: false - get_checksum: false - get_mime: false - failed_when: false - register: resolvconf_stat - - # Used in vars/ -- name: Fetch resolv.conf - when: resolvconf_stat.stat.exists - slurp: - src: /etc/resolv.conf - register: resolvconf_slurp - - name: NetworkManager | Check if host has NetworkManager # noqa command-instead-of-module - Should we use service_facts for this? command: systemctl is-active --quiet NetworkManager.service @@ -59,87 +26,6 @@ changed_when: false check_mode: false -- name: Check systemd-resolved - # noqa command-instead-of-module - Should we use service_facts for this? - command: systemctl is-active systemd-resolved - register: systemd_resolved_enabled - failed_when: false - changed_when: false - check_mode: false - -- name: Set default dns if remove_default_searchdomains is false - set_fact: - default_searchdomains: ["default.svc.{{ dns_domain }}", "svc.{{ dns_domain }}"] - when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | length == 0) - -- name: Set dns facts - set_fact: - resolvconf: >- - {%- if resolvconf.rc == 0 and resolvconfd_path.stat.isdir is defined and resolvconfd_path.stat.isdir -%}true{%- else -%}false{%- endif -%} - -- name: Check if kubelet is configured - stat: - path: "{{ kube_config_dir }}/kubelet.env" - get_attributes: false - get_checksum: false - get_mime: false - register: kubelet_configured - changed_when: false - -- name: Check if early DNS configuration stage - set_fact: - dns_early: "{{ not kubelet_configured.stat.exists }}" - -- name: Target resolv.conf files - set_fact: - resolvconffile: /etc/resolv.conf - base: >- - {%- if resolvconf | bool -%}/etc/resolvconf/resolv.conf.d/base{%- endif -%} - head: >- - {%- if resolvconf | bool -%}/etc/resolvconf/resolv.conf.d/head{%- endif -%} - when: ansible_facts['os_family'] != "Flatcar" and not is_fedora_coreos - -- name: Target temporary resolvconf cloud init file (Flatcar Container Linux by Kinvolk / Fedora CoreOS) - set_fact: - resolvconffile: /tmp/resolveconf_cloud_init_conf - when: ansible_facts['os_family'] == "Flatcar" or is_fedora_coreos - -- name: Check if /etc/dhclient.conf exists - stat: - path: /etc/dhclient.conf - get_attributes: false - get_checksum: false - get_mime: false - register: dhclient_stat - -- name: Target dhclient conf file for /etc/dhclient.conf - set_fact: - dhclientconffile: /etc/dhclient.conf - when: dhclient_stat.stat.exists - -- name: Check if /etc/dhcp/dhclient.conf exists - stat: - path: /etc/dhcp/dhclient.conf - get_attributes: false - get_checksum: false - get_mime: false - register: dhcp_dhclient_stat - -- name: Target dhclient conf file for /etc/dhcp/dhclient.conf - set_fact: - dhclientconffile: /etc/dhcp/dhclient.conf - when: dhcp_dhclient_stat.stat.exists - -- name: Target dhclient hook file for Red Hat family - set_fact: - dhclienthookfile: /etc/dhcp/dhclient.d/zdnsupdate.sh - when: ansible_facts['os_family'] == "RedHat" - -- name: Target dhclient hook file for Debian family - set_fact: - dhclienthookfile: /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate - when: ansible_facts['os_family'] == "Debian" - - name: Set etcd vars if using kubeadm mode set_fact: etcd_cert_dir: "{{ kube_cert_dir }}" diff --git a/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml index d72957a1b..dc6044cd3 100644 --- a/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml +++ b/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml @@ -102,15 +102,6 @@ msg: "Hostname must consist of lower case alphanumeric characters, '.' or '-', and must start and end with an alphanumeric character" when: not ignore_assert_errors -- name: Stop if /etc/resolv.conf has no configured nameservers - assert: - that: configured_nameservers | length>0 - fail_msg: "nameserver should not be empty in /etc/resolv.conf" - when: - - upstream_dns_servers | length == 0 - - not disable_host_nameservers - - dns_mode in ['coredns', 'coredns_dual'] - - name: Stop if download_localhost is enabled for Flatcar Container Linux assert: that: ansible_facts['os_family'] != "Flatcar" diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml index e55153e66..014d8ba68 100644 --- a/roles/kubernetes/preinstall/tasks/main.yml +++ b/roles/kubernetes/preinstall/tasks/main.yml @@ -3,47 +3,20 @@ - name: Disable swap import_tasks: 0010-swapoff.yml when: - - not dns_late - kubelet_fail_swap_on - name: Set facts import_tasks: 0020-set_facts.yml tags: - - resolvconf - facts - name: Check settings import_tasks: 0040-verify-settings.yml - when: - - not dns_late tags: - asserts - name: Create directories import_tasks: 0050-create_directories.yml - when: - - not dns_late - -- name: Apply resolvconf settings - import_tasks: 0060-resolvconf.yml - when: - - dns_mode != 'none' - - resolvconf_mode == 'host_resolvconf' - - systemd_resolved_enabled.rc != 0 - - networkmanager_enabled.rc != 0 - tags: - - bootstrap_os - - resolvconf - -- name: Apply systemd-resolved settings - import_tasks: 0061-systemd-resolved.yml - when: - - dns_mode != 'none' - - resolvconf_mode == 'host_resolvconf' - - systemd_resolved_enabled.rc == 0 - tags: - - bootstrap_os - - resolvconf - name: Apply networkmanager unmanaged devices settings import_tasks: 0062-networkmanager-unmanaged-devices.yml @@ -52,58 +25,18 @@ tags: - bootstrap_os -- name: Apply networkmanager DNS settings - import_tasks: 0063-networkmanager-dns.yml - when: - - dns_mode != 'none' - - resolvconf_mode == 'host_resolvconf' - - networkmanager_enabled.rc == 0 - tags: - - bootstrap_os - - resolvconf - - name: Apply system configurations import_tasks: 0080-system-configurations.yml - when: - - not dns_late tags: - bootstrap_os - name: Configure NTP import_tasks: 0081-ntp-configurations.yml when: - - not dns_late - ntp_enabled tags: - bootstrap_os -- name: Configure dhclient - import_tasks: 0100-dhclient-hooks.yml - when: - - dns_mode != 'none' - - resolvconf_mode == 'host_resolvconf' - - dhclientconffile is defined - - ansible_facts['os_family'] != "Flatcar" - tags: - - bootstrap_os - - resolvconf - -- name: Configure dhclient dhclient hooks - import_tasks: 0110-dhclient-hooks-undo.yml - when: - - dns_mode != 'none' - - resolvconf_mode != 'host_resolvconf' - - dhclientconffile is defined - - ansible_facts['os_family'] != "Flatcar" - tags: - - bootstrap_os - - resolvconf - -# We need to make sure the network is restarted early enough so that docker can later pick up the correct system -# nameservers and search domains -- name: Flush handlers - meta: flush_handlers - - name: Check if we are running inside a Azure VM stat: path: /var/lib/waagent/ @@ -111,8 +44,6 @@ get_checksum: false get_mime: false register: azure_check - when: - - not dns_late tags: - bootstrap_os