refactor: extract DNS configuration from kubernetes/preinstall into dns_config role (#13603)

Extract tasks related to host DNS management from kubernetes/preinstall
into a new kubernetes/dns_config role.

Previously, kubernetes/preinstall was re-invoked at the end of cluster
playbooks with dns_late: true to apply resolvconf changes after cluster
DNS came up. Because of this reuse, several unrelated tasks in preinstall
were guarded with 'when: not dns_late'.

Move resolvconf, systemd-resolved, networkmanager-dns, and dhclient tasks,
along with their associated templates, facts, variables, and handlers, into
kubernetes/dns_config. Update cluster.yml, upgrade_cluster.yml, scale.yml,
and reset.yml to invoke the new role directly and remove dns_late guards.

Signed-off-by: Dasmat13 <gptdasmat@gmail.com>

docs/advanced/dns-stack.md

@@ -206,10 +206,11 @@ There are three modes available:
 This activates the classic Kubespray behavior that modifies the hosts ``/etc/resolv.conf`` file and dhclient
 configuration to point to the cluster dns server (either coredns or coredns_dual, depending on dns_mode).
 
-As cluster DNS is not available on early deployment stage, this mode is split into 2 stages. In the first
+As cluster DNS is not available during early deployment, the ``kubernetes/dns_config`` role automatically
-stage (``dns_early: true``), ``/etc/resolv.conf`` is configured to use the DNS servers found in ``upstream_dns_servers``
+detects whether kubelet has been configured. Before kubelet is set up, ``/etc/resolv.conf`` is configured
-and ``nameservers``. Later, ``/etc/resolv.conf`` is reconfigured to use the cluster DNS server first, leaving
+to use the DNS servers found in ``upstream_dns_servers`` and ``nameservers``. After the cluster DNS is
-the other nameservers as backups.
+running, the ``kubernetes/dns_config`` role is invoked again and reconfigures ``/etc/resolv.conf`` to use
+the cluster DNS server first, leaving the other nameservers as backups.
 
 Also note, existing records will be purged from the `/etc/resolv.conf`,
 including resolvconf's base/head/cloud-init config files and those that come from dhclient.

extra_playbooks/upgrade-only-k8s.yml

@@ -35,6 +35,7 @@
   roles:
     - { role: kubespray_defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
+    - { role: kubernetes/dns_config, tags: [preinstall, resolvconf] }
 
 - name: Handle upgrades to control plane components first to maintain backwards compat.
   hosts: kube_control_plane

inventory/sample/group_vars/all/all.yml

@@ -30,7 +30,7 @@ loadbalancer_apiserver_healthcheck_port: 8081
 ### OTHER OPTIONAL VARIABLES
 
 ## By default, Kubespray collects nameservers on the host. It then adds the previously collected nameservers in nameserverentries.
-## If true, Kubespray does not include host nameservers in nameserverentries in dns_late stage. However, It uses the nameserver to make sure cluster installed safely in dns_early stage.
+## If true, Kubespray does not include host nameservers in nameserverentries after the cluster DNS is running. However, it uses the host nameservers during initial setup to ensure the cluster installs safely.
 ## Use this option with caution, you may need to define your dns servers. Otherwise, the outbound queries such as www.google.com may fail.
 # disable_host_nameservers: false
 

playbooks/cluster.yml

@@ -13,6 +13,7 @@
   roles:
     - { role: kubespray_defaults }
     - { role: kubernetes/preinstall, tags: preinstall }
+    - { role: kubernetes/dns_config, tags: [preinstall, resolvconf] }
     - { role: "container-engine", tags: "container-engine", when: deploy_container_engine }
     - { role: download, tags: download, when: "not skip_downloads" }
 
@@ -93,4 +94,4 @@
   environment: "{{ proxy_disable_env }}"
   roles:
     - { role: kubespray_defaults }
-    - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }
+    - { role: kubernetes/dns_config, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }

playbooks/reset.yml

@@ -31,5 +31,5 @@
   environment: "{{ proxy_disable_env }}"
   roles:
     - { role: kubespray_defaults}
-    - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_early: true }
+    - { role: kubernetes/dns_config, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_early: true }
     - { role: reset, tags: reset }

playbooks/scale.yml

@@ -19,6 +19,7 @@
   roles:
     - { role: kubespray_defaults, when: "not skip_downloads and download_run_once and not download_localhost" }
     - { role: kubernetes/preinstall, tags: preinstall, when: "not skip_downloads and download_run_once and not download_localhost" }
+    - { role: kubernetes/dns_config, tags: [preinstall, resolvconf], when: "not skip_downloads and download_run_once and not download_localhost" }
     - { role: download, tags: download, when: "not skip_downloads and download_run_once and not download_localhost" }
 
 - name: Target only workers to get kubelet installed and checking in on any new nodes(engine)
@@ -29,6 +30,7 @@
   roles:
     - { role: kubespray_defaults }
     - { role: kubernetes/preinstall, tags: preinstall }
+    - { role: kubernetes/dns_config, tags: [preinstall, resolvconf] }
     - { role: container-engine, tags: "container-engine", when: deploy_container_engine }
     - { role: download, tags: download, when: "not skip_downloads" }
     - role: etcd
@@ -90,4 +92,4 @@
   environment: "{{ proxy_disable_env }}"
   roles:
     - { role: kubespray_defaults }
-    - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }
+    - { role: kubernetes/dns_config, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }

playbooks/upgrade_cluster.yml

@@ -13,6 +13,7 @@
   roles:
     - { role: kubespray_defaults, when: "not skip_downloads and download_run_once and not download_localhost"}
     - { role: kubernetes/preinstall, tags: preinstall, when: "not skip_downloads and download_run_once and not download_localhost" }
+    - { role: kubernetes/dns_config, tags: [preinstall, resolvconf], when: "not skip_downloads and download_run_once and not download_localhost" }
     - { role: download, tags: download, when: "not skip_downloads and download_run_once and not download_localhost" }
 
 - name: Prepare nodes for upgrade
@@ -23,6 +24,7 @@
   roles:
     - { role: kubespray_defaults }
     - { role: kubernetes/preinstall, tags: preinstall }
+    - { role: kubernetes/dns_config, tags: [preinstall, resolvconf] }
     - { role: download, tags: download, when: "not skip_downloads" }
 
 - name: Upgrade container engine on non-cluster nodes
@@ -129,4 +131,4 @@
   environment: "{{ proxy_disable_env }}"
   roles:
     - { role: kubespray_defaults }
-    - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }
+    - { role: kubernetes/dns_config, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }

roles/kubernetes/dns_config/defaults/main.yml

@@ -0,0 +1,22 @@
+---
+nameservers: []
+cloud_resolver: []
+disable_host_nameservers: false
+# Kubespray sets this to true after clusterDNS is running to apply changes to the host resolv.conf
+dns_late: false
+# DNS resolver option timeout and retry attempts.
+dns_timeout: 2
+dns_attempts: 2
+
+# Set to false to disable the backup parameter, set to true to accumulate backups of config files.
+leave_etc_backup_files: true
+
+# Remove default cluster search domains (``default.svc.{{ dns_domain }}, svc.{{ dns_domain }}``).
+remove_default_searchdomains: false
+
+# Flatcar Container Linux by Kinvolk cloud init config file to define /etc/resolv.conf content
+# for hostnet pods and infra needs
+resolveconf_cloud_init_conf: /etc/resolveconf_cloud_init.conf
+
+# Sets DNSStubListener=no, useful if you get "0.0.0.0:53: bind: address already in use"
+systemd_resolved_disable_stub_listener: "{{ ansible_facts['os_family'] == 'Flatcar' }}"

roles/kubernetes/dns_config/handlers/main.yml

@@ -0,0 +1,126 @@
+---
+- name: DNS config | apply resolvconf cloud-init
+  command: /usr/bin/coreos-cloudinit --from-file {{ resolveconf_cloud_init_conf }}
+  when: ansible_facts['os_family'] == "Flatcar"
+  listen: DNS config | propagate resolvconf to k8s components
+
+- name: DNS config | reload NetworkManager
+  service:
+    name: NetworkManager.service
+    state: restarted
+  listen: DNS config | update resolvconf for networkmanager
+
+- name: DNS config | reload kubelet
+  service:
+    name: kubelet
+    state: restarted
+  notify:
+    - DNS config | kube-controller configured
+    - DNS config | kube-apiserver configured
+    - DNS config | restart kube-controller-manager docker
+    - DNS config | restart kube-controller-manager crio/containerd
+    - DNS config | restart kube-apiserver docker
+    - DNS config | restart kube-apiserver crio/containerd
+  when: not dns_early | bool
+  listen:
+    - DNS config | propagate resolvconf to k8s components
+    - DNS config | update resolvconf for networkmanager
+
+# FIXME(mattymo): Also restart for kubeadm mode
+- name: DNS config | kube-apiserver configured
+  stat:
+    path: "{{ kube_manifest_dir }}/kube-apiserver.yaml"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: kube_apiserver_set
+  when: ('kube_control_plane' in group_names) and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'
+  listen: DNS config | propagate resolvconf to k8s components
+
+# FIXME(mattymo): Also restart for kubeadm mode
+- name: DNS config | kube-controller configured
+  stat:
+    path: "{{ kube_manifest_dir }}/kube-controller-manager.yaml"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: kube_controller_set
+  when: ('kube_control_plane' in group_names) and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'
+  listen: DNS config | propagate resolvconf to k8s components
+
+- name: DNS config | restart kube-controller-manager docker
+  shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -f name=k8s_POD_kube-controller-manager* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f"
+  args:
+    executable: /bin/bash
+  when:
+    - container_manager == "docker"
+    - ('kube_control_plane' in group_names)
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+    - kube_controller_set.stat.exists
+  listen: DNS config | propagate resolvconf to k8s components
+
+- name: DNS config | restart kube-controller-manager crio/containerd
+  shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-controller-manager* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'"
+  args:
+    executable: /bin/bash
+  register: dns_config_restart_controller_manager
+  retries: 10
+  delay: 1
+  until: dns_config_restart_controller_manager.rc == 0
+  when:
+    - container_manager in ['crio', 'containerd']
+    - ('kube_control_plane' in group_names)
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+    - kube_controller_set.stat.exists
+  listen: DNS config | propagate resolvconf to k8s components
+
+- name: DNS config | restart kube-apiserver docker
+  shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -f name=k8s_POD_kube-apiserver* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f"
+  args:
+    executable: /bin/bash
+  when:
+    - container_manager == "docker"
+    - ('kube_control_plane' in group_names)
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+    - kube_apiserver_set.stat.exists
+  listen: DNS config | propagate resolvconf to k8s components
+
+- name: DNS config | restart kube-apiserver crio/containerd
+  shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-apiserver* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'"
+  args:
+    executable: /bin/bash
+  register: dns_config_restart_apiserver
+  retries: 10
+  until: dns_config_restart_apiserver.rc == 0
+  delay: 1
+  when:
+    - container_manager in ['crio', 'containerd']
+    - ('kube_control_plane' in group_names)
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+    - kube_apiserver_set.stat.exists
+  listen: DNS config | propagate resolvconf to k8s components
+
+# When running this as the last phase ensure we wait for kube-apiserver to come up
+- name: DNS config | wait for the apiserver to be running
+  uri:
+    url: "{{ _kube_apiserver_local_endpoint }}/healthz"
+    validate_certs: false
+  register: result
+  until: result.status == 200
+  retries: 60
+  delay: 1
+  when:
+    - dns_late
+    - ('kube_control_plane' in group_names)
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+  listen: DNS config | propagate resolvconf to k8s components
+
+- name: DNS config | Restart systemd-resolved
+  service:
+    name: systemd-resolved
+    state: restarted

roles/kubernetes/dns_config/tasks/dhclient-hooks-undo.yml

@@ -9,10 +9,10 @@
     state: absent
     backup: "{{ leave_etc_backup_files }}"
     marker: "# Ansible entries {mark}"
-  notify: Preinstall | propagate resolvconf to k8s components
+  notify: DNS config | propagate resolvconf to k8s components
 
 - name: Remove kubespray specific dhclient hook
   file:
     path: "{{ dhclienthookfile }}"
     state: absent
-  notify: Preinstall | propagate resolvconf to k8s components
+  notify: DNS config | propagate resolvconf to k8s components

roles/kubernetes/dns_config/tasks/dhclient-hooks.yml

@@ -17,7 +17,7 @@
     backup: "{{ leave_etc_backup_files }}"
     marker: "# Ansible entries {mark}"
     mode: "0644"
-  notify: Preinstall | propagate resolvconf to k8s components
+  notify: DNS config | propagate resolvconf to k8s components
 
 - name: Configure dhclient hooks for resolv.conf (non-RH)
   template:
@@ -25,7 +25,7 @@
     dest: "{{ dhclienthookfile }}"
     owner: root
     mode: "0755"
-  notify: Preinstall | propagate resolvconf to k8s components
+  notify: DNS config | propagate resolvconf to k8s components
   when: ansible_facts['os_family'] not in [ "RedHat", "Suse" ]
 
 - name: Configure dhclient hooks for resolv.conf (RH-only)
@@ -34,5 +34,5 @@
     dest: "{{ dhclienthookfile }}"
     owner: root
     mode: "0755"
-  notify: Preinstall | propagate resolvconf to k8s components
+  notify: DNS config | propagate resolvconf to k8s components
   when: ansible_facts['os_family'] == "RedHat"

roles/kubernetes/dns_config/tasks/main.yml

@@ -0,0 +1,75 @@
+---
+- name: Set DNS facts
+  import_tasks: set_facts.yml
+  tags:
+    - resolvconf
+    - facts
+
+- name: Stop if /etc/resolv.conf has no configured nameservers
+  assert:
+    that: configured_nameservers | length > 0
+    fail_msg: "nameserver should not be empty in /etc/resolv.conf"
+  when:
+    - upstream_dns_servers | length == 0
+    - not disable_host_nameservers
+    - dns_mode in ['coredns', 'coredns_dual']
+  tags:
+    - asserts
+
+- name: Apply resolvconf settings
+  import_tasks: resolvconf.yml
+  when:
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+    - systemd_resolved_enabled.rc != 0
+    - networkmanager_enabled.rc != 0
+  tags:
+    - bootstrap_os
+    - resolvconf
+
+- name: Apply systemd-resolved settings
+  import_tasks: systemd-resolved.yml
+  when:
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+    - systemd_resolved_enabled.rc == 0
+  tags:
+    - bootstrap_os
+    - resolvconf
+
+- name: Apply networkmanager DNS settings
+  import_tasks: networkmanager-dns.yml
+  when:
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+    - networkmanager_enabled.rc == 0
+  tags:
+    - bootstrap_os
+    - resolvconf
+
+- name: Configure dhclient
+  import_tasks: dhclient-hooks.yml
+  when:
+    - dns_mode != 'none'
+    - resolvconf_mode == 'host_resolvconf'
+    - dhclientconffile is defined
+    - ansible_facts['os_family'] != "Flatcar"
+  tags:
+    - bootstrap_os
+    - resolvconf
+
+- name: Configure dhclient dhclient hooks
+  import_tasks: dhclient-hooks-undo.yml
+  when:
+    - dns_mode != 'none'
+    - resolvconf_mode != 'host_resolvconf'
+    - dhclientconffile is defined
+    - ansible_facts['os_family'] != "Flatcar"
+  tags:
+    - bootstrap_os
+    - resolvconf
+
+# We need to make sure the network is restarted early enough so that docker can later pick up the correct system
+# nameservers and search domains
+- name: Flush handlers
+  meta: flush_handlers

roles/kubernetes/dns_config/tasks/networkmanager-dns.yml

@@ -10,7 +10,7 @@
   when:
     - ('127.0.0.53' not in nameserverentries
        or systemd_resolved_enabled.rc != 0)
-  notify: Preinstall | update resolvconf for networkmanager
+  notify: DNS config | update resolvconf for networkmanager
 
 - name: Set default dns if remove_default_searchdomains is false
   set_fact:
@@ -25,7 +25,7 @@
     value: "{{ (default_searchdomains | default([]) + searchdomains) | join(',') }}"
     mode: '0600'
     backup: "{{ leave_etc_backup_files }}"
-  notify: Preinstall | update resolvconf for networkmanager
+  notify: DNS config | update resolvconf for networkmanager
 
 - name: NetworkManager | Add DNS options to NM configuration
   community.general.ini_file:
@@ -35,4 +35,4 @@
     value: "ndots:{{ ndots }},timeout:{{ dns_timeout }},attempts:{{ dns_attempts }}"
     mode: '0600'
     backup: "{{ leave_etc_backup_files }}"
-  notify: Preinstall | update resolvconf for networkmanager
+  notify: DNS config | update resolvconf for networkmanager

roles/kubernetes/dns_config/tasks/resolvconf.yml

@@ -19,7 +19,7 @@
     backup: "{{ not resolvconf_stat.stat.islnk }}"
     marker: "# Ansible entries {mark}"
     mode: "0644"
-  notify: Preinstall | propagate resolvconf to k8s components
+  notify: DNS config | propagate resolvconf to k8s components
 
 - name: Remove search/domain/nameserver options before block
   replace:
@@ -29,7 +29,7 @@
   with_nested:
     - "{{ [resolvconffile, base | default(''), head | default('')] | select | list }}"
     - [ 'search\s', 'nameserver\s', 'domain\s', 'options\s' ]
-  notify: Preinstall | propagate resolvconf to k8s components
+  notify: DNS config | propagate resolvconf to k8s components
 
 - name: Remove search/domain/nameserver options after block
   replace:
@@ -40,7 +40,7 @@
   with_nested:
     - "{{ [resolvconffile, base | default(''), head | default('')] | select | list }}"
     - [ 'search\s', 'nameserver\s', 'domain\s', 'options\s' ]
-  notify: Preinstall | propagate resolvconf to k8s components
+  notify: DNS config | propagate resolvconf to k8s components
 
 - name: Get temporary resolveconf cloud init file content
   command: cat {{ resolvconffile }}
@@ -53,5 +53,5 @@
     src: resolvconf.j2
     owner: root
     mode: "0644"
-  notify: Preinstall | propagate resolvconf to k8s components
+  notify: DNS config | propagate resolvconf to k8s components
   when: ansible_facts['os_family'] == "Flatcar"

roles/kubernetes/dns_config/tasks/set_facts.yml

@@ -0,0 +1,122 @@
+---
+- name: Check resolvconf
+  command: which resolvconf
+  register: resolvconf
+  failed_when: false
+  changed_when: false
+  check_mode: false
+
+- name: Check existence of /etc/resolvconf/resolv.conf.d
+  stat:
+    path: /etc/resolvconf/resolv.conf.d
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  failed_when: false
+  register: resolvconfd_path
+
+- name: Check status of /etc/resolv.conf
+  stat:
+    path: /etc/resolv.conf
+    follow: false
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  failed_when: false
+  register: resolvconf_stat
+
+  # Used in vars/
+- name: Fetch resolv.conf
+  when: resolvconf_stat.stat.exists
+  slurp:
+    src: /etc/resolv.conf
+  register: resolvconf_slurp
+
+- name: NetworkManager | Check if host has NetworkManager
+  # noqa command-instead-of-module - Should we use service_facts for this?
+  command: systemctl is-active --quiet NetworkManager.service
+  register: networkmanager_enabled
+  failed_when: false
+  changed_when: false
+  check_mode: false
+
+- name: Check systemd-resolved
+  # noqa command-instead-of-module - Should we use service_facts for this?
+  command: systemctl is-active systemd-resolved
+  register: systemd_resolved_enabled
+  failed_when: false
+  changed_when: false
+  check_mode: false
+
+- name: Set default dns if remove_default_searchdomains is false
+  set_fact:
+    default_searchdomains: ["default.svc.{{ dns_domain }}", "svc.{{ dns_domain }}"]
+  when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | length == 0)
+
+- name: Set dns facts
+  set_fact:
+    resolvconf: >-
+      {%- if resolvconf.rc == 0 and resolvconfd_path.stat.isdir is defined and resolvconfd_path.stat.isdir -%}true{%- else -%}false{%- endif -%}
+
+- name: Check if kubelet is configured
+  stat:
+    path: "{{ kube_config_dir }}/kubelet.env"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: kubelet_configured
+  changed_when: false
+
+- name: Check if early DNS configuration stage
+  set_fact:
+    dns_early: "{{ not kubelet_configured.stat.exists }}"
+
+- name: Target resolv.conf files
+  set_fact:
+    resolvconffile: /etc/resolv.conf
+    base: >-
+      {%- if resolvconf | bool -%}/etc/resolvconf/resolv.conf.d/base{%- endif -%}
+    head: >-
+      {%- if resolvconf | bool -%}/etc/resolvconf/resolv.conf.d/head{%- endif -%}
+  when: ansible_facts['os_family'] != "Flatcar" and not is_fedora_coreos
+
+- name: Target temporary resolvconf cloud init file (Flatcar Container Linux by Kinvolk / Fedora CoreOS)
+  set_fact:
+    resolvconffile: /tmp/resolveconf_cloud_init_conf
+  when: ansible_facts['os_family'] == "Flatcar" or is_fedora_coreos
+
+- name: Check if /etc/dhclient.conf exists
+  stat:
+    path: /etc/dhclient.conf
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: dhclient_stat
+
+- name: Target dhclient conf file for /etc/dhclient.conf
+  set_fact:
+    dhclientconffile: /etc/dhclient.conf
+  when: dhclient_stat.stat.exists
+
+- name: Check if /etc/dhcp/dhclient.conf exists
+  stat:
+    path: /etc/dhcp/dhclient.conf
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: dhcp_dhclient_stat
+
+- name: Target dhclient conf file for /etc/dhcp/dhclient.conf
+  set_fact:
+    dhclientconffile: /etc/dhcp/dhclient.conf
+  when: dhcp_dhclient_stat.stat.exists
+
+- name: Target dhclient hook file for Red Hat family
+  set_fact:
+    dhclienthookfile: /etc/dhcp/dhclient.d/zdnsupdate.sh
+  when: ansible_facts['os_family'] == "RedHat"
+
+- name: Target dhclient hook file for Debian family
+  set_fact:
+    dhclienthookfile: /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate
+  when: ansible_facts['os_family'] == "Debian"

roles/kubernetes/dns_config/tasks/systemd-resolved.yml

@@ -12,4 +12,4 @@
     owner: root
     group: root
     mode: "0644"
-  notify: Preinstall | Restart systemd-resolved
+  notify: DNS config | Restart systemd-resolved

roles/kubernetes/preinstall/defaults/main.yml

@@ -3,23 +3,11 @@
 ignore_assert_errors: false
 # Set to false to disable the backup parameter, set to true to accumulate backups of config files.
 leave_etc_backup_files: true
-nameservers: []
-cloud_resolver: []
-disable_host_nameservers: false
-# Kubespray sets this to true after clusterDNS is running to apply changes to the host resolv.conf
-dns_late: false
-# DNS resolver option timeout and retry attempts.
-dns_timeout: 2
-dns_attempts: 2
-
 # Set to true if your network does not support IPv6
 # This may be necessary for pulling Docker images from
 # GCE docker repository
 disable_ipv6_dns: false
 
-# Remove default cluster search domains (``default.svc.{{ dns_domain }}, svc.{{ dns_domain }}``).
-remove_default_searchdomains: false
-
 kube_owner: kube
 kube_cert_group: kube-cert
 kube_config_dir: /etc/kubernetes
@@ -27,10 +15,6 @@ kube_cert_dir: "{{ kube_config_dir }}/ssl"
 kube_cert_compat_dir: /etc/kubernetes/pki
 kubelet_flexvolumes_plugins_dir: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
 
-# Flatcar Container Linux by Kinvolk cloud init config file to define /etc/resolv.conf content
-# for hostnet pods and infra needs
-resolveconf_cloud_init_conf: /etc/resolveconf_cloud_init.conf
-
 # sysctl_file_path to add sysctl conf to
 sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf"
 
@@ -106,9 +90,6 @@ redhat_os_family_extensions:
   - "UnionTech"
   - "UniontechOS"
 
-# Sets DNSStubListener=no, useful if you get "0.0.0.0:53: bind: address already in use"
-systemd_resolved_disable_stub_listener: "{{ ansible_facts['os_family'] == 'Flatcar' }}"
-
 # Used to disable File Access Policy Daemon service.
 # If service is enabled, the CNI plugin installation will fail
 disable_fapolicyd: true

roles/kubernetes/preinstall/handlers/main.yml

@@ -1,129 +1,8 @@
 ---
-- name: Preinstall | apply resolvconf cloud-init
-  command: /usr/bin/coreos-cloudinit --from-file {{ resolveconf_cloud_init_conf }}
-  when: ansible_facts['os_family'] == "Flatcar"
-  listen: Preinstall | propagate resolvconf to k8s components
-
 - name: Preinstall | reload NetworkManager
   service:
     name: NetworkManager.service
     state: restarted
-  listen: Preinstall | update resolvconf for networkmanager
-
-- name: Preinstall | reload kubelet
-  service:
-    name: kubelet
-    state: restarted
-  notify:
-    - Preinstall | kube-controller configured
-    - Preinstall | kube-apiserver configured
-    - Preinstall | restart kube-controller-manager docker
-    - Preinstall | restart kube-controller-manager crio/containerd
-    - Preinstall | restart kube-apiserver docker
-    - Preinstall | restart kube-apiserver crio/containerd
-  when: not dns_early | bool
-  listen:
-    - Preinstall | propagate resolvconf to k8s components
-    - Preinstall | update resolvconf for networkmanager
-
-# FIXME(mattymo): Also restart for kubeadm mode
-- name: Preinstall | kube-apiserver configured
-  stat:
-    path: "{{ kube_manifest_dir }}/kube-apiserver.yaml"
-    get_attributes: false
-    get_checksum: false
-    get_mime: false
-  register: kube_apiserver_set
-  when: ('kube_control_plane' in group_names) and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'
-  listen: Preinstall | propagate resolvconf to k8s components
-
-# FIXME(mattymo): Also restart for kubeadm mode
-- name: Preinstall | kube-controller configured
-  stat:
-    path: "{{ kube_manifest_dir }}/kube-controller-manager.yaml"
-    get_attributes: false
-    get_checksum: false
-    get_mime: false
-  register: kube_controller_set
-  when: ('kube_control_plane' in group_names) and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'
-  listen: Preinstall | propagate resolvconf to k8s components
-
-- name: Preinstall | restart kube-controller-manager docker
-  shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -f name=k8s_POD_kube-controller-manager* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f"
-  args:
-    executable: /bin/bash
-  when:
-    - container_manager == "docker"
-    - ('kube_control_plane' in group_names)
-    - dns_mode != 'none'
-    - resolvconf_mode == 'host_resolvconf'
-    - kube_controller_set.stat.exists
-  listen: Preinstall | propagate resolvconf to k8s components
-
-- name: Preinstall | restart kube-controller-manager crio/containerd
-  shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-controller-manager* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'"
-  args:
-    executable: /bin/bash
-  register: preinstall_restart_controller_manager
-  retries: 10
-  delay: 1
-  until: preinstall_restart_controller_manager.rc == 0
-  when:
-    - container_manager in ['crio', 'containerd']
-    - ('kube_control_plane' in group_names)
-    - dns_mode != 'none'
-    - resolvconf_mode == 'host_resolvconf'
-    - kube_controller_set.stat.exists
-  listen: Preinstall | propagate resolvconf to k8s components
-
-- name: Preinstall | restart kube-apiserver docker
-  shell: "set -o pipefail && {{ docker_bin_dir }}/docker ps -f name=k8s_POD_kube-apiserver* -q | xargs --no-run-if-empty {{ docker_bin_dir }}/docker rm -f"
-  args:
-    executable: /bin/bash
-  when:
-    - container_manager == "docker"
-    - ('kube_control_plane' in group_names)
-    - dns_mode != 'none'
-    - resolvconf_mode == 'host_resolvconf'
-    - kube_apiserver_set.stat.exists
-  listen: Preinstall | propagate resolvconf to k8s components
-
-- name: Preinstall | restart kube-apiserver crio/containerd
-  shell: "set -o pipefail && {{ bin_dir }}/crictl pods --name kube-apiserver* -q | xargs -I% --no-run-if-empty bash -c '{{ bin_dir }}/crictl stopp % && {{ bin_dir }}/crictl rmp %'"
-  args:
-    executable: /bin/bash
-  register: preinstall_restart_apiserver
-  retries: 10
-  until: preinstall_restart_apiserver.rc == 0
-  delay: 1
-  when:
-    - container_manager in ['crio', 'containerd']
-    - ('kube_control_plane' in group_names)
-    - dns_mode != 'none'
-    - resolvconf_mode == 'host_resolvconf'
-    - kube_apiserver_set.stat.exists
-  listen: Preinstall | propagate resolvconf to k8s components
-
-# When running this as the last phase ensure we wait for kube-apiserver to come up
-- name: Preinstall | wait for the apiserver to be running
-  uri:
-    url: "{{ _kube_apiserver_local_endpoint }}/healthz"
-    validate_certs: false
-  register: result
-  until: result.status == 200
-  retries: 60
-  delay: 1
-  when:
-    - dns_late
-    - ('kube_control_plane' in group_names)
-    - dns_mode != 'none'
-    - resolvconf_mode == 'host_resolvconf'
-  listen: Preinstall | propagate resolvconf to k8s components
-
-- name: Preinstall | Restart systemd-resolved
-  service:
-    name: systemd-resolved
-    state: restarted
 
 - name: Preinstall | restart ntp
   service:

roles/kubernetes/preinstall/tasks/0020-set_facts.yml

@@ -18,39 +18,6 @@
   tags:
     - facts
 
-- name: Check resolvconf
-  command: which resolvconf
-  register: resolvconf
-  failed_when: false
-  changed_when: false
-  check_mode: false
-
-- name: Check existence of /etc/resolvconf/resolv.conf.d
-  stat:
-    path: /etc/resolvconf/resolv.conf.d
-    get_attributes: false
-    get_checksum: false
-    get_mime: false
-  failed_when: false
-  register: resolvconfd_path
-
-- name: Check status of /etc/resolv.conf
-  stat:
-    path: /etc/resolv.conf
-    follow: false
-    get_attributes: false
-    get_checksum: false
-    get_mime: false
-  failed_when: false
-  register: resolvconf_stat
-
-  # Used in vars/
-- name: Fetch resolv.conf
-  when: resolvconf_stat.stat.exists
-  slurp:
-    src: /etc/resolv.conf
-  register: resolvconf_slurp
-
 - name: NetworkManager | Check if host has NetworkManager
   # noqa command-instead-of-module - Should we use service_facts for this?
   command: systemctl is-active --quiet NetworkManager.service
@@ -59,87 +26,6 @@
   changed_when: false
   check_mode: false
 
-- name: Check systemd-resolved
-  # noqa command-instead-of-module - Should we use service_facts for this?
-  command: systemctl is-active systemd-resolved
-  register: systemd_resolved_enabled
-  failed_when: false
-  changed_when: false
-  check_mode: false
-
-- name: Set default dns if remove_default_searchdomains is false
-  set_fact:
-    default_searchdomains: ["default.svc.{{ dns_domain }}", "svc.{{ dns_domain }}"]
-  when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | length == 0)
-
-- name: Set dns facts
-  set_fact:
-    resolvconf: >-
-      {%- if resolvconf.rc == 0 and resolvconfd_path.stat.isdir is defined and resolvconfd_path.stat.isdir -%}true{%- else -%}false{%- endif -%}
-
-- name: Check if kubelet is configured
-  stat:
-    path: "{{ kube_config_dir }}/kubelet.env"
-    get_attributes: false
-    get_checksum: false
-    get_mime: false
-  register: kubelet_configured
-  changed_when: false
-
-- name: Check if early DNS configuration stage
-  set_fact:
-    dns_early: "{{ not kubelet_configured.stat.exists }}"
-
-- name: Target resolv.conf files
-  set_fact:
-    resolvconffile: /etc/resolv.conf
-    base: >-
-      {%- if resolvconf | bool -%}/etc/resolvconf/resolv.conf.d/base{%- endif -%}
-    head: >-
-      {%- if resolvconf | bool -%}/etc/resolvconf/resolv.conf.d/head{%- endif -%}
-  when: ansible_facts['os_family'] != "Flatcar" and not is_fedora_coreos
-
-- name: Target temporary resolvconf cloud init file (Flatcar Container Linux by Kinvolk / Fedora CoreOS)
-  set_fact:
-    resolvconffile: /tmp/resolveconf_cloud_init_conf
-  when: ansible_facts['os_family'] == "Flatcar" or is_fedora_coreos
-
-- name: Check if /etc/dhclient.conf exists
-  stat:
-    path: /etc/dhclient.conf
-    get_attributes: false
-    get_checksum: false
-    get_mime: false
-  register: dhclient_stat
-
-- name: Target dhclient conf file for /etc/dhclient.conf
-  set_fact:
-    dhclientconffile: /etc/dhclient.conf
-  when: dhclient_stat.stat.exists
-
-- name: Check if /etc/dhcp/dhclient.conf exists
-  stat:
-    path: /etc/dhcp/dhclient.conf
-    get_attributes: false
-    get_checksum: false
-    get_mime: false
-  register: dhcp_dhclient_stat
-
-- name: Target dhclient conf file for /etc/dhcp/dhclient.conf
-  set_fact:
-    dhclientconffile: /etc/dhcp/dhclient.conf
-  when: dhcp_dhclient_stat.stat.exists
-
-- name: Target dhclient hook file for Red Hat family
-  set_fact:
-    dhclienthookfile: /etc/dhcp/dhclient.d/zdnsupdate.sh
-  when: ansible_facts['os_family'] == "RedHat"
-
-- name: Target dhclient hook file for Debian family
-  set_fact:
-    dhclienthookfile: /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate
-  when: ansible_facts['os_family'] == "Debian"
-
 - name: Set etcd vars if using kubeadm mode
   set_fact:
     etcd_cert_dir: "{{ kube_cert_dir }}"

roles/kubernetes/preinstall/tasks/0040-verify-settings.yml

@@ -102,15 +102,6 @@
     msg: "Hostname must consist of lower case alphanumeric characters, '.' or '-', and must start and end with an alphanumeric character"
   when: not ignore_assert_errors
 
-- name: Stop if /etc/resolv.conf has no configured nameservers
-  assert:
-    that: configured_nameservers | length>0
-    fail_msg: "nameserver should not be empty in /etc/resolv.conf"
-  when:
-    - upstream_dns_servers | length == 0
-    - not disable_host_nameservers
-    - dns_mode in ['coredns', 'coredns_dual']
-
 - name: Stop if download_localhost is enabled for Flatcar Container Linux
   assert:
     that: ansible_facts['os_family'] != "Flatcar"

roles/kubernetes/preinstall/tasks/main.yml

@@ -3,47 +3,20 @@
 - name: Disable swap
   import_tasks: 0010-swapoff.yml
   when:
-    - not dns_late
     - kubelet_fail_swap_on
 
 - name: Set facts
   import_tasks: 0020-set_facts.yml
   tags:
-    - resolvconf
     - facts
 
 - name: Check settings
   import_tasks: 0040-verify-settings.yml
-  when:
-    - not dns_late
   tags:
     - asserts
 
 - name: Create directories
   import_tasks: 0050-create_directories.yml
-  when:
-    - not dns_late
-
-- name: Apply resolvconf settings
-  import_tasks: 0060-resolvconf.yml
-  when:
-    - dns_mode != 'none'
-    - resolvconf_mode == 'host_resolvconf'
-    - systemd_resolved_enabled.rc != 0
-    - networkmanager_enabled.rc != 0
-  tags:
-    - bootstrap_os
-    - resolvconf
-
-- name: Apply systemd-resolved settings
-  import_tasks: 0061-systemd-resolved.yml
-  when:
-    - dns_mode != 'none'
-    - resolvconf_mode == 'host_resolvconf'
-    - systemd_resolved_enabled.rc == 0
-  tags:
-    - bootstrap_os
-    - resolvconf
 
 - name: Apply networkmanager unmanaged devices settings
   import_tasks: 0062-networkmanager-unmanaged-devices.yml
@@ -52,58 +25,18 @@
   tags:
     - bootstrap_os
 
-- name: Apply networkmanager DNS settings
-  import_tasks: 0063-networkmanager-dns.yml
-  when:
-    - dns_mode != 'none'
-    - resolvconf_mode == 'host_resolvconf'
-    - networkmanager_enabled.rc == 0
-  tags:
-    - bootstrap_os
-    - resolvconf
-
 - name: Apply system configurations
   import_tasks: 0080-system-configurations.yml
-  when:
-    - not dns_late
   tags:
     - bootstrap_os
 
 - name: Configure NTP
   import_tasks: 0081-ntp-configurations.yml
   when:
-    - not dns_late
     - ntp_enabled
   tags:
     - bootstrap_os
 
-- name: Configure dhclient
-  import_tasks: 0100-dhclient-hooks.yml
-  when:
-    - dns_mode != 'none'
-    - resolvconf_mode == 'host_resolvconf'
-    - dhclientconffile is defined
-    - ansible_facts['os_family'] != "Flatcar"
-  tags:
-    - bootstrap_os
-    - resolvconf
-
-- name: Configure dhclient dhclient hooks
-  import_tasks: 0110-dhclient-hooks-undo.yml
-  when:
-    - dns_mode != 'none'
-    - resolvconf_mode != 'host_resolvconf'
-    - dhclientconffile is defined
-    - ansible_facts['os_family'] != "Flatcar"
-  tags:
-    - bootstrap_os
-    - resolvconf
-
-# We need to make sure the network is restarted early enough so that docker can later pick up the correct system
-# nameservers and search domains
-- name: Flush handlers
-  meta: flush_handlers
-
 - name: Check if we are running inside a Azure VM
   stat:
     path: /var/lib/waagent/
@@ -111,8 +44,6 @@
     get_checksum: false
     get_mime: false
   register: azure_check
-  when:
-    - not dns_late
   tags:
     - bootstrap_os