Merge pull request #799 from kubernetes-incubator/docker_dns

Implement "dockerd --dns-xxx" based dns mode

roles/dnsmasq/defaults/main.yml

@@ -18,9 +18,6 @@ dnsmasq_version: 2.72
 dnsmasq_image_repo: "andyshinn/dnsmasq"
 dnsmasq_image_tag: "{{ dnsmasq_version }}"
 
-# Skip dnsmasq setup
-skip_dnsmasq: false
-
 # Limits for dnsmasq/kubedns apps
 dns_cpu_limit: 100m
 dns_memory_limit: 170Mi

roles/dnsmasq/meta/main.yml

@@ -2,5 +2,5 @@
 dependencies:
   - role: download
     file: "{{ downloads.dnsmasq }}"
-    when: not skip_dnsmasq|default(false) and download_localhost|default(false)
+    when: dns_mode == 'dnsmasq_kubedns' and download_localhost|default(false)
     tags: [download, dnsmasq]

roles/dnsmasq/templates/01-kube-dns.conf.j2

@@ -15,15 +15,17 @@ local=/{{ bogus_domains }}
 {% for srv in upstream_dns_servers %}
 server={{ srv }}
 {% endfor %}
-{% else %}
+no-resolv
+{% elif resolvconf_mode == 'host_resolvconf' %}
+{# The default resolver is only needed when the hosts resolv.conf was modified by us. If it was not modified, we can rely on dnsmasq to reuse the systems resolv.conf #}
 server={{ default_resolver }}
+no-resolv
 {% endif %}
 
 {% if kube_log_level == '4' %}
 log-queries
 {% endif %}
 bogus-priv
-no-resolv
 no-negcache
 cache-size=1000
 max-cache-ttl=10

roles/docker/tasks/main.yml

@@ -14,6 +14,10 @@
       skip: true
   tags: facts
 
+- include: set_facts_dns.yml
+  when: dns_mode != 'none' and resolvconf_mode == 'docker_dns'
+  tags: facts
+
 - name: check for minimum kernel version
   fail:
     msg: >
@@ -63,6 +67,13 @@
   with_items: "{{ docker_package_info.pkgs }}"
   when: (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"]) and (docker_package_info.pkgs|length > 0)
 
+- name: check minimum docker version for docker_dns mode. You need at least docker version >= 1.12 for resolvconf_mode=docker_dns
+  shell: docker version -f "{{ '{{' }}.Client.Version{{ '}}' }}"
+  register: docker_version
+  failed_when: docker_version.stdout|version_compare('1.12', '<')
+  changed_when: false
+  when: dns_mode != 'none' and resolvconf_mode == 'docker_dns'
+
 - name: Set docker systemd config
   include: systemd.yml
 

roles/docker/tasks/set_facts_dns.yml

@@ -0,0 +1,61 @@
+---
+
+- name: set dns server for docker
+  set_fact:
+    docker_dns_servers: |-
+      {%- if dns_mode == 'kubedns' -%}
+        {{ [ skydns_server ] }}
+      {%- elif dns_mode == 'dnsmasq_kubedns' -%}
+        {{ [ dns_server ] }}
+      {%- endif -%}
+
+- name: set base docker dns facts
+  set_fact:
+    docker_dns_search_domains:
+      - 'default.svc.{{ dns_domain }}'
+      - 'svc.{{ dns_domain }}'
+    docker_dns_options:
+      - ndots:{{ ndots }}
+      - timeout:2
+      - attempts:2
+
+- name: add upstream dns servers (only when dnsmasq is not used)
+  set_fact:
+    docker_dns_servers: "{{ docker_dns_servers + upstream_dns_servers|default([]) }}"
+  when: dns_mode == 'kubedns'
+
+- name: add global searchdomains
+  set_fact:
+    docker_dns_search_domains: "{{ docker_dns_search_domains + searchdomains|default([]) }}"
+
+- name: check system nameservers
+  shell: grep "^nameserver" /etc/resolv.conf | sed 's/^nameserver\s*//'
+  changed_when: False
+  register: system_nameservers
+
+- name: check system search domains
+  shell: grep "^search" /etc/resolv.conf | sed 's/^search\s*//'
+  changed_when: False
+  register: system_search_domains
+
+- name: add system nameservers to docker options
+  set_fact:
+    docker_dns_servers: "{{ docker_dns_servers + [item] }}"
+  with_items: "{{ system_nameservers.stdout_lines|default([]) }}"
+
+- name: add system search domains to docker options
+  set_fact:
+    docker_dns_search_domains: "{{ docker_dns_search_domains + [item] }}"
+  with_items: "{{ system_search_domains.stdout.split(' ') }}"
+
+- name: check number of nameservers
+  fail: msg="Too many nameservers"
+  when: docker_dns_servers|length > 3
+
+- name: check number of search domains
+  fail: msg="Too many search domains"
+  when: docker_dns_search_domains|length > 6
+
+- name: check length of search domains
+  fail: msg="Search domains exceeded limit of 256 characters"
+  when: docker_dns_search_domains|join(' ')|length > 256

roles/docker/tasks/systemd.yml

@@ -21,4 +21,11 @@
     dest: "/etc/systemd/system/docker.service.d/docker-options.conf"
   notify: restart docker
 
+- name: Write docker dns systemd drop-in
+  template:
+    src: docker-dns.conf.j2
+    dest: "/etc/systemd/system/docker.service.d/docker-dns.conf"
+  notify: restart docker
+  when: dns_mode != 'none' and resolvconf_mode == 'docker_dns'
+
 - meta: flush_handlers

roles/docker/templates/docker-dns.conf.j2

@@ -0,0 +1,6 @@
+[Service]
+Environment="DOCKER_DNS_OPTIONS=\
+    {% for d in docker_dns_servers %}--dns {{ d }} {% endfor %} \
+    {% for d in docker_dns_search_domains %}--dns-search {{ d }} {% endfor %} \
+    {% for o in docker_dns_options %}--dns-opt {{ o }} {% endfor %} \
+"

roles/docker/templates/docker.service.j2

@@ -22,6 +22,7 @@ ExecStart={{ docker_bin_dir }}/docker daemon \
           $DOCKER_OPTS \
           $DOCKER_STORAGE_OPTIONS \
           $DOCKER_NETWORK_OPTIONS \
+          $DOCKER_DNS_OPTIONS \
           $INSECURE_REGISTRY
 TasksMax=infinity
 LimitNOFILE=1048576

roles/kubernetes-apps/ansible/tasks/main.yaml

@@ -12,7 +12,7 @@
     - {file: kubedns-rc.yml, type: rc}
     - {file: kubedns-svc.yml, type: svc}
   register: manifests
-  when: inventory_hostname == groups['kube-master'][0]
+  when: dns_mode != 'none' and inventory_hostname == groups['kube-master'][0]
   tags: dnsmasq
 
 - name: Kubernetes Apps | Start Resources
@@ -24,7 +24,7 @@
     filename: "{{kube_config_dir}}/{{item.item.file}}"
     state: "{{item.changed | ternary('latest','present') }}"
   with_items: "{{ manifests.results }}"
-  when: inventory_hostname == groups['kube-master'][0]
+  when: dns_mode != 'none' and inventory_hostname == groups['kube-master'][0]
   tags: dnsmasq
 
 - include: tasks/calico-policy-controller.yml

roles/kubernetes/node/templates/kubelet.j2

@@ -12,9 +12,9 @@ KUBELET_HOSTNAME="--hostname-override={{ ansible_hostname }}"
 {% set kubelet_args_base %}--pod-manifest-path={{ kube_manifest_dir }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}{% endset %}
 
 {# DNS settings for kubelet #}
-{% if dns_setup|bool and skip_dnsmasq|bool %}
+{% if dns_mode == 'kubedns' %}
 {% set kubelet_args_cluster_dns %}--cluster_dns={{ skydns_server }}{% endset %}
-{% elif dns_setup|bool %}
+{% elif dns_mode == 'dnsmasq_kubedns' %}
 {% set kubelet_args_cluster_dns %}--cluster_dns={{ dns_server }}{% endset %}
 {% else %}
 {% set kubelet_args_cluster_dns %}{% endset %}

roles/kubernetes/preinstall/tasks/main.yml

@@ -172,6 +172,7 @@
   tags: [bootstrap-os, etchosts]
 
 - include: resolvconf.yml
+  when: dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'
   tags: [bootstrap-os, resolvconf]
 
 - name: Check if we are running inside a Azure VM

roles/kubernetes/preinstall/tasks/set_resolv_facts.yml

@@ -67,7 +67,7 @@
 - name: pick dnsmasq cluster IP or default resolver
   set_fact:
     dnsmasq_server: |-
-      {%- if skip_dnsmasq|bool and not dns_early|bool -%}
+      {%- if dns_mode == 'kubedns' and not dns_early|bool -%}
         {{ [ skydns_server ] + upstream_dns_servers|default([]) }}
       {%- elif dns_early|bool -%}
         {{ upstream_dns_servers|default([]) }}

roles/reset/tasks/main.yml

@@ -1,7 +1,7 @@
 ---
 
 - name: reset | stop services
-  service: name={{item}} state=stopped
+  service: name={{ item }} state=stopped
   with_items:
     - kubelet
     - etcd
@@ -16,13 +16,26 @@
     - etcd
   register: services_removed
 
+- name: reset | remove docker dropins
+  file:
+    path: "/etc/systemd/system/docker.service.d/{{ item }}"
+    state: absent
+  with_items:
+    - docker-dns.conf
+    - docker-options.conf
+  register: docker_dropins_removed
+
 - name: reset | systemctl daemon-reload
   command: systemctl daemon-reload
-  when: services_removed.changed
+  when: services_removed.changed or docker_dropins_removed.changed
 
 - name: reset | remove all containers
   shell: "{{ docker_bin_dir }}/docker ps -aq | xargs -r docker rm -fv"
 
+- name: reset | restart docker if needed
+  service: name=docker state=restarted
+  when: docker_dropins_removed.changed
+
 - name: reset | gather mounted kubelet dirs
   shell: mount | grep /var/lib/kubelet | awk '{print $3}' | tac
   register: mounted_dirs
@@ -42,6 +55,40 @@
     - /etc/cni
     - /etc/nginx
     - /etc/dnsmasq.d
+    - /etc/dnsmasq.conf
+    - /etc/dnsmasq.d-available
     - /etc/etcd.env
     - /etc/calico
     - /opt/cni
+    - /etc/dhcp/dhclient.d/zdnsupdate.sh
+    - /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate
+    - "{{ bin_dir }}/kubelet"
+
+- name: reset | remove dns settings from dhclient.conf
+  blockinfile:
+    dest: "{{ item }}"
+    state: absent
+    follow: yes
+    marker: "# Ansible entries {mark}"
+  failed_when: false
+  with_items:
+    - /etc/dhclient.conf
+    - /etc/dhcp/dhclient.conf
+
+- name: reset | remove host entries from /etc/hosts
+  blockinfile:
+    dest: "/etc/hosts"
+    state: absent
+    follow: yes
+    marker: "# Ansible inventory hosts {mark}"
+
+- name: reset | Restart network
+  service:
+    name: >-
+      {% if ansible_os_family == "RedHat" -%}
+      network
+      {%- elif ansible_os_family == "Debian" -%}
+      networking
+      {%- endif %}
+    state: restarted
+  when: ansible_os_family != "CoreOS"