Merge branch 'master' into hostname-alias

roles/kubernetes/master/defaults/main.yml

@@ -10,3 +10,27 @@ kube_users_dir: "{{ kube_config_dir }}/users"
 # An experimental dev/test only dynamic volumes provisioner,
 # for PetSets. Works for kube>=v1.3 only.
 kube_hostpath_dynamic_provisioner: "false"
+
+# This is where you can drop yaml/json files and the kubelet will run those
+# pods on startup
+kube_manifest_dir: "{{ kube_config_dir }}/manifests"
+
+# This directory is where all the additional config stuff goes
+# the kubernetes normally puts in /srv/kubernets.
+# This puts them in a sane location.
+# Editting this value will almost surely break something. Don't
+# change it. Things like the systemd scripts are hard coded to
+# look in here. Don't do it.
+kube_config_dir: /etc/kubernetes
+
+# change to 0.0.0.0 to enable insecure access from anywhere (not recommended)
+kube_apiserver_insecure_bind_address: 127.0.0.1
+
+# Logging directory (sysvinit systems)
+kube_log_dir: "/var/log/kubernetes"
+
+# ETCD cert dir for connecting apiserver to etcd
+etcd_config_dir: /etc/ssl/etcd
+etcd_cert_dir: "{{ etcd_config_dir }}/ssl"
+
+

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -3,6 +3,8 @@ kind: Pod
 metadata:
   name: kube-apiserver
   namespace: kube-system
+  labels:
+    k8s-app: kube-apiserver
 spec:
   hostNetwork: true
   containers:
@@ -14,9 +16,12 @@ spec:
     - --advertise-address={{ ip | default(ansible_default_ipv4.address) }}
     - --etcd-servers={{ etcd_access_endpoint }}
     - --etcd-quorum-read=true
+    - --etcd-cafile={{ etcd_cert_dir }}/ca.pem
+    - --etcd-certfile={{ etcd_cert_dir }}/node.pem
+    - --etcd-keyfile={{ etcd_cert_dir }}/node-key.pem
     - --insecure-bind-address={{ kube_apiserver_insecure_bind_address }}
     - --apiserver-count={{ kube_apiserver_count }}
-    - --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,ServiceAccount,ResourceQuota
+    - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota
     - --service-cluster-ip-range={{ kube_service_addresses }}
     - --client-ca-file={{ kube_cert_dir }}/ca.pem
     - --basic-auth-file={{ kube_users_dir }}/known_users.csv
@@ -50,6 +55,9 @@ spec:
     - mountPath: /etc/ssl/certs
       name: ssl-certs-host
       readOnly: true
+    - mountPath: {{ etcd_cert_dir }}
+      name: etcd-certs
+      readOnly: true
     - mountPath: /var/log/
       name: logfile
   volumes:
@@ -59,6 +67,9 @@ spec:
   - hostPath:
       path: /etc/ssl/certs/
     name: ssl-certs-host
+  - hostPath:
+      path: {{ etcd_cert_dir }}
+    name: etcd-certs
   - hostPath:
       path: /var/log/
     name: logfile

roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2

@@ -3,6 +3,8 @@ kind: Pod
 metadata:
   name: kube-controller-manager
   namespace: kube-system
+  labels:
+    k8s-app: kube-controller
 spec:
   hostNetwork: true
   containers:

roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2

@@ -3,6 +3,8 @@ kind: Pod
 metadata:
   name: kube-scheduler
   namespace: kube-system
+  labels:
+    k8s-app: kube-scheduler
 spec:
   hostNetwork: true
   containers:

roles/kubernetes/node/defaults/main.yml

@@ -1,6 +1,13 @@
 # This is where all the cert scripts and certs will be located
 kube_cert_dir: "{{ kube_config_dir }}/ssl"
 
+# change to 0.0.0.0 to enable insecure access from anywhere (not recommended)
+kube_apiserver_insecure_bind_address: 127.0.0.1
+
+# This is where you can drop yaml/json files and the kubelet will run those
+# pods on startup
+kube_manifest_dir: "{{ kube_config_dir }}/manifests"
+
 dns_domain: "{{ cluster_name }}"
 
 # resolv.conf to base dns config
@@ -15,5 +22,16 @@ kube_proxy_masquerade_all: true
 #   - extensions/v1beta1/daemonsets=true
 #   - extensions/v1beta1/deployments=true
 
+# Logging directory (sysvinit systems)
+kube_log_dir: "/var/log/kubernetes"
+
+# This directory is where all the additional config stuff goes
+# the kubernetes normally puts in /srv/kubernets.
+# This puts them in a sane location.
+# Editting this value will almost surely break something. Don't
+# change it. Things like the systemd scripts are hard coded to
+# look in here. Don't do it.
+kube_config_dir: /etc/kubernetes
+
 nginx_image_repo: nginx
 nginx_image_tag: 1.11.4-alpine

roles/kubernetes/node/meta/main.yml

@@ -2,4 +2,6 @@
 dependencies:
   - role: download
     file: "{{ downloads.hyperkube }}"
+  - role: download
+    file: "{{ downloads.pod_infra }}"
   - role: kubernetes/secrets

roles/kubernetes/node/tasks/main.yml

@@ -11,6 +11,13 @@
     owner: kube
   when: kube_network_plugin == "calico"
 
+- name: Write Canal cni config
+  template:
+    src: "cni-canal.conf.j2"
+    dest: "/etc/cni/net.d/10-canal.conf"
+    owner: kube
+  when: kube_network_plugin == "canal"
+
 - name: Write kubelet config file
   template: src=kubelet.j2 dest={{ kube_config_dir }}/kubelet.env backup=yes
   notify:

roles/kubernetes/node/templates/cni-calico.conf.j2

@@ -1,6 +1,10 @@
 {
   "name": "calico-k8s-network",
   "type": "calico",
+  "etcd_endpoints": "{{ etcd_access_endpoint }}",
+  "etcd_cert_file": "{{ etcd_cert_dir }}/node.pem",
+  "etcd_key_file": "{{ etcd_cert_dir }}/node-key.pem",
+  "etcd_ca_cert_file": "{{ etcd_cert_dir }}/ca.pem",
   "log_level": "info",
   "ipam": {
     "type": "calico-ipam"

roles/kubernetes/node/templates/cni-canal.conf.j2

@@ -0,0 +1,15 @@
+{
+  "name": "canal-k8s-network",
+  "type": "flannel",
+  "delegate": {
+    "type": "calico",
+    "etcd_endpoints": "{{ etcd_access_endpoint }}",
+    "log_level": "info",
+    "policy": {
+      "type": "k8s"
+    },
+    "kubernetes": {
+      "kubeconfig": "{{ kube_config_dir }}/node-kubeconfig.yaml"
+    }
+  }
+}

roles/kubernetes/node/templates/kubelet.j2

@@ -20,13 +20,13 @@ KUBELET_REGISTER_NODE="--register-node=false"
 {% endif %}
 # location of the api-server
 {% if dns_setup|bool and skip_dnsmasq|bool %}
-KUBELET_ARGS="--cluster_dns={{ skydns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }} --resolv-conf={{ kube_resolv_conf }}"
+KUBELET_ARGS="--cluster_dns={{ skydns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }} --resolv-conf={{ kube_resolv_conf }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"
 {% elif dns_setup|bool %}
-KUBELET_ARGS="--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }} --resolv-conf={{ kube_resolv_conf }}"
+KUBELET_ARGS="--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }} --resolv-conf={{ kube_resolv_conf }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"
 {% else %}
-KUBELET_ARGS="--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}"
+KUBELET_ARGS="--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"
 {% endif %}
-{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "weave"] %}
+{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "weave", "canal"] %}
 KUBELET_NETWORK_PLUGIN="--network-plugin=cni --network-plugin-dir=/etc/cni/net.d"
 {% elif kube_network_plugin is defined and kube_network_plugin == "weave" %}
 DOCKER_SOCKET="--docker-endpoint=unix:/var/run/weave/weave.sock"

roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2

@@ -3,6 +3,8 @@ kind: Pod
 metadata:
   name: kube-proxy
   namespace: kube-system
+  labels:
+    k8s-app: kube-proxy
 spec:
   hostNetwork: true
   containers:

roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2

@@ -3,6 +3,8 @@ kind: Pod
 metadata:
   name: nginx-proxy
   namespace: kube-system
+  labels:
+    k8s-app: kube-nginx
 spec:
   hostNetwork: true
   containers:

roles/kubernetes/preinstall/defaults/main.yml

@@ -21,8 +21,7 @@ kube_log_dir: "/var/log/kubernetes"
 # pods on startup
 kube_manifest_dir: "{{ kube_config_dir }}/manifests"
 
-# change to 0.0.0.0 to enable insecure access from anywhere (not recommended)
-kube_apiserver_insecure_bind_address: 127.0.0.1
+epel_rpm_download_url: "https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm"
 
 common_required_pkgs:
   - python-httplib2
@@ -46,3 +45,6 @@ openstack_username: "{{ lookup('env','OS_USERNAME')  }}"
 openstack_password: "{{ lookup('env','OS_PASSWORD')  }}"
 openstack_region: "{{ lookup('env','OS_REGION_NAME')  }}"
 openstack_tenant_id: "{{ lookup('env','OS_TENANT_ID')  }}"
+
+# All clients access each node individually, instead of using a load balancer.
+etcd_multiaccess: true

roles/kubernetes/preinstall/tasks/main.yml

@@ -74,7 +74,7 @@
   with_items:
     - "/etc/cni/net.d"
     - "/opt/cni/bin"
-  when: kube_network_plugin in ["calico", "weave"] and "{{ inventory_hostname in groups['k8s-cluster'] }}"
+  when: kube_network_plugin in ["calico", "weave", "canal"] and "{{ inventory_hostname in groups['k8s-cluster'] }}"
 
 - name: Update package management cache (YUM)
   yum: update_cache=yes name='*'
@@ -91,7 +91,7 @@
   changed_when: False
 
 - name: Install epel-release on RedHat/CentOS
-  shell: rpm -qa | grep epel-release || rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+  shell: rpm -qa | grep epel-release || rpm -ivh {{ epel_rpm_download_url }}
   when: ansible_distribution in ["CentOS","RedHat"] and
         ansible_distribution_major_version >= 7
   changed_when: False

roles/kubernetes/preinstall/tasks/set_facts.yml

@@ -23,14 +23,14 @@
 
 - set_fact: etcd_address="{{ ip | default(ansible_default_ipv4['address']) }}"
 - set_fact: etcd_access_address="{{ access_ip | default(etcd_address) }}"
-- set_fact: etcd_peer_url="http://{{ etcd_access_address }}:2380"
-- set_fact: etcd_client_url="http://{{ etcd_access_address }}:2379"
+- set_fact: etcd_peer_url="https://{{ etcd_access_address }}:2380"
+- set_fact: etcd_client_url="https://{{ etcd_access_address }}:2379"
 - set_fact: etcd_authority="127.0.0.1:2379"
-- set_fact: etcd_endpoint="http://{{ etcd_authority }}"
+- set_fact: etcd_endpoint="https://{{ etcd_authority }}"
 - set_fact:
     etcd_access_addresses: |-
       {% for item in groups['etcd'] -%}
-        http://{{ hostvars[item].etcd_access_address }}:2379{% if not loop.last %},{% endif %}
+        https://{{ item }}:2379{% if not loop.last %},{% endif %}
       {%- endfor %}
 - set_fact: etcd_access_endpoint="{% if etcd_multiaccess %}{{ etcd_access_addresses }}{% else %}{{ etcd_endpoint }}{% endif %}"
 - set_fact:
@@ -39,12 +39,10 @@
       {%   if inventory_hostname == host %}{{"etcd"+loop.index|string }}{% endif %}
       {% endfor %}
 - set_fact:
-    etcd_proxy_member_name: |-
-      {% for host in groups['k8s-cluster'] %}
-      {%   if inventory_hostname == host %}{{"etcd-proxy"+loop.index|string }}{% endif %}
-      {% endfor %}
-- set_fact:
-    is_etcd_proxy: "{{ inventory_hostname in groups['k8s-cluster'] }}"
+    etcd_peer_addresses: |-
+      {% for item in groups['etcd'] -%}
+        {{ "etcd"+loop.index|string }}=https://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2380{% if not loop.last %},{% endif %}
+      {%- endfor %}
 - set_fact:
     is_etcd_master: "{{ inventory_hostname in groups['etcd'] }}"
 - set_fact:

roles/kubernetes/secrets/defaults/main.yml

@@ -6,3 +6,16 @@ kube_token_dir: "{{ kube_config_dir }}/tokens"
 
 # This is where to save basic auth file
 kube_users_dir: "{{ kube_config_dir }}/users"
+
+# This directory is where all the additional config stuff goes
+# the kubernetes normally puts in /srv/kubernets.
+# This puts them in a sane location.
+# Editting this value will almost surely break something. Don't
+# change it. Things like the systemd scripts are hard coded to
+# look in here. Don't do it.
+kube_config_dir: /etc/kubernetes
+
+# This directory is where all the additional scripts go
+# that Kubernetes normally puts in /srv/kubernetes.
+# This puts them in a sane location
+kube_script_dir: "{{ bin_dir }}/kubernetes-scripts"

roles/kubernetes/secrets/tasks/check-tokens.yml

@@ -27,7 +27,7 @@
     sync_tokens: true
   when: >-
       {%- set tokens = {'sync': False} -%}
-      {%- for server in groups['kube-master']
+      {%- for server in groups['kube-master'] | intersect(play_hosts)
          if (not hostvars[server].known_tokens.stat.exists) or
          (hostvars[server].known_tokens.stat.checksum != known_tokens_master.stat.checksum|default('')) -%}
          {%- set _ = tokens.update({'sync': True}) -%}

roles/kubernetes/secrets/tasks/gen_certs.yml

@@ -27,31 +27,30 @@
     master_certs: ['ca-key.pem', 'admin.pem', 'admin-key.pem', 'apiserver-key.pem', 'apiserver.pem']
     node_certs: ['ca.pem', 'node.pem', 'node-key.pem']
 
-- name: Gen_certs | Get the certs from first master
-  slurp:
-    src: "{{ kube_cert_dir }}/{{ item }}"
+- name: Gen_certs | Gather master certs
+  shell: "tar cfz - -C {{ kube_cert_dir }} {{ master_certs|join(' ') }} {{ node_certs|join(' ') }} | base64 --wrap=0"
+  register: master_cert_data
   delegate_to: "{{groups['kube-master'][0]}}"
-  register: slurp_certs
-  with_items: '{{ master_certs + node_certs }}'
-  when: sync_certs|default(false)
   run_once: true
-  notify: set secret_changed
+  when: sync_certs|default(false)
+
+- name: Gen_certs | Gather node certs
+  shell: "tar cfz - -C {{ kube_cert_dir }} {{ node_certs|join(' ') }} | base64 --wrap=0"
+  register: node_cert_data
+  delegate_to: "{{groups['kube-master'][0]}}"
+  run_once: true
+  when: sync_certs|default(false)
 
 - name: Gen_certs | Copy certs on masters
-  copy:
-    content: "{{ item.content|b64decode }}"
-    dest: "{{ item.source }}"
-  with_items: '{{slurp_certs.results}}'
+  shell: "echo '{{master_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ kube_cert_dir }}"
+  changed_when: false
   when: inventory_hostname in groups['kube-master'] and sync_certs|default(false) and
         inventory_hostname != groups['kube-master'][0]
 
 - name: Gen_certs | Copy certs on nodes
-  copy:
-    content: "{{ item.content|b64decode }}"
-    dest: "{{ item.source }}"
-  with_items: '{{slurp_certs.results}}'
-  when: item.item in node_certs and
-        inventory_hostname in groups['kube-node'] and sync_certs|default(false) and
+  shell: "echo '{{node_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ kube_cert_dir }}"
+  changed_when: false
+  when: inventory_hostname in groups['kube-node'] and  sync_certs|default(false) and
         inventory_hostname != groups['kube-master'][0]
 
 - name: Gen_certs | check certificate permissions
@@ -66,21 +65,21 @@
   when: inventory_hostname in groups['kube-master']
   changed_when: false
 
-- name: Gen_certs | target ca-certificates directory
+- name: Gen_certs | target ca-certificates path
   set_fact:
-    ca_cert_dir: |-
+    ca_cert_path: |-
       {% if ansible_os_family == "Debian" -%}
-      /usr/local/share/ca-certificates
+      /usr/local/share/ca-certificates/kube-ca.crt
       {%- elif ansible_os_family == "RedHat" -%}
-      /etc/pki/ca-trust/source/anchors
+      /etc/pki/ca-trust/source/anchors/kube-ca.crt
       {%- elif ansible_os_family == "CoreOS" -%}
-      /etc/ssl/certs
+      /etc/ssl/certs/kube-ca.pem
       {%- endif %}
 
 - name: Gen_certs | add CA to trusted CA dir
   copy:
     src: "{{ kube_cert_dir }}/ca.pem"
-    dest: "{{ ca_cert_dir }}/kube-ca.crt"
+    dest: "{{ ca_cert_path }}"
     remote_src: true
   register: kube_ca_cert
 
@@ -88,7 +87,7 @@
   command: update-ca-certificates
   when: kube_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS"]
 
-- name: Gen_certs | update ca-certificatesa (RedHat)
+- name: Gen_certs | update ca-certificates (RedHat)
   command: update-ca-trust extract
   when: kube_ca_cert.changed and ansible_os_family == "RedHat"
 

roles/kubernetes/secrets/tasks/gen_tokens.yml

@@ -43,20 +43,15 @@
   delegate_to: "{{groups['kube-master'][0]}}"
   when: sync_tokens|default(false)
 
-- name: Gen_tokens | Get the tokens from first master
-  slurp:
-    src: "{{ item }}"
-  register: slurp_tokens
-  with_items: '{{tokens_list.stdout_lines}}'
-  run_once: true
+- name: Gen_tokens | Gather tokens
+  shell: "tar cfz - {{ tokens_list.stdout_lines | join(' ') }} | base64 --wrap=0"
+  register: tokens_data
   delegate_to: "{{groups['kube-master'][0]}}"
+  run_once: true
   when: sync_tokens|default(false)
-  notify: set secret_changed
 
 - name: Gen_tokens | Copy tokens on masters
-  copy:
-    content: "{{ item.content|b64decode }}"
-    dest: "{{ item.source }}"
-  with_items: '{{slurp_tokens.results}}'
+  shell: "echo '{{ tokens_data.stdout|quote }}' | base64 -d | tar xz -C /"
+  changed_when: false
   when: inventory_hostname in groups['kube-master'] and sync_tokens|default(false) and
         inventory_hostname != groups['kube-master'][0]