update README, local connection

Fix calico with systemd

.travis.yml

@@ -0,0 +1,41 @@
+sudo: required
+dist: trusty
+language: python
+python: "2.7"
+
+addons:
+  hosts:
+    - node1
+
+env:
+  - SITE=cluster.yml
+
+before_install:
+  - sudo apt-get update -qq
+
+install:
+  # Install Ansible.
+  - sudo -H pip install ansible
+  - sudo -H pip install netaddr
+
+cache:
+  directories:
+    - $HOME/releases
+    - $HOME/.cache/pip
+
+before_script:
+  - export PATH=$PATH:/usr/local/bin
+
+script:
+  # Check the role/playbook's syntax.
+  - "sudo -H ansible-playbook -i inventory/local-tests.cfg $SITE --syntax-check"
+
+  # Run the role/playbook with ansible-playbook.
+  - "sudo -H ansible-playbook -i inventory/local-tests.cfg $SITE --connection=local"
+
+  # Run the role/playbook again, checking to make sure it's idempotent.
+  - >
+    sudo -H ansible-playbook -i inventory/local-tests.cfg $SITE --connection=local
+    | tee /dev/stderr | grep -q 'changed=0.*failed=0'
+    && (echo 'Idempotence test: pass' && exit 0)
+    || (echo 'Idempotence test: fail' && exit 1)

README.md

@@ -1,14 +1,15 @@
+[![Build Status](https://travis-ci.org/ansibl8s/setup-kubernetes.svg)](https://travis-ci.org/ansibl8s/setup-kubernetes)
 kubernetes-ansible
 ========
 
-Install and configure a kubernetes cluster including network plugin.
+Install and configure a Multi-Master/HA kubernetes cluster including network plugin.
 
 ### Requirements
-Tested on **Debian Jessie** and **Ubuntu** (14.10, 15.04, 15.10).
+Tested on **Debian Wheezy/Jessie** and **Ubuntu** (14.10, 15.04, 15.10).
+Should work on **RedHat/Fedora/Centos** platforms (to be tested)
 * The target servers must have access to the Internet in order to pull docker imaqes.
 * The firewalls are not managed, you'll need to implement your own rules the way you used to.
-
+* Ansible v1.9.x and python-netaddr
-Ansible v1.9.x
 
 ### Components
 * [kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.1.3
@@ -25,7 +26,7 @@ These defaults are good for tests purposes.
 Edit the inventory according to the number of servers
 ```
 [downloader]
-10.115.99.1
+localhost ansible_connection=local ansible_python_interpreter=python2
 
 [kube-master]
 10.115.99.31
@@ -54,14 +55,6 @@ You can jump directly to "*Available apps, installation procedure*"
 
 Ansible
 -------------------------
-### Download binaries
-A role allows to download required binaries. They will be stored in a directory defined by the variable
-**'local_release_dir'** (by default /tmp).
-Please ensure that you have enough disk space there (about **300M**).
-
-**Note**: Whenever you'll need to change the version of a software, you'll have to erase the content of this directory.
-
-
 ### Variables
 The main variables to change are located in the directory ```inventory/group_vars/all.yml```.
 
@@ -74,7 +67,7 @@ In node-mesh mode the nodes peers with all the nodes in order to exchange routes
 ```
 
 [downloader]
-node1 ansible_ssh_host=10.99.0.26
+localhost ansible_connection=local ansible_python_interpreter=python2
 
 [kube-master]
 node1 ansible_ssh_host=10.99.0.26
@@ -117,8 +110,10 @@ kube-master
 
 - hosts: k8s-cluster
   roles:
-    - { role: etcd, tags: etcd }
+    - { role: kubernetes/preinstall, tags: preinstall }
     - { role: docker, tags: docker }
+    - { role: kubernetes/node, tags: node }
+    - { role: etcd, tags: etcd }
     - { role: dnsmasq, tags: dnsmasq }
     - { role: network_plugin, tags: ['calico', 'flannel', 'network'] }
 
@@ -126,10 +121,6 @@ kube-master
   roles:
     - { role: kubernetes/master, tags: master }
 
-- hosts: kube-node
-  roles:
-    - { role: kubernetes/node, tags: node }
-
 ```
 
 ### Run
@@ -258,7 +249,7 @@ Finally update the playbook ```apps.yml``` with the chosen roles, and run it
 ```
 
 ```
-ansible-playbook -i environments/dev/inventory apps.yml -u root
+ansible-playbook -i inventory/inventory.cfg apps.yml -u root
 ```
 
 

cluster.yml

@@ -6,15 +6,13 @@
 
 - hosts: k8s-cluster
   roles:
-    - { role: etcd, tags: etcd }
+    - { role: kubernetes/preinstall, tags: preinstall }
     - { role: docker, tags: docker }
+    - { role: kubernetes/node, tags: node }
+    - { role: etcd, tags: etcd }
     - { role: dnsmasq, tags: dnsmasq }
     - { role: network_plugin, tags: ['calico', 'flannel', 'network'] }
 
 - hosts: kube-master
   roles:
     - { role: kubernetes/master, tags: master }
-
-- hosts: kube-node
-  roles:
-    - { role: kubernetes/node, tags: node }

inventory/group_vars/all.yml

@@ -68,7 +68,7 @@ dns_setup: true
 dns_domain: "{{ cluster_name }}"
 #
 # # Ip address of the kubernetes dns service
-dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(253)|ipaddr('address') }}"
+dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}"
 
 # For multi masters architecture:
 # kube-proxy doesn't support multiple apiservers for the time being so you'll need to configure your own loadbalancer

inventory/inventory.example

@@ -1,5 +1,5 @@
 [downloader]
-node1 ansible_ssh_host=10.99.0.26
+localhost ansible_connection=local ansible_python_interpreter=python2
 
 [kube-master]
 node1 ansible_ssh_host=10.99.0.26

inventory/local-tests.cfg

@@ -0,0 +1,17 @@
+node1 ansible_connection=local local_release_dir={{ansible_env.HOME}}/releases
+
+[downloader]
+node1
+
+[kube-master]
+node1
+
+[etcd]
+node1
+
+[kube-node]
+node1
+
+[k8s-cluster:children]
+kube-node
+kube-master

roles/dnsmasq/handlers/main.yml

@@ -1,3 +0,0 @@
----
-- name: restart dnsmasq
-  command: systemctl restart dnsmasq

roles/dnsmasq/tasks/main.yml

@@ -5,6 +5,7 @@
     regexp: "^{{ hostvars[item].ansible_default_ipv4.address }} {{ item }}$"
     line: "{{ hostvars[item].ansible_default_ipv4.address }} {{ item }}"
     state: present
+    backup: yes
   when: hostvars[item].ansible_default_ipv4.address is defined
   with_items: groups['all']
 
@@ -14,6 +15,7 @@
     regexp: ".*{{ apiserver_loadbalancer_domain_name }}$"
     line: "{{ loadbalancer_apiserver.address }} lb-apiserver.kubernetes.local"
     state: present
+    backup: yes
   when: loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined
 
 - name: clean hosts file
@@ -21,20 +23,11 @@
     dest: /etc/hosts
     regexp: "{{ item }}"
     state: absent
+    backup: yes
   with_items:
     - '^127\.0\.0\.1(\s+){{ inventory_hostname }}.*'
     - '^::1(\s+){{ inventory_hostname }}.*'
 
-- name: install dnsmasq and bindr9utils
-  apt:
-    name: "{{ item }}"
-    state: present
-    update_cache: yes
-  with_items:
-    - dnsmasq
-    - bind9utils
-  when: inventory_hostname in groups['kube-master']
-
 - name: ensure dnsmasq.d directory exists
   file:
     path: /etc/dnsmasq.d
@@ -46,24 +39,63 @@
     src: 01-kube-dns.conf.j2
     dest: /etc/dnsmasq.d/01-kube-dns.conf
     mode: 755
-  notify:
+    backup: yes
-    - restart dnsmasq
   when: inventory_hostname in groups['kube-master']
 
-- name: enable dnsmasq
+- name: create dnsmasq pod template
-  service:
+  template: src=dnsmasq-pod.yml dest=/etc/kubernetes/manifests/dnsmasq-pod.manifest
-    name: dnsmasq
-    state: started
-    enabled: yes
   when: inventory_hostname in groups['kube-master']
 
-- name: update resolv.conf with new DNS setup
+- name: Check for dnsmasq port
-  template:
+  wait_for:
-    src: resolv.conf.j2
+    port: 53
-    dest: /etc/resolv.conf
+    delay: 5
-    mode: 644
+    timeout: 100
+  when: inventory_hostname in groups['kube-master']
+
+- name: check resolvconf
+  stat: path=/etc/resolvconf/resolv.conf.d/head
+  register: resolvconf
+
+- name: target resolv.conf file
+  set_fact:
+    resolvconffile: >
+      {%- if resolvconf.stat.exists == True -%}
+      /etc/resolvconf/resolv.conf.d/head
+      {%- else -%}
+      /etc/resolv.conf
+      {%- endif -%}
+
+- name: Add search resolv.conf
+  lineinfile:
+    line: search {{ [ 'default.svc.' + dns_domain, 'svc.' + dns_domain, dns_domain ] | join(' ') }}
+    dest: "{{resolvconffile}}"
+    state: present
+    insertafter: EOF
+    backup: yes
+    follow: yes
+
+- name: Add all masters as nameserver
+  lineinfile:
+    line: nameserver {{ hostvars[item]['ansible_default_ipv4']['address'] }}
+    dest: "{{resolvconffile}}"
+    state: present
+    insertafter: EOF
+    backup: yes
+    follow: yes
+  with_items: groups['kube-master']
 
 - name: disable resolv.conf modification by dhclient
-  copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/nodnsupdate mode=u+x
+  copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/nodnsupdate mode=u+x backup=yes
+  when: ansible_os_family == "Debian"
+
+- name: disable resolv.conf modification by dhclient
+  copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x backup=yes
+  when: ansible_os_family == "RedHat"
+
+- name: update resolvconf
+  command: resolvconf -u
+  changed_when: False
+  when: resolvconf.stat.exists == True
 
 - meta: flush_handlers

roles/dnsmasq/templates/dnsmasq-pod.yml

@@ -0,0 +1,49 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: dnsmasq
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+    - name: dnsmasq
+      image: andyshinn/dnsmasq:2.72
+      command:
+        - dnsmasq
+      args:
+        - -k
+        - "-7"
+        - /etc/dnsmasq.d
+        - --local-service
+      securityContext:
+        capabilities:
+          add:
+            - NET_ADMIN
+      imagePullPolicy: Always
+      resources:
+        limits:
+          cpu: 100m
+          memory: 256M
+      ports:
+        - name: dns
+          containerPort: 53
+          hostPort: 53
+          protocol: UDP
+        - name: dns-tcp
+          containerPort: 53
+          hostPort: 53
+          protocol: TCP
+      volumeMounts:
+        - name: etcdnsmasqd
+          mountPath: /etc/dnsmasq.d
+        - name: etcdnsmasqdavailable
+          mountPath: /etc/dnsmasq.d-available
+
+  volumes:
+    - name: etcdnsmasqd
+      hostPath:
+        path: /etc/dnsmasq.d
+    - name: etcdnsmasqdavailable
+      hostPath:
+        path: /etc/dnsmasq.d-available

roles/dnsmasq/templates/resolv.conf.j2

@@ -1,9 +0,0 @@
-; generated by ansible
-search {{ [ 'default.svc.' + dns_domain, 'svc.' + dns_domain, dns_domain ] | join(' ') }}
-{% if inventory_hostname in groups['kube-master'] %}
-nameserver {{ ansible_default_ipv4.address }}
-{% else %}
-{% for host in groups['kube-master'] %}
-nameserver {{ hostvars[host]['ansible_default_ipv4']['address'] }}
-{% endfor %}
-{% endif %}

roles/docker/.gitignore

@@ -0,0 +1,2 @@
+.*.swp
+.vagrant

roles/docker/files/systemd-docker.service

@@ -1,17 +0,0 @@
-[Unit]
-Description=Docker Application Container Engine
-Documentation=https://docs.docker.com
-After=network.target docker.socket
-Requires=docker.socket
-
-[Service]
-EnvironmentFile=-/etc/default/docker
-Type=notify
-ExecStart=/usr/bin/docker daemon -H fd:// $DOCKER_OPTS
-MountFlags=slave
-LimitNOFILE=1048576
-LimitNPROC=1048576
-LimitCORE=infinity
-
-[Install]
-WantedBy=multi-user.target

roles/docker/handlers/main.yml

@@ -1,12 +0,0 @@
----
-- name: restart docker
-  command: /bin/true
-  notify:
-    - reload systemd
-    - restart docker service
-
-- name: reload systemd
-  shell: systemctl daemon-reload
-
-- name: restart docker service
-  service: name=docker state=restarted

roles/docker/tasks/configure.yml

@@ -1,16 +0,0 @@
----
-- name: enable docker
-  service:
-    name: docker
-    enabled: yes
-    state: started
-  tags:
-    - docker
-
-#- name: login to arkena's docker registry
-#  shell : >
-#    docker login --username={{ dockerhub_user }}
-#    --password={{ dockerhub_pass }}
-#    --email={{ dockerhub_email }}
-
-- meta: flush_handlers

roles/docker/tasks/install.yml

@@ -1,24 +0,0 @@
----
-- name: Install prerequisites for https transport
-  apt: pkg={{ item }} state=present update_cache=yes
-  with_items:
-    - apt-transport-https
-    - ca-certificates
-
-- name: Configure docker apt repository
-  template: src=docker.list.j2 dest=/etc/apt/sources.list.d/docker.list backup=yes
-
-- name: Install docker-engine
-  apt: pkg={{ item }} state=present force=yes update_cache=yes
-  with_items:
-    - aufs-tools
-    - cgroupfs-mount
-    - docker-engine=1.9.1-0~{{ ansible_distribution_release }}
-
-- name: Copy default docker configuration
-  template: src=default-docker.j2 dest=/etc/default/docker backup=yes
-  notify: restart docker
-
-- name: Copy Docker systemd unit file
-  copy: src=systemd-docker.service dest=/lib/systemd/system/docker.service backup=yes
-  notify: restart docker

roles/docker/tasks/main.yml

@@ -1,3 +1,53 @@
 ---
-- include: install.yml
+- name: gather os specific variables
-- include: configure.yml
+  include_vars: "{{ item }}"
+  with_first_found:
+    - files:
+      - "{{ ansible_distribution|lower }}-{{ ansible_distribution_version|lower|replace('/', '_') }}.yml"
+      - "{{ ansible_distribution|lower }}-{{ ansible_distribution_release }}.yml"
+      - "{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml"
+      - "{{ ansible_distribution|lower }}.yml"
+      - "{{ ansible_os_family|lower }}.yml"
+      - defaults.yml
+      paths:
+      - ../vars
+
+- name: check for minimum kernel version
+  fail:
+    msg: >
+          docker requires a minimum kernel version of
+          {{ docker_kernel_min_version }} on
+          {{ ansible_distribution }}-{{ ansible_distribution_version }}
+  when: ansible_kernel|version_compare(docker_kernel_min_version, "<")
+
+- name: ensure docker requirements packages are installed
+  action: "{{ docker_package_info.pkg_mgr }}"
+  args: docker_package_info.args
+  with_items: docker_package_info.pre_pkgs
+  when: docker_package_info.pre_pkgs|length > 0
+
+- name: ensure docker repository public key is installed
+  action: "{{ docker_repo_key_info.pkg_key }}"
+  args: docker_repo_key_info.args
+  with_items: docker_repo_key_info.repo_keys
+  when: docker_repo_key_info.repo_keys|length > 0
+
+- name: ensure docker repository is enabled
+  action: "{{ docker_repo_info.pkg_repo }}"
+  args: docker_repo_info.args
+  with_items: docker_repo_info.repos
+  when: docker_repo_info.repos|length > 0
+
+- name: ensure docker packages are installed
+  action: "{{ docker_package_info.pkg_mgr }}"
+  args: docker_package_info.args
+  with_items: docker_package_info.pkgs
+  when: docker_package_info.pkgs|length > 0
+
+- name: ensure docker service is started and enabled
+  service:
+    name: "{{ item }}"
+    enabled: yes
+    state: started
+  with_items:
+    - docker

roles/docker/templates/default-docker.j2

@@ -1,13 +0,0 @@
-# Docker Upstart and SysVinit configuration file
-
-# Customize location of Docker binary (especially for development testing).
-#DOCKER="/usr/local/bin/docker"
-
-# Use DOCKER_OPTS to modify the daemon startup options.
-#DOCKER_OPTS=""
-
-# If you need Docker to use an HTTP proxy, it can also be specified here.
-#export http_proxy="http://127.0.0.1:3128/"
-
-# This is also a handy place to tweak where Docker's temporary files go.
-#export TMPDIR="/mnt/bigdrive/docker-tmp"

roles/docker/templates/docker.list.j2

@@ -1 +0,0 @@
-deb https://apt.dockerproject.org/repo {{ansible_distribution|lower}}-{{ ansible_distribution_release}} main

roles/docker/vars/centos-6.yml

@@ -0,0 +1,24 @@
+docker_kernel_min_version: '2.6.32-431'
+
+docker_package_info:
+  pkg_mgr: yum
+  args:
+      name: "{{ item }}"
+      state: latest
+      update_cache: yes
+  pre_pkgs:
+    - epel-release
+    - curl
+    - device-mapper-libs
+  pkgs:
+    - docker-io
+
+docker_repo_key_info:
+  pkg_key: ''
+  args: {}
+  repo_keys: []
+
+docker_repo_info:
+  pkg_repo: ''
+  args: {}
+  repos: []

roles/docker/vars/debian.yml

@@ -0,0 +1,36 @@
+docker_kernel_min_version: '3.2'
+
+docker_package_info:
+  pkg_mgr: apt
+  args:
+    pkg: "{{ item }}"
+    update_cache: yes
+    cache_valid_time: 600
+    state: latest
+  pre_pkgs: 
+    - apt-transport-https
+    - curl
+    - software-properties-common
+  pkgs:
+    - docker-engine
+
+docker_repo_key_info:
+  pkg_key: apt_key
+  args:
+    id: "{{ item }}"
+    keyserver: hkp://p80.pool.sks-keyservers.net:80
+    state: present
+  repo_keys:
+    - 58118E89F3A912897C070ADBF76221572C52609D 
+
+docker_repo_info:
+  pkg_repo: apt_repository
+  args:
+    repo: "{{ item }}"
+    update_cache: yes
+    state: present
+  repos:
+    - >
+       deb https://apt.dockerproject.org/repo 
+       {{ ansible_distribution|lower }}-{{ ansible_distribution_release|lower }}
+       main

roles/docker/vars/fedora-20.yml

@@ -0,0 +1,22 @@
+docker_kernel_min_version: '0'
+
+docker_package_info:
+  pkg_mgr: yum
+  args:
+      name: "{{ item }}"
+      state: latest
+      update_cache: yes
+  pre_pkgs: 
+    - curl
+  pkgs:
+    - docker-io
+
+docker_repo_key_info:
+  pkg_key: ''
+  args: {}
+  repo_keys: []
+
+docker_repo_info:
+  pkg_repo: ''
+  args: {}
+  repos: []

roles/docker/vars/main.yml

@@ -1,4 +0,0 @@
----
-#dockerhub_user:
-#dockerhub_pass:
-#dockerhub_email:

roles/docker/vars/redhat.yml

@@ -0,0 +1,22 @@
+docker_kernel_min_version: '0'
+
+docker_package_info:
+  pkg_mgr: yum
+  args:
+      name: "{{ item }}"
+      state: latest
+      update_cache: yes
+  pre_pkgs: 
+    - curl
+  pkgs:
+    - docker
+
+docker_repo_key_info:
+  pkg_key: ''
+  args: {}
+  repo_keys: []
+
+docker_repo_info:
+  pkg_repo: ''
+  args: {}
+  repos: []

roles/download/defaults/main.yml

@@ -1,15 +1,42 @@
 ---
-etcd_version: v2.2.2
+local_release_dir: /tmp
-flannel_version: 0.5.5
 
+flannel_version: 0.5.5
+calico_version: v0.13.0
+calico_plugin_version: v0.7.0
 kube_version: v1.1.3
+
 kubectl_checksum: "01b9bea18061a27b1cf30e34fd8ab45cfc096c9a9d57d0ed21072abb40dd3d1d"
 kubelet_checksum: "62191c66f2d670dd52ddf1d88ef81048977abf1ffaa95ee6333299447eb6a482"
 
-calico_version: v0.13.0
-
-etcd_download_url: "https://github.com/coreos/etcd/releases/download"
-flannel_download_url: "https://github.com/coreos/flannel/releases/download"
 kube_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/amd64"
-calico_download_url: "https://github.com/Metaswitch/calico-docker/releases/download"
 
+flannel_download_url: "https://github.com/coreos/flannel/releases/download/v{{ flannel_version }}/flannel-{{ flannel_version }}-linux-amd64.tar.gz"
+
+calico_download_url: "https://github.com/Metaswitch/calico-docker/releases/download/{{calico_version}}/calicoctl"
+
+calico_plugin_download_url: "https://github.com/projectcalico/calico-kubernetes/releases/download/{{calico_plugin_version}}/calico_kubernetes"
+
+downloads:
+  - name: calico
+    dest: calico/bin/calicoctl
+    url: "{{calico_download_url}}"
+
+  - name: calico-plugin
+    dest: calico/bin/calico
+    url: "{{calico_plugin_download_url}}"
+
+  - name: flannel
+    dest: flannel/flannel-{{ flannel_version }}-linux-amd64.tar.gz
+    url: "{{flannel_download_url}}"
+    unarchive: yes
+
+  - name: kubernetes-kubelet
+    dest: kubernetes/bin/kubelet
+    sha256: "{{kubelet_checksum}}"
+    url: "{{ kube_download_url }}/kubelet"
+
+  - name: kubernetes-kubectl
+    dest: kubernetes/bin/kubectl
+    sha256: "{{kubectl_checksum}}"
+    url: "{{ kube_download_url }}/kubectl"

roles/download/tasks/calico.yml

@@ -1,21 +0,0 @@
----
-- name: Create calico release directory
-  local_action: file
-     path={{ local_release_dir }}/calico/bin
-     recurse=yes
-     state=directory
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: Check if calicoctl has been downloaded
-  local_action: stat
-     path={{ local_release_dir }}/calico/bin/calicoctl
-  register: c_tar
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-# issues with get_url module and redirects, to be tested again in the near future
-- name: Download calico
-  local_action: shell
-    curl -o {{ local_release_dir }}/calico/bin/calicoctl -Ls {{ calico_download_url }}/{{ calico_version }}/calicoctl
-  when: not c_tar.stat.exists
-  register: dl_calico
-  delegate_to: "{{ groups['kube-master'][0] }}"

roles/download/tasks/etcd.yml

@@ -1,42 +0,0 @@
----
-- name: Create etcd release directory
-  local_action: file
-     path={{ local_release_dir }}/etcd/bin
-     recurse=yes
-     state=directory
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: Check if etcd release archive has been downloaded
-  local_action: stat
-     path={{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz
-  register: e_tar
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-# issues with get_url module and redirects, to be tested again in the near future
-- name: Download etcd
-  local_action: shell
-    curl -o {{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz -Ls {{ etcd_download_url }}/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz
-  when: not e_tar.stat.exists
-  register: dl_etcd
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: Extract etcd archive
-  local_action: unarchive
-     src={{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz
-     dest={{ local_release_dir }}/etcd copy=no
-  when: dl_etcd|changed
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: Pick up only etcd binaries
-  local_action: copy
-     src={{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/{{ item }}
-     dest={{ local_release_dir }}/etcd/bin
-  with_items:
-    - etcdctl
-    - etcd
-  when: dl_etcd|changed
-
-- name: Delete unused etcd files
-  local_action: file
-     path={{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64 state=absent
-  when: dl_etcd|changed

roles/download/tasks/flannel.yml

@@ -1,39 +0,0 @@
----
-- name: Create flannel release directory
-  local_action: file
-     path={{ local_release_dir }}/flannel
-     recurse=yes
-     state=directory
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: Check if flannel release archive has been downloaded
-  local_action: stat
-     path={{ local_release_dir }}/flannel/flannel-{{ flannel_version }}-linux-amd64.tar.gz
-  register: f_tar
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-# issues with get_url module and redirects, to be tested again in the near future
-- name: Download flannel
-  local_action: shell
-    curl -o {{ local_release_dir }}/flannel/flannel-{{ flannel_version }}-linux-amd64.tar.gz -Ls {{ flannel_download_url }}/v{{ flannel_version }}/flannel-{{ flannel_version }}-linux-amd64.tar.gz
-  when: not f_tar.stat.exists
-  register: dl_flannel
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: Extract flannel archive
-  local_action: unarchive
-     src={{ local_release_dir }}/flannel/flannel-{{ flannel_version }}-linux-amd64.tar.gz
-     dest={{ local_release_dir }}/flannel copy=no
-  when: dl_flannel|changed
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: Pick up only flannel binaries
-  local_action: copy
-     src={{ local_release_dir }}/flannel/flannel-{{ flannel_version }}/flanneld
-     dest={{ local_release_dir }}/flannel/bin
-  when: dl_flannel|changed
-
-- name: Delete unused flannel files
-  local_action: file
-     path={{ local_release_dir }}/flannel/flannel-{{ flannel_version }} state=absent
-  when: dl_flannel|changed

roles/download/tasks/kubernetes.yml

@@ -1,17 +0,0 @@
----
-- name: Create kubernetes binary directory
-  local_action: file
-     path="{{ local_release_dir }}/kubernetes/bin"
-     state=directory
-     recurse=yes
-
-- name: Download kubelet and kubectl
-  local_action: get_url
-    url="{{ kube_download_url }}/{{ item.name }}"
-    dest="{{ local_release_dir }}/kubernetes/bin"
-    sha256sum="{{ item.checksum }}"
-  with_items:
-    - name: kubelet
-      checksum: "{{ kubelet_checksum }}"
-    - name: kubectl
-      checksum: "{{ kubectl_checksum }}"

roles/download/tasks/main.yml

@@ -1,5 +1,19 @@
 ---
-- include: kubernetes.yml
+- name: Create dest directories
-- include: etcd.yml
+  file: path={{local_release_dir}}/{{item.dest|dirname}} state=directory recurse=yes
-- include: calico.yml
+  with_items: downloads
-- include: flannel.yml
+
+- name: Download items
+  get_url:
+    url: "{{item.url}}"
+    dest: "{{local_release_dir}}/{{item.dest}}"
+    sha256sum: "{{item.sha256 | default(omit)}}"
+  with_items: downloads
+
+- name: Extract archives
+  unarchive:
+     src: "{{ local_release_dir }}/{{item.dest}}"
+     dest: "{{ local_release_dir }}/{{item.dest|dirname}}"
+     copy: no
+  when: "{{item.unarchive is defined and item.unarchive == True}}"
+  with_items: downloads

roles/etcd/handlers/main.yml

@@ -1,14 +0,0 @@
----
-- name: reload systemd
-  command: systemctl daemon-reload
-
-- name: restart reloaded-etcd2
-  service:
-    name: etcd2
-    state: restarted
-
-- name: restart etcd2
-  command: /bin/true
-  notify:
-    - reload systemd
-    - restart reloaded-etcd2

roles/etcd/tasks/configure.yml

@@ -1,16 +0,0 @@
----
-- name: Copy etcd2.service systemd file
-  template:
-    src: systemd-etcd2.service.j2
-    dest: /lib/systemd/system/etcd2.service
-    backup: yes
-  notify:
-    - restart etcd2
-
-- name: Create etcd2 environment vars file
-  template:
-    src: etcd2-environment.j2
-    dest: /etc/etcd2-environment
-
-- name: Ensure etcd2 is running
-  service: name=etcd2 state=started enabled=yes

roles/etcd/tasks/install.yml

@@ -1,17 +0,0 @@
----
-- name: Create etcd user
-  user: name=etcd shell=/bin/nologin home=/var/lib/etcd2
-
-- name: Install etcd binaries
-  copy:
-     src={{ local_release_dir }}/etcd/bin/{{ item }}
-     dest={{ bin_dir }}
-     owner=etcd
-     mode=0755
-  with_items:
-    - etcdctl
-    - etcd
-  notify: restart etcd2
-
-- name: Create etcd2 binary symlink
-  file: src=/usr/local/bin/etcd dest=/usr/local/bin/etcd2 state=link

roles/etcd/tasks/main.yml

@@ -1,3 +1,13 @@
 ---
-- include: install.yml
+- name: ETCD2 | Stop etcd2 service
-- include: configure.yml
+  service: name=etcd state=stopped
+  ignore_errors: yes
+
+- name: ETCD2 | create etcd pod template
+  template: src=etcd-pod.yml dest=/etc/kubernetes/manifests/etcd-pod.manifest
+
+- name: ETCD2 | Check for etcd2 port
+  wait_for:
+    port: 2379
+    delay: 5
+    timeout: 100

roles/etcd/templates/etcd-pod.yml

@@ -0,0 +1,54 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: etcd
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+    - name: etcd
+      image: quay.io/coreos/etcd:v2.2.2
+      resources:
+        limits:
+          cpu: 100m
+          memory: 256M
+      args:
+{% if inventory_hostname in groups['etcd'] %}
+        - --name
+        - etcd-{{inventory_hostname}}-master
+        - --advertise-client-urls
+        - "http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379"
+        - --listen-peer-urls
+        - http://0.0.0.0:2380
+        - --initial-advertise-peer-urls
+        - http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2380
+        - --data-dir
+        - /var/etcd/data
+        - --initial-cluster-state
+        - new
+{% else %}
+        - --proxy
+        - 'on'
+{% endif %}
+        - --listen-client-urls
+        - "http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)  }}:2379,http://127.0.0.1:2379"
+        - --initial-cluster
+        - "{% for host in groups['etcd'] %}etcd-{{host}}-master=http://{{ hostvars[host]['ip'] | default( hostvars[host]['ansible_default_ipv4']['address'])   }}:2380{% if not loop.last %},{% endif %}{% endfor %}"
+        - --initial-cluster-token
+        - etcd-k8s-cluster
+      ports:
+        - name: etcd-client
+          containerPort: 2379
+          hostPort: 2379
+        - name: etcd-peer
+          containerPort: 2380
+          hostPort: 2380
+      volumeMounts:
+        - name: varetcd
+          mountPath: /var/etcd
+          readOnly: false
+  volumes:
+    - name: varetcd
+      hostPath:
+        path: /containers/pods/etcd-{{inventory_hostname}}/rootfs/var/etcd

roles/etcd/templates/etcd2-environment.j2

@@ -1,20 +0,0 @@
-ETCD_DATA_DIR="/var/lib/etcd2"
-{% if inventory_hostname in groups['etcd'] %}
-{% set etcd = {} %}
-{% for host in groups['etcd'] %}
-{% if inventory_hostname == host %}
-{% set _dummy = etcd.update({'name':"master"+loop.index|string}) %}
-{% endif %}
-{% endfor %}
-ETCD_ADVERTISE_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379"
-ETCD_INITIAL_ADVERTISE_PEER_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)  }}:2380"
-ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}master{{ loop.index|string }}=http://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:2380{% if not loop.last %},{% endif %}{% endfor %}"
-ETCD_INITIAL_CLUSTER_STATE="new"
-ETCD_INITIAL_CLUSTER_TOKEN="k8s_etcd"
-ETCD_LISTEN_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)  }}:2379,http://127.0.0.1:2379"
-ETCD_LISTEN_PEER_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)  }}:2380"
-ETCD_NAME="{{ etcd.name }}"
-{% else  %}
-ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}master{{ loop.index|string }}=http://{{ host }}:2380{% if not loop.last %},{% endif %}{% endfor %}"
-ETCD_LISTEN_CLIENT_URLS="http://127.0.0.1:23799"
-{% endif %}

roles/etcd/templates/systemd-etcd2.service.j2

@@ -1,18 +0,0 @@
-[Unit]
-Description=etcd2
-Conflicts=etcd.service
-
-[Service]
-User=etcd
-EnvironmentFile=/etc/etcd2-environment
-{% if inventory_hostname in groups['etcd'] %}
-ExecStart={{ bin_dir }}/etcd2
-{% else %}
-ExecStart={{ bin_dir }}/etcd2 -proxy on
-{% endif %}
-Restart=always
-RestartSec=10s
-LimitNOFILE=40000
-
-[Install]
-WantedBy=multi-user.target

roles/kubernetes/master/handlers/main.yml

@@ -2,24 +2,13 @@
 - name: reload systemd
   command: systemctl daemon-reload
 
-- name: restart kubelet
+- name: restart systemd-kubelet
   command: /bin/true
   notify:
     - reload systemd
-    - restart reloaded-kubelet
+    - restart kubelet
 
-- name: restart reloaded-kubelet
+- name: restart kubelet
   service:
     name: kubelet
     state: restarted
-
-- name: restart proxy
-  command: /bin/true
-  notify:
-    - reload systemd
-    - restart reloaded-proxy
-
-- name: restart reloaded-proxy
-  service:
-    name: kube-proxy
-    state: restarted

roles/kubernetes/master/tasks/main.yml

@@ -5,11 +5,16 @@
     dest: /etc/bash_completion.d/kubectl.sh
 
 - name: Install kubectl binary
-  copy:
+  synchronize:
-     src={{ local_release_dir }}/kubernetes/bin/kubectl
+     src: "{{ local_release_dir }}/kubernetes/bin/kubectl"
-     dest={{ bin_dir }}
+     dest: "{{ bin_dir }}/kubectl"
-     owner=kube
+     archive: no
-     mode=0755
+     checksum: yes
+     times: yes
+  delegate_to: "{{ groups['downloader'][0] }}"
+
+- name: Perms kubectl binary
+  file: path={{ bin_dir }}/kubectl owner=kube mode=0755 state=file
 
 - name: populate users for basic auth in API
   lineinfile:
@@ -27,11 +32,13 @@
     recursive: yes
     delete: yes
     rsync_opts: [ '--one-file-system']
+    set_remote_user: false
   with_items:
     - "{{ kube_token_dir }}"
     - "{{ kube_cert_dir }}"
     - "{{ kube_users_dir }}"
   delegate_to: "{{ groups['kube-master'][0] }}"
+  when: inventory_hostname != "{{ groups['kube-master'][0] }}"
 
 # Write manifests
 - name: Write kube-apiserver manifest
@@ -47,12 +54,7 @@
   wait_for:
     port: "{{kube_apiserver_insecure_port}}"
     delay: 10
-
+    timeout: 60
-- name: install required python module 'httplib2'
-  apt:
-    name: "python-httplib2"
-    state: present
-  when: inventory_hostname == groups['kube-master'][0]
 
 - name: Create 'kube-system' namespace
   uri:

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -10,7 +10,6 @@ spec:
     command:
     - /hyperkube
     - apiserver
-    - --insecure-bind-address=0.0.0.0
     - --etcd-servers={% for srv in groups['etcd'] %}http://{{ srv }}:2379{% if not loop.last %},{% endif %}{% endfor %}
 
     - --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota

roles/kubernetes/node/files/kube-gen-token.sh

@@ -19,7 +19,10 @@ token_file="${token_dir}/known_tokens.csv"
 
 create_accounts=($@)
 
-touch "${token_file}"
+if [ ! -e "${token_file}" ]; then
+  touch "${token_file}"
+fi
+
 for account in "${create_accounts[@]}"; do
   if grep ",${account}," "${token_file}" ; then
     continue

roles/kubernetes/node/handlers/main.yml

@@ -1,20 +1,14 @@
 ---
-- name: restart daemons
-  command: /bin/true
-  notify:
-    - reload systemd
-    - restart reloaded-kubelet
-
 - name: reload systemd
   command: systemctl daemon-reload
 
-- name: restart kubelet
+- name: restart systemd-kubelet
   command: /bin/true
   notify:
     - reload systemd
-    - restart reloaded-kubelet
+    - restart kubelet
 
-- name: restart reloaded-kubelet
+- name: restart kubelet
   service:
     name: kubelet
     state: restarted

roles/kubernetes/node/tasks/gen_tokens.yml

@@ -4,6 +4,7 @@
     src=kube-gen-token.sh
     dest={{ kube_script_dir }}
     mode=u+x
+  when: inventory_hostname == groups['kube-master'][0]
 
 - name: tokens | generate tokens for master components
   command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
@@ -14,6 +15,7 @@
     - "{{ groups['kube-master'] }}"
   register: gentoken
   changed_when: "'Added' in gentoken.stdout"
+  when: inventory_hostname == groups['kube-master'][0]
 
 - name: tokens | generate tokens for node components
   command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
@@ -24,3 +26,23 @@
     - "{{ groups['kube-node'] }}"
   register: gentoken
   changed_when: "'Added' in gentoken.stdout"
+  when: inventory_hostname == groups['kube-master'][0]
+
+- name: tokens | generate tokens for calico
+  command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
+  environment:
+    TOKEN_DIR: "{{ kube_token_dir }}"
+  with_nested:
+    - [ "system:calico" ]
+    - "{{ groups['k8s-cluster'] }}"
+  register: gentoken
+  changed_when: "'Added' in gentoken.stdout"
+  when: kube_network_plugin == "calico"
+  delegate_to: "{{ groups['kube-master'][0] }}"
+
+- name: tokens | get the calico token values
+  slurp:
+    src: "{{ kube_token_dir }}/system:calico-{{ inventory_hostname }}.token"
+  register: calico_token
+  when: kube_network_plugin == "calico"
+  delegate_to: "{{ groups['kube-master'][0] }}"

roles/kubernetes/node/tasks/install.yml

@@ -1,13 +1,48 @@
 ---
-- name: Write kubelet systemd init file
+- debug: msg="{{init_system == "systemd"}}"
+- debug: msg="{{init_system}}"
+
+- name: install | Write kubelet systemd init file
   template: src=kubelet.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes
+  when: init_system == "systemd"
+  notify: restart systemd-kubelet
+
+- name: install | Write kubelet initd script
+  template: src=deb-kubelet.initd.j2 dest=/etc/init.d/kubelet owner=root mode=755 backup=yes
+  when: init_system == "sysvinit" and ansible_os_family == "Debian"
   notify: restart kubelet
 
-- name: Install kubelet binary
+- name: install | Write kubelet initd script
-  copy:
+  template: src=rh-kubelet.initd.j2 dest=/etc/init.d/kubelet owner=root mode=755 backup=yes
-     src={{ local_release_dir }}/kubernetes/bin/kubelet
+  when: init_system == "sysvinit" and ansible_os_family == "RedHat"
-     dest={{ bin_dir }}
+  notify: restart kubelet
-     owner=kube
+
-     mode=0755
+- name: install | Install kubelet binary
+  synchronize:
+     src: "{{ local_release_dir }}/kubernetes/bin/kubelet"
+     dest: "{{ bin_dir }}/kubelet"
+     times: yes
+     archive: no
+  delegate_to: "{{ groups['downloader'][0] }}"
   notify:
     - restart kubelet
+
+- name: install | Perms kubelet binary
+  file: path={{ bin_dir }}/kubelet owner=kube mode=0755 state=file
+
+- name: install | Calico-plugin | Directory
+  file: path=/usr/libexec/kubernetes/kubelet-plugins/net/exec/calico/ state=directory
+  when: kube_network_plugin == "calico"
+
+- name: install | Calico-plugin | Binary
+  synchronize:
+    src: "{{ local_release_dir }}/calico/bin/calico"
+    dest: "/usr/libexec/kubernetes/kubelet-plugins/net/exec/calico/calico"
+    times: yes
+    archive: no
+  delegate_to: "{{ groups['downloader'][0] }}"
+  when: kube_network_plugin == "calico"
+  notify: restart kubelet
+
+- name: install | Perms calico plugin binary
+  file: path=/usr/libexec/kubernetes/kubelet-plugins/net/exec/calico/calico owner=kube mode=0755 state=file

roles/kubernetes/node/tasks/main.yml

@@ -21,14 +21,14 @@
     system=yes
     groups={{ kube_cert_group }}
 
-- include: install.yml
-
 - include: secrets.yml
   tags:
     - secrets
 
+- include: install.yml
+
 - name: Write kubelet config file
-  template: src=kubelet.j2 dest={{ kube_config_dir }}/kubelet.conf backup=yes
+  template: src=kubelet.j2 dest={{ kube_config_dir }}/kubelet backup=yes
   notify:
     - restart kubelet
 

roles/kubernetes/node/tasks/secrets.yml

@@ -18,8 +18,6 @@
   when: inventory_hostname == groups['kube-master'][0]
 
 - include: gen_tokens.yml
-  run_once: true
-  when: inventory_hostname == groups['kube-master'][0]
 
 # Sync certs between nodes
 - user:
@@ -45,8 +43,10 @@
     recursive: yes
     delete: yes
     rsync_opts: [ '--one-file-system']
+    set_remote_user: false
   with_items:
     - "{{ kube_cert_dir}}/ca.pem"
     - "{{ kube_cert_dir}}/node.pem"
     - "{{ kube_cert_dir}}/node-key.pem"
   delegate_to: "{{ groups['kube-master'][0] }}"
+  when: inventory_hostname not in "{{ groups['kube-master'] }}"

roles/kubernetes/node/templates/deb-kubelet.initd.j2

@@ -0,0 +1,119 @@
+#!/bin/bash
+#
+### BEGIN INIT INFO
+# Provides:   kubelet 
+# Required-Start:    $local_fs $network $syslog
+# Required-Stop:
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: The Kubernetes node container manager
+# Description:
+#   The Kubernetes container manager maintains docker state against a state file.
+### END INIT INFO
+
+
+# PATH should only include /usr/* if it runs after the mountnfs.sh script
+PATH=/sbin:/usr/sbin:/bin:/usr/bin
+DESC="The Kubernetes container manager"
+NAME=kubelet
+DAEMON={{ bin_dir }}/kubelet
+DAEMON_ARGS=""
+DAEMON_LOG_FILE=/var/log/$NAME.log
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+DAEMON_USER=root
+
+# Exit if the package is not installed
+[ -x "$DAEMON" ] || exit 0
+
+# Read configuration variable file if it is present
+[ -r /etc/kubernetes/$NAME ] && . /etc/kubernetes/$NAME
+
+# Define LSB log_* functions.
+# Depend on lsb-base (>= 3.2-14) to ensure that this file is present
+# and status_of_proc is working.
+. /lib/lsb/init-functions
+
+#
+# Function that starts the daemon/service
+#
+do_start()
+{
+        # Return
+        #   0 if daemon has been started
+        #   1 if daemon was already running
+        #   2 if daemon could not be started
+        start-stop-daemon --start --quiet --background --no-close \
+                --make-pidfile --pidfile $PIDFILE \
+                --exec $DAEMON -c $DAEMON_USER --test > /dev/null \
+                || return 1
+        start-stop-daemon --start --quiet --background --no-close \
+                --make-pidfile --pidfile $PIDFILE \
+                --exec $DAEMON -c $DAEMON_USER -- \
+                $DAEMON_ARGS >> $DAEMON_LOG_FILE 2>&1 \
+                || return 2
+}
+
+#
+# Function that stops the daemon/service
+#
+do_stop()
+{
+        # Return
+        #   0 if daemon has been stopped
+        #   1 if daemon was already stopped
+        #   2 if daemon could not be stopped
+        #   other if a failure occurred
+        start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME
+        RETVAL="$?"
+        [ "$RETVAL" = 2 ] && return 2
+        # Many daemons don't delete their pidfiles when they exit.
+        rm -f $PIDFILE
+        return "$RETVAL"
+}
+
+
+case "$1" in
+  start)
+        log_daemon_msg "Starting $DESC" "$NAME"
+        do_start
+        case "$?" in
+                0|1) log_end_msg 0 || exit 0 ;;
+                2) log_end_msg 1 || exit 1 ;;
+        esac
+        ;;
+  stop)
+        log_daemon_msg "Stopping $DESC" "$NAME"
+        do_stop
+        case "$?" in
+                0|1) log_end_msg 0 ;;
+                2) exit 1 ;;
+        esac
+        ;;
+  status)
+        status_of_proc -p $PIDFILE "$DAEMON" "$NAME" && exit 0 || exit $?
+        ;;
+
+  restart|force-reload)
+        log_daemon_msg "Restarting $DESC" "$NAME"
+        do_stop
+        case "$?" in
+          0|1)
+                do_start
+                case "$?" in
+                        0) log_end_msg 0 ;;
+                        1) log_end_msg 1 ;; # Old process is still running
+                        *) log_end_msg 1 ;; # Failed to start
+                esac
+                ;;
+          *)
+                # Failed to stop
+                log_end_msg 1
+                ;;
+        esac
+        ;;
+  *)
+        echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
+        exit 3
+        ;;
+esac

roles/kubernetes/node/templates/kubelet.j2

@@ -22,3 +22,7 @@ KUBELET_NETWORK_PLUGIN="--network_plugin={{ kube_network_plugin }}"
 {% endif %}
 # Should this cluster be allowed to run privileged docker containers
 KUBE_ALLOW_PRIV="--allow_privileged=true"
+{% if init_system == "sysvinit" %}
+DAEMON_ARGS="$KUBE_LOGTOSTDERR $KUBE_LOG_LEVEL $KUBE_ALLOW_PRIV $KUBELET_API_SERVER $KUBELET_ADDRESS \
+$KUBELET_HOSTNAME $KUBELET_REGISTER_NODE $KUBELET_ARGS $KUBELET_ARGS $KUBELET_NETWORK_PLUGIN"
+{% endif %}

roles/kubernetes/node/templates/kubelet.service.j2

@@ -8,8 +8,7 @@ After=docker.service
 {% endif %}
 
 [Service]
-EnvironmentFile=/etc/kubernetes/kubelet.conf
+EnvironmentFile=/etc/kubernetes/kubelet
-EnvironmentFile=/etc/network-environment
 ExecStart={{ bin_dir }}/kubelet \
 	    $KUBE_LOGTOSTDERR \
 	    $KUBE_LOG_LEVEL \

roles/kubernetes/node/templates/rh-kubelet.initd.j2

@@ -0,0 +1,129 @@
+#!/bin/bash
+#
+#       /etc/rc.d/init.d/kubelet
+#
+# chkconfig:   2345 95 95
+# description: Daemon for kubelet (kubernetes.io)
+
+### BEGIN INIT INFO
+# Provides:       kubelet
+# Required-Start: $local_fs $network $syslog cgconfig
+# Required-Stop:
+# Should-Start:
+# Should-Stop:
+# Default-Start: 2 3 4 5
+# Default-Stop:  0 1 6
+# Short-Description: start and stop kubelet
+# Description:
+#   The Kubernetes container manager maintains docker state against a state file.
+### END INIT INFO
+
+# Source function library.
+. /etc/rc.d/init.d/functions
+
+prog="kubelet"
+exec="{{ bin_dir }}/$prog"
+pidfile="/var/run/$prog.pid"
+lockfile="/var/lock/subsys/$prog"
+logfile="/var/log/$prog"
+
+[ -e /etc/kubernetes/$prog ] && . /etc/kubernetes/$prog
+
+start() {
+    if [ ! -x $exec ]; then
+      if [ ! -e $exec ]; then
+        echo "Docker executable $exec not found"
+      else
+        echo "You do not have permission to execute the Docker executable $exec"
+      fi	      
+      exit 5
+    fi
+
+    check_for_cleanup
+
+    if ! [ -f $pidfile ]; then
+        printf "Starting $prog:\t"
+        echo "\n$(date)\n" >> $logfile
+        $exec $DAEMON_ARGS &>> $logfile &
+        pid=$!
+        echo $pid >> $pidfile
+        touch $lockfile
+        success
+        echo
+    else
+        failure
+        echo
+        printf "$pidfile still exists...\n"
+        exit 7
+    fi
+}
+
+stop() {
+    echo -n $"Stopping $prog: "
+    killproc -p $pidfile -d 300 $prog
+    retval=$?
+    echo
+    [ $retval -eq 0 ] && rm -f $lockfile
+    return $retval
+}
+
+restart() {
+    stop
+    start
+}
+
+reload() {
+    restart
+}
+
+force_reload() {
+    restart
+}
+
+rh_status() {
+    status -p $pidfile $prog
+}
+
+rh_status_q() {
+    rh_status >/dev/null 2>&1
+}
+
+
+check_for_cleanup() {
+    if [ -f ${pidfile} ]; then
+        /bin/ps -fp $(cat ${pidfile}) > /dev/null || rm ${pidfile}
+    fi
+}
+
+case "$1" in
+    start)
+        rh_status_q && exit 0
+        $1
+        ;;
+    stop)
+        rh_status_q || exit 0
+        $1
+        ;;
+    restart)
+        $1
+        ;;
+    reload)
+        rh_status_q || exit 7
+        $1
+        ;;
+    force-reload)
+        force_reload
+        ;;
+    status)
+        rh_status
+        ;;
+    condrestart|try-restart)
+        rh_status_q || exit 0
+        restart
+        ;;
+    *)
+        echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}"
+        exit 2
+esac
+
+exit $?

roles/kubernetes/preinstall/defaults/main.yml

@@ -0,0 +1,15 @@
+---
+common_required_pkgs:
+  - python-httplib2
+  - openssl
+  - curl
+
+debian_required_pkgs:
+  - python-apt
+  - python-pip
+
+rh_required_pkgs:
+  - libselinux-python
+
+pypy_version: 2.4.0
+python_pypy_url: "https://bitbucket.org/pypy/pypy/downloads/pypy-{{ pypy_version }}.tar.bz2"

roles/kubernetes/preinstall/files/bootstrap.sh

@@ -0,0 +1,29 @@
+#/bin/bash
+set -e
+
+BINDIR="/usr/local/bin"
+
+cd $BINDIR
+
+if [[ -e $BINDIR/.bootstrapped ]]; then
+  exit 0
+fi
+
+PYPY_VERSION=2.4.0
+
+wget -O - https://bitbucket.org/pypy/pypy/downloads/pypy-$PYPY_VERSION-linux64.tar.bz2 |tar -xjf -
+mv -n pypy-$PYPY_VERSION-linux64 pypy
+
+## library fixup
+mkdir -p pypy/lib
+ln -snf /lib64/libncurses.so.5.9 $BINDIR/pypy/lib/libtinfo.so.5
+
+cat > $BINDIR/python <<EOF
+#!/bin/bash
+LD_LIBRARY_PATH=$BINDIR/pypy/lib:$LD_LIBRARY_PATH exec $BINDIR/pypy/bin/pypy "\$@"
+EOF
+
+chmod +x $BINDIR/python
+$BINDIR/python --version
+
+touch $BINDIR/.bootstrapped

roles/kubernetes/preinstall/files/runner

@@ -0,0 +1,3 @@
+#!/bin/bash
+BINDIR="/usr/local/bin"
+LD_LIBRARY_PATH=$BINDIR/pypy/lib:$LD_LIBRARY_PATH $BINDIR/pypy/bin/$(basename $0) $@

roles/kubernetes/preinstall/tasks/main.yml

@@ -0,0 +1,40 @@
+---
+- name: "Identify init system"
+  shell: >
+    $(pgrep systemd > /dev/null && systemctl status > /dev/null);
+    if [ $? -eq 0 ] ; then
+      echo systemd;
+    else
+      echo sysvinit;
+    fi
+  always_run: True
+  register: init_system_output
+  changed_when: False
+
+- set_fact:
+    init_system: "{{ init_system_output.stdout }}"
+
+- name: Install packages requirements
+  action:
+    module: "{{ ansible_pkg_mgr }}"
+    name: "{{ item }}"
+    state: latest
+  with_items: common_required_pkgs
+
+- name: Install debian packages requirements
+  apt:
+    name: "{{ item }}"
+    state: latest
+  when: ansible_os_family == "Debian"
+  with_items: debian_required_pkgs
+
+- name: Install redhat packages requirements
+  action:
+    module: "{{ ansible_pkg_mgr }}"
+    name: "{{ item }}"
+    state: latest
+  when: ansible_os_family == "RedHat"
+  with_items: rh_required_pkgs
+
+- include: python-bootstrap.yml
+  when: ansible_os_family not in [ "Debian", "RedHat" ]

roles/kubernetes/preinstall/tasks/python-bootstrap.yml

@@ -0,0 +1,41 @@
+---
+- name: Python | Check if bootstrap is needed
+  raw: stat {{ bin_dir}}/.bootstrapped
+  register: need_bootstrap
+  ignore_errors: True
+
+- name: Python | Run bootstrap.sh
+  script: bootstrap.sh
+  when: need_bootstrap | failed
+
+- set_fact:
+    ansible_python_interpreter: "{{ bin_dir }}/python"
+
+- name: Python | Check if we need to install pip
+  shell: "{{ansible_python_interpreter}} -m pip --version"
+  register: need_pip
+  ignore_errors: True
+  changed_when: false
+  when: need_bootstrap | failed
+
+- name: Python | Copy get-pip.py
+  copy: src=get-pip.py dest=~/get-pip.py
+  when: need_pip | failed
+
+- name: Python | Install pip
+  shell: "{{ansible_python_interpreter}} ~/get-pip.py"
+  when: need_pip | failed
+
+- name: Python | Remove get-pip.py
+  file: path=~/get-pip.py state=absent
+  when: need_pip | failed
+
+- name: Python | Install pip launcher
+  copy: src=runner dest={{ bin_dir }}/pip mode=0755
+  when: need_pip | failed
+
+- name: Install required python modules
+  pip:
+    name: "{{ item }}"
+  with_items: pip_python_modules
+

roles/network_plugin/handlers/main.yml

@@ -1,6 +1,17 @@
 ---
+- name : reload systemd
+  shell: systemctl daemon-reload
+
+- name: restart systemd-calico-node
+  command: /bin/true
+  notify:
+    - reload systemd
+    - restart calico-node
+
 - name: restart calico-node
-  service: name=calico-node state=restarted
+  service:
+    name: calico-node
+    state: restarted
 
 - name: restart docker
   service: name=docker state=restarted
@@ -23,6 +34,3 @@
 
 - name: start docker
   service: name=docker state=started
-
-- name : reload systemd
-  shell: systemctl daemon-reload

roles/network_plugin/tasks/calico.yml

@@ -1,41 +1,73 @@
 ---
+
 - name: Calico | Install calicoctl bin
-  copy:
+  synchronize:
-     src={{ local_release_dir }}/calico/bin/calicoctl
+    src: "{{ local_release_dir }}/calico/bin/calicoctl"
-     dest={{ bin_dir }}
+    dest: "{{ bin_dir }}/calicoctl"
-     mode=0755
+    archive: no
+    times: yes
+  delegate_to: "{{ groups['downloader'][0] }}"
   notify: restart calico-node
 
-- name: Calico | Create calicoctl symlink (needed by kubelet)
+- name: Calico | install calicoctl
-  file: src=/usr/local/bin/calicoctl dest=/usr/bin/calicoctl state=link
+  file: path={{ bin_dir }}/calicoctl mode=0755 state=file
 
-- name: Calico | Configure calico-node desired pool
+- name: Calico | Create calicoctl symlink (needed by kubelet)
-  shell: calicoctl pool add {{ kube_pods_subnet }}
+  file:
-  environment:
+    src: /usr/local/bin/calicoctl
-     ETCD_AUTHORITY: "{{ groups['etcd'][0] }}:2379"
+    dest: /usr/bin/calicoctl
+    state: link
+
+- name: Calico | Check if calico network pool has already been configured
+  uri:
+    url: "http://127.0.0.1:2379/v2/keys/calico/v1/ipam/v4/pool"
+    return_content: yes
+    status_code: 200,404
+  register: calico_conf
   run_once: true
   delegate_to: "{{ groups['etcd'][0] }}"
 
+- name: Calico | Configure calico network pool
+  shell: calicoctl pool add {{ kube_pods_subnet }}
+  run_once: true
+  when: calico_conf.status == 404
+  delegate_to: "{{ groups['etcd'][0] }}"
+
+- name: Calico | Get calico configuration from etcd
+  uri:
+    url: "http://127.0.0.1:2379/v2/keys/calico/v1/ipam/v4/pool"
+    return_content: yes
+  register: calico_pools
+  run_once: true
+  delegate_to: "{{ groups['etcd'][0] }}"
+
+- name: Calico | Check if calico pool is properly configured
+  fail:
+    msg: 'Only one network pool must be configured and it must be the subnet {{ kube_pods_subnet }}.
+    Please erase calico configuration and run the playbook again ("etcdctl rm --recursive /calico/v1/ipam/v4/pool")'
+  when: ( calico_pools.json['node']['nodes'] | length > 1 ) or
+        ( not calico_pools.json['node']['nodes'][0]['key'] | search(".*{{ kube_pods_subnet | ipaddr('network') }}.*") )
+  run_once: true
+  delegate_to: "{{ groups['etcd'][0] }}"
+
+- name: Calico | Write calico-node configuration
+  template: src=calico/calico.conf.j2 dest=/usr/libexec/kubernetes/kubelet-plugins/net/exec/calico/calico_kubernetes.ini
+  notify: restart calico-node
+
 - name: Calico | Write calico-node systemd init file
   template: src=calico/calico-node.service.j2 dest=/etc/systemd/system/calico-node.service
-  register: newservice
+  when: init_system == "systemd"
-  notify:
+  notify: restart systemd-calico-node
-    - reload systemd
-    - restart calico-node
 
-- name: Calico | daemon-reload
+- name: Calico | Write calico-node initd script
-  command: systemctl daemon-reload
+  template: src=calico/deb-calico.initd.j2 dest=/etc/init.d/calico-node owner=root mode=755
-  when: newservice|changed
+  when: init_system == "sysvinit" and ansible_os_family == "Debian"
-  changed_when: False
+  notify: restart calico-node
+
+- name: Calico | Write calico-node initd script
+  template: src=calico/rh-calico.initd.j2 dest=/etc/init.d/calico-node owner=root mode=755
+  when: init_system == "sysvinit" and ansible_os_family == "RedHat"
+  notify: restart calico-node
 
 - name: Calico | Enable calico-node
   service: name=calico-node enabled=yes state=started
-
-- name: Calico | Disable node mesh
-  shell: calicoctl bgp node-mesh off
-  when: peer_with_router|default(false) and inventory_hostname in groups['kube-node']
-
-- name: Calico | Configure peering with router(s)
-  shell: calicoctl node bgp peer add {{ item.router_id }} as {{ item.as }}
-  with_items: peers
-  when: peer_with_router|default(false) and inventory_hostname in groups['kube-node']

roles/network_plugin/tasks/flannel.yml

@@ -3,14 +3,18 @@
   user: name=flannel shell=/bin/nologin
 
 - name: Install flannel binaries
-  copy: 
+  synchronize:
-     src={{ local_release_dir }}/flannel/bin/flanneld
+     src: "{{ local_release_dir }}/flannel/bin/flanneld"
-     dest={{ bin_dir }}
+     dest: "{{ bin_dir }}/flanneld"
-     owner=flannel
+     archive: no
-     mode=u+x
+     times: yes
+  delegate_to: "{{ groups['downloader'][0] }}"
   notify:
     - restart flannel
 
+- name: Perms flannel binary
+  file: path={{ bin_dir }}/flanneld owner=flannel mode=0755 state=file
+
 - name: Write flannel.service systemd file
   template:
     src: flannel/systemd-flannel.service.j2

roles/network_plugin/tasks/main.yml

@@ -2,10 +2,7 @@
 - name: "Test if network plugin is defined"
   fail: msg="ERROR, One network_plugin variable must be defined (Flannel or Calico)"
   when: ( kube_network_plugin is defined and kube_network_plugin == "calico" and kube_network_plugin == "flannel" ) or
-        kube_network_plugin is not defined 
+        kube_network_plugin is not defined
-
-- name: Write network-environment
-  template: src=network-environment.j2 dest=/etc/network-environment mode=u+x
 
 - include: flannel.yml
   when: kube_network_plugin == "flannel"

roles/network_plugin/templates/calico/calico-node.service.j2

@@ -5,13 +5,12 @@ Requires=docker.service
 After=docker.service etcd2.service
 
 [Service]
-EnvironmentFile=/etc/network-environment
 User=root
 PermissionsStartOnly=true
 {% if inventory_hostname in groups['kube-node'] and peer_with_router|default(false)%}
-ExecStart={{ bin_dir }}/calicoctl node --kubernetes --ip=${DEFAULT_IPV4} --as={{ local_as }} --detach=false
+ExecStart={{ bin_dir }}/calicoctl node --kubernetes --ip={{ip | default(ansible_default_ipv4.address) }} --as={{ local_as }} --detach=false
 {%     else %}
-ExecStart={{ bin_dir }}/calicoctl node --kubernetes --ip=${DEFAULT_IPV4} --detach=false
+ExecStart={{ bin_dir }}/calicoctl node --kubernetes --ip={{ip | default(ansible_default_ipv4.address) }} --detach=false
 {%     endif %}
 Restart=always
 Restart=10

roles/network_plugin/templates/calico/calico.conf.j2

@@ -0,0 +1,17 @@
+[config]
+CALICO_IPAM=true
+
+# Location of etcd cluster used by Calico.  By default, this uses the etcd
+# instance running on the Kubernetes Master
+ETCD_AUTHORITY=127.0.0.1:2379
+
+# The kubernetes-apiserver location - used by the calico plugin
+{% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %}
+KUBE_API_ROOT=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }}/api/v1/
+{% else %}
+KUBE_API_ROOT=https://{{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address']) }}:{{kube_apiserver_port}}/api/v1/
+{% endif %}
+# Kubernetes authentication token
+{% if calico_token is defined | default('') %}
+KUBE_AUTH_TOKEN={{ calico_token.content|b64decode }}
+{% endif %}

roles/network_plugin/templates/calico/deb-calico.initd.j2

@@ -0,0 +1,114 @@
+#!/bin/bash
+#
+### BEGIN INIT INFO
+# Provides:   calico-node
+# Required-Start:    $local_fs $network $syslog
+# Required-Stop:
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Calico docker container
+# Description:
+#   Runs calico as a docker container
+### END INIT INFO
+
+PATH=/sbin:/usr/sbin:/bin:/usr/bin
+DESC="Calico-node Docker"
+NAME=calico-node
+DAEMON={{ bin_dir }}/calicoctl
+DAEMON_ARGS=""
+DOCKER=$(which docker)
+SCRIPTNAME=/etc/init.d/$NAME
+DAEMON_USER=root
+
+# Exit if the binary is not present
+[ -x "$DAEMON" ] || exit 0
+
+# Exit if the docker package is not installed
+[ -x "$DOCKER" ] || exit 0
+
+# Read configuration variable file if it is present
+[ -r /etc/network-environment ] && . /etc/network-environment
+
+# Define LSB log_* functions.
+# Depend on lsb-base (>= 3.2-14) to ensure that this file is present
+# and status_of_proc is working.
+. /lib/lsb/init-functions
+
+do_status()
+{
+    if [ $($DOCKER ps | awk '{ print $2 }' | grep calico/node | wc -l) -eq 1 ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+# Function that starts the daemon/service
+#
+do_start()
+{
+    do_status
+    retval=$?
+    if [ $retval -ne 0 ]; then
+        ${DAEMON} node --ip=${DEFAULT_IPV4} >>/dev/null && return 0 || return 2
+    else
+        return 1
+    fi
+}
+
+#
+# Function that stops the daemon/service
+#
+do_stop()
+{
+    ${DAEMON} node stop >> /dev/null || ${DAEMON} node stop --force >> /dev/null
+}
+
+
+case "$1" in
+  start)
+        log_daemon_msg "Starting $DESC" "$NAME"
+        do_start
+        case "$?" in
+                0|1) log_end_msg 0 || exit 0 ;;
+                2) log_end_msg 1 || exit 1 ;;
+        esac
+        ;;
+  stop)
+        log_daemon_msg "Stopping $DESC" "$NAME"
+        if do_stop; then
+            log_end_msg 0
+        else
+            log_failure_msg "Can't stop calico-node"
+            log_end_msg 1
+        fi
+        ;;
+  status)
+        if do_status; then
+            log_end_msg 0
+        else
+            log_failure_msg "Calico-node is not running"
+            log_end_msg 1
+        fi
+        ;;
+
+  restart|force-reload)
+        log_daemon_msg "Restarting $DESC" "$NAME"
+        if do_stop; then
+            if do_start; then
+                log_end_msg 0
+                exit 0
+            else
+                rc="$?"
+            fi
+        else
+           rc="$?"
+        fi
+        log_failure_msg "Can't restart Calico-node"
+        log_end_msg ${rc}
+        ;;
+  *)
+        echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
+        exit 3
+        ;;
+esac

roles/network_plugin/templates/calico/rh-calico.initd.j2

@@ -0,0 +1,130 @@
+#!/bin/bash
+#
+#       /etc/rc.d/init.d/calico-node
+#
+# chkconfig:   2345 95 95
+# description: Daemon for calico-node (http://www.projectcalico.org/)
+
+### BEGIN INIT INFO
+# Provides:       calico-node
+# Required-Start: $local_fs $network $syslog cgconfig
+# Required-Stop:
+# Should-Start:
+# Should-Stop:
+# Default-Start: 2 3 4 5
+# Default-Stop:  0 1 6
+# Short-Description: start and stop calico-node
+# Description:
+#   Manage calico-docker container
+### END INIT INFO
+
+# Source function library.
+. /etc/rc.d/init.d/functions
+
+prog="calicoctl"
+exec="{{ bin_dir }}/$prog"
+dockerexec="$(which docker)"
+logfile="/var/log/$prog"
+
+[ -e /etc/network-environment ] && for i in $(cat /etc/network-environment | egrep '(^$|^#)'); do export $i; done
+
+do_status()
+{
+    if [ $($dockerexec ps | awk '{ print $2 }' | grep calico/node | wc -l) -ne 1 ]; then
+        return 1
+    fi
+}
+
+do_start() {
+    if [ ! -x $exec ]; then
+      if [ ! -e $exec ]; then
+        echo "calico-node executable $exec not found"
+      else
+        echo "You do not have permission to execute the calico-node executable $exec"
+      fi	      
+      exit 5
+    fi
+
+    [ -x "$dockerexec" ] || exit 0
+
+    do_status
+    retval=$?
+    if [ $retval -ne 0 ]; then
+        printf "Starting $prog:\t"
+        echo "\n$(date)\n" >> $logfile
+        $exec node --ip=${DEFAULT_IPV4} &>>$logfile 
+        success
+        echo
+    else
+        echo -n "calico-node's already running"
+        success
+        exit 0
+    fi
+}
+
+do_stop() {
+    echo -n $"Stopping $prog: "
+    $exec node stop >> /dev/null || $exec node stop --force >> /dev/null
+    retval=$?
+    echo
+    return $retval
+}
+
+restart() {
+    do_stop
+    do_start
+}
+
+reload() {
+    restart
+}
+
+force_reload() {
+    restart
+}
+
+case "$1" in
+    start)
+        do_start
+        case "$?" in
+                0|1) success || exit 0 ;;
+                2) failure || exit 1 ;;
+        esac
+        ;;
+    stop)
+        echo -n "Stopping $DESC" "$NAME"
+        if do_stop; then
+            success
+            echo
+        else
+            echo -n "Can't stop calico-node"
+            failure
+            echo
+        fi
+        ;;
+    restart)
+        $1
+        ;;
+    reload)
+        $1
+        ;;
+    force-reload)
+        force_reload
+        ;;
+    status)
+        if do_status; then
+            echo -n "Calico-node is running"
+            success
+            echo
+        else
+            echo -n "Calico-node is not running"
+            failure
+            echo
+        fi
+        ;;
+    *)
+        echo $"Usage: $0 {start|stop|status|restart|reload|force-reload}"
+        exit 2
+esac
+
+exit $?

roles/network_plugin/templates/network-environment.j2

@@ -1,22 +0,0 @@
-#! /usr/bin/bash
-{% if kube_network_plugin == "calico" %}
-# This node's IPv4 address
-CALICO_IPAM=true
-DEFAULT_IPV4={{ip | default(ansible_default_ipv4.address) }}
-
-# The kubernetes master IP
-KUBERNETES_MASTER={{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address']) }}
-
-# Location of etcd cluster used by Calico.  By default, this uses the etcd
-# instance running on the Kubernetes Master
-{% if inventory_hostname in groups['etcd'] %}
-ETCD_AUTHORITY="127.0.0.1:2379"
-{% else %}
-ETCD_AUTHORITY="127.0.0.1:23799"
-{% endif %}
-
-# The kubernetes-apiserver location - used by the calico plugin
-KUBE_API_ROOT=http://{{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address']) }}:{{kube_apiserver_insecure_port}}/api/v1/
-{% else %}
-FLANNEL_ETCD_PREFIX="--etcd-prefix=/{{ cluster_name }}/network"
-{% endif %}