Flannel running as pod

roles/network_plugin/handlers/main.yml

@@ -1,36 +1,30 @@
 ---
-- name : reload systemd
-  shell: systemctl daemon-reload
-
 - name: restart systemd-calico-node
   command: /bin/true
   notify:
     - reload systemd
     - restart calico-node
 
+- name: restart systemd-docker
+  command: /bin/true
+  notify:
+    - reload systemd
+    - restart docker
+
+- name: delete default docker bridge
+  command: ip link delete docker0
+  ignore_errors: yes
+  notify: restart docker
+
+- name : reload systemd
+  shell: systemctl daemon-reload
+
 - name: restart calico-node
   service:
     name: calico-node
     state: restarted
 
 - name: restart docker
-  service: name=docker state=restarted
-
-- name: restart flannel
-  service: name=flannel state=restarted
-  notify:
-    - reload systemd
-    - stop docker
-    - delete docker0
-    - start docker
-  when: inventory_hostname in groups['kube-node']
-
-- name: stop docker
-  service: name=docker state=stopped
-
-- name: delete docker0
-  command: ip link delete docker0
-  ignore_errors: yes
-
-- name: start docker
-  service: name=docker state=started
+  service:
+    name: docker
+    state: restarted

roles/network_plugin/tasks/calico.yml

@@ -1,5 +1,4 @@
 ---
-
 - name: Calico | Install calicoctl bin
   synchronize:
     src: "{{ local_release_dir }}/calico/bin/calicoctl"
@@ -18,6 +17,10 @@
     dest: /usr/bin/calicoctl
     state: link
 
+- wait_for:
+    port: 2379
+  when: inventory_hostname in groups['kube-master']
+
 - name: Calico | Check if calico network pool has already been configured
   uri:
     url: "http://127.0.0.1:2379/v2/keys/calico/v1/ipam/v4/pool"
@@ -71,3 +74,16 @@
 
 - name: Calico | Enable calico-node
   service: name=calico-node enabled=yes state=started
+
+- name: Calico | Disable node mesh
+  shell: calicoctl bgp node-mesh off
+  environment:
+     ETCD_AUTHORITY: "{{ groups['etcd'][0] }}:2379"
+  when: peer_with_router|default(false) and inventory_hostname in groups['kube-node']
+
+- name: Calico | Configure peering with router(s)
+  shell: calicoctl node bgp peer add {{ item.router_id }} as {{ item.as }}
+  environment:
+     ETCD_AUTHORITY: "{{ groups['etcd'][0] }}:2379"
+  with_items: peers
+  when: peer_with_router|default(false) and inventory_hostname in groups['kube-node']

roles/network_plugin/tasks/flannel.yml

@@ -1,57 +1,34 @@
 ---
-- name: Create flannel user
-  user: name=flannel shell=/bin/nologin
-
-- name: Install flannel binaries
-  synchronize:
-     src: "{{ local_release_dir }}/flannel/bin/flanneld"
-     dest: "{{ bin_dir }}/flanneld"
-     archive: no
-     times: yes
-  delegate_to: "{{ groups['downloader'][0] }}"
-  notify:
-    - restart flannel
-
-- name: Perms flannel binary
-  file: path={{ bin_dir }}/flanneld owner=flannel mode=0755 state=file
-
-- name: Write flannel.service systemd file
+- name: Flannel | Write flannel configuration
   template:
-    src: flannel/systemd-flannel.service.j2
-    dest: /etc/systemd/system/flannel.service
-  notify: restart flannel
+    src: flannel/network.json
+    dest: /etc/flannel-network.json
+    backup: yes
+  when: inventory_hostname in groups['kube-node']
 
-- name: Write docker.service systemd file
+- name: Flannel | Create flannel pod manifest
   template:
-    src: flannel/systemd-docker.service.j2
-    dest: /lib/systemd/system/docker.service
-  notify: restart docker
+    src: flannel/flannel-pod.yml
+    dest: /etc/kubernetes/manifests/flannel-pod.manifest
+  notify: delete default docker bridge
 
-- name: Set fact for ectcd command conf file location
-  set_fact:
-    conf_file: "/tmp/flannel-conf.json"
-  run_once: true
-  delegate_to: "{{ groups['kube-master'][0] }}"
+- name: Flannel | Wait for flannel subnet.env file presence
+  wait_for:
+    path: /run/flannel/subnet.env
+    delay: 5
 
-- name: Create flannel config file to go in etcd
-  template: src=flannel/flannel-conf.json.j2 dest={{ conf_file }}
-  run_once: true
-  delegate_to: "{{ groups['kube-master'][0] }}"
+- name: Get flannel_subnet from subnet.env
+  shell: cat /run/flannel/subnet.env | awk -F'=' '$1 == "FLANNEL_SUBNET" {print $2}'
+  register: flannel_subnet_output
+  changed_when: false
 
-- name: Flannel configuration into etcd
-  shell: "{{ bin_dir }}/etcdctl set /{{ cluster_name }}/network/config < {{ conf_file }}"
-  delegate_to: "{{ groups['kube-master'][0] }}"
-  notify: restart flannel
+- set_fact:
+    flannel_subnet: "{{ flannel_subnet_output.stdout }}"
 
-- name: Clean up the flannel config file
-  file: path=/tmp/flannel-config.json state=absent
-  run_once: true
-  delegate_to: "{{ groups['kube-master'][0] }}"
+- name: Get flannel_mtu from subnet.env
+  shell: cat /run/flannel/subnet.env | awk -F'=' '$1 == "FLANNEL_MTU" {print $2}'
+  register: flannel_mtu_output
+  changed_when: false
 
-- name: Launch Flannel
-  service: name=flannel state=started enabled=yes
-  notify:
-    - restart flannel
-
-- name: Enable Docker
-  service: name=docker enabled=yes state=started
+- set_fact:
+    flannel_mtu: "{{ flannel_mtu_output.stdout }}"

roles/network_plugin/tasks/main.yml

@@ -7,7 +7,24 @@
 - include: flannel.yml
   when: kube_network_plugin == "flannel"
 
-- include: calico.yml
-  when: kube_network_plugin == "calico"
+- name: Set docker daemon options
+  template:
+    src: docker
+    dest: "/etc/default/docker"
+    owner: root
+    group: root
+    mode: 0644
+  notify:
+    - restart docker
+
+- name: Write docker.service systemd file
+  template:
+    src: systemd-docker.service
+    dest: /lib/systemd/system/docker.service
+  notify: restart systemd-docker
+  when: init_system == "systemd"
 
 - meta: flush_handlers
+
+- include: calico.yml
+  when: kube_network_plugin == "calico"

roles/network_plugin/templates/docker

@@ -0,0 +1,6 @@
+# Deployed by Ansible
+{% if init_system == "sysvinit" and kube_network_plugin == "flannel" and ansible_os_family == "Debian" %}
+DOCKER_OPTS="--bip={{ flannel_subnet }} --mtu={{ flannel_mtu }}"
+{% elif kube_network_plugin == "flannel" %}
+OPTIONS="--bip={{ flannel_subnet }} --mtu={{ flannel_mtu }}"
+{% endif %}

roles/network_plugin/templates/flannel/flannel-pod.yml

@@ -0,0 +1,46 @@
+---
+  kind: "Pod"
+  apiVersion: "v1"
+  metadata:
+    name: "flannel"
+    namespace: "kube-system"
+    labels:
+      app: "flannel"
+      version: "v0.1"
+  spec:
+    volumes:
+      - name: "subnetenv"
+        hostPath:
+          path: "/run/flannel"
+      - name: "networkconfig"
+        hostPath:
+          path: "/etc/flannel-network.json"
+    containers:
+      - name: "flannel-server-helper"
+        image: "gcr.io/google_containers/flannel-server-helper:0.1"
+        args:
+          - "--network-config=/etc/flannel-network.json"
+          - "--etcd-prefix=/{{ cluster_name }}/network"
+          - "--etcd-server=http://{{ groups['etcd'][0] }}:2379"
+        volumeMounts:
+          - name: "networkconfig"
+            mountPath: "/etc/flannel-network.json"
+        imagePullPolicy: "Always"
+      - name: "flannel-container"
+        image: "quay.io/coreos/flannel:0.5.5"
+        command:
+          - "/bin/sh"
+          - "-c"
+          - "/opt/bin/flanneld -etcd-endpoints {% for srv in groups['etcd'] %}http://{{ srv }}:2379{% if not loop.last %},{% endif %}{% endfor %} -etcd-prefix /{{ cluster_name }}/network 1>>/var/log/flannel_server.log 2>&1"
+        ports:
+          - hostPort: 10253
+            containerPort: 10253
+        resources:
+          limits:
+            cpu: "100m"
+        volumeMounts:
+          - name: "subnetenv"
+            mountPath: "/run/flannel"
+        securityContext:
+          privileged: true
+    hostNetwork: true

roles/network_plugin/templates/flannel/systemd-docker.service.j2

@@ -1,17 +0,0 @@
-[Unit]
-Description=Docker Application Container Engine
-Documentation=http://docs.docker.com
-After=network.target docker.socket flannel.service
-Requires=docker.socket
-
-[Service]
-EnvironmentFile=/run/flannel/subnet.env
-EnvironmentFile=-/etc/default/docker
-ExecStart=/usr/bin/docker -d -H fd:// --bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU} $DOCKER_OPTS
-MountFlags=slave
-LimitNOFILE=1048576
-LimitNPROC=1048576
-LimitCORE=infinity
-
-[Install]
-WantedBy=multi-user.target

roles/network_plugin/templates/flannel/systemd-flannel.service.j2

@@ -1,12 +0,0 @@
-[Unit]
-Description=Flannel Network Overlay
-Documentation=https://coreos.com/flannel/docs/latest
-
-[Service]
-EnvironmentFile=/etc/network-environment
-ExecStart={{ bin_dir }}/flanneld \
-       $FLANNEL_ETCD_PREFIX
-Restart=on-failure
-
-[Install]
-WantedBy=multi-user.target

roles/network_plugin/templates/systemd-docker.service

@@ -0,0 +1,28 @@
+[Unit]
+Description=Docker Application Container Engine
+Documentation=http://docs.docker.com
+{% if ansible_os_family == "RedHat" %}
+After=network.target
+Wants=docker-storage-setup.service
+{% elif ansible_os_family == "Debian" %}
+After=network.target docker.socket
+Requires=docker.socket
+{% endif %}
+
+[Service]
+Type=notify
+EnvironmentFile=-/etc/default/docker
+Environment=GOTRACEBACK=crash
+ExecStart=/usr/bin/docker daemon \
+          $OPTIONS \
+          $DOCKER_STORAGE_OPTIONS \
+          $DOCKER_NETWORK_OPTIONS \
+          $INSECURE_REGISTRY
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=infinity
+MountFlags=slave
+TimeoutStartSec=1min
+
+[Install]
+WantedBy=multi-user.target