Add calico/routereflector support

Add BGP route reflectors support in order to optimize BGP topology
for deployments with Calico network plugin.

Also bump version of calico/ctl for some bug fixes.

roles/network_plugin/calico/defaults/main.yml

@@ -12,6 +12,9 @@ overwrite_hyperkube_cni: true
 calico_cert_dir: /etc/calico/certs
 etcd_cert_dir: /etc/ssl/etcd/ssl
 
+# Global as_num (/calico/bgp/v1/global/as_num)
+global_as_num: "64512"
+
 # You can set MTU value here. If left undefined or empty, it will
 # not be specified in calico CNI config, so Calico will use built-in
 # defaults. The value should be a number, not a string.

roles/network_plugin/calico/rr/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+# Global as_num (/calico/bgp/v1/global/as_num)
+# should be the same as in calico role
+global_as_num: "64512"
+
+calico_cert_dir: /etc/calico/certs
+etcd_cert_dir: /etc/ssl/etcd/ssl

roles/network_plugin/calico/rr/handlers/main.yml

@@ -0,0 +1,15 @@
+---
+- name: restart calico-rr
+  command: /bin/true
+  notify:
+    - Calico-rr | reload systemd
+    - Calico-rr | reload calico-rr
+
+- name : Calico-rr | reload systemd
+  shell: systemctl daemon-reload
+  when: ansible_service_mgr == "systemd"
+
+- name: Calico-rr | reload calico-rr
+  service:
+    name: calico-rr
+    state: restarted

roles/network_plugin/calico/rr/meta/main.yml

@@ -0,0 +1,6 @@
+dependencies:
+  - role: kubernetes/secrets
+  - role: docker
+    when: ansible_os_family != "CoreOS"
+  - role: download
+    file: "{{ downloads.calico_rr }}"

roles/network_plugin/calico/rr/tasks/main.yml

@@ -0,0 +1,63 @@
+---
+# Required from inventory:
+#   calico_rr_ip - which specific IP to use for RR, defaults to
+#   "ip" from inventory or "ansible_default_ipv4.address"
+
+- name: Calico-rr | Set IP fact
+  set_fact:
+    rr_ip: "{{ calico_rr_ip | default(ip) | default(ansible_default_ipv4.address) }}"
+
+- name: Calico | Create calico certs directory
+  file:
+    dest: "{{ calico_cert_dir }}"
+    state: directory
+    mode: 0750
+    owner: root
+    group: root
+
+- name: Calico | Link etcd certificates for calico-node
+  file:
+    src: "{{ kube_cert_dir }}/{{ item.s }}"
+    dest: "{{ calico_cert_dir }}/{{ item.d }}"
+    state: hard
+    force: yes
+  with_items:
+    - {s: "ca.pem", d: "ca_cert.crt"}
+    - {s: "node.pem", d: "cert.crt"}
+    - {s: "node-key.pem", d: "key.pem"}
+
+- name: Calico-rr | Create dir for logs
+  file:
+    path: /var/log/calico-rr
+    state: directory
+    mode: 0755
+    owner: root
+    group: root
+
+- name: Calico-rr | Write calico-rr.env for systemd init file
+  template: src=calico-rr.env.j2 dest=/etc/calico/calico-rr.env
+  when: ansible_service_mgr == "systemd"
+  notify: restart calico-rr
+
+- name: Calico-rr | Write calico-rr systemd init file
+  template: src=calico-rr.service.j2 dest=/etc/systemd/system/calico-rr.service
+  when: ansible_service_mgr == "systemd"
+  notify: restart calico-rr
+
+- name: Calico-rr | Configure route reflector
+  command: |-
+    {{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} \
+    set /calico/bgp/v1/rr_v4/{{ rr_ip }} \
+    '{
+       "ip": "{{ rr_ip }}",
+       "cluster_id": "{{ cluster_id }}"
+     }'
+  delegate_to: "{{groups['etcd'][0]}}"
+
+- meta: flush_handlers
+
+- name: Calico-rr | Enable calico-rr
+  service:
+    name: calico-rr
+    state: started
+    enabled: yes

roles/network_plugin/calico/rr/templates/calico-rr.env.j2

@@ -0,0 +1,6 @@
+ETCD_ENDPOINTS="{{ etcd_access_endpoint }}"
+ETCD_CA_CERT_FILE="{{ calico_cert_dir }}/ca_cert.crt"
+ETCD_CERT_FILE="{{ calico_cert_dir }}/cert.crt"
+ETCD_KEY_FILE="{{ calico_cert_dir }}/key.pem"
+IP="{{ rr_ip }}"
+IP6=""

roles/network_plugin/calico/rr/templates/calico-rr.service.j2

@@ -0,0 +1,27 @@
+[Unit]
+Description=calico-rr
+After=docker.service
+Requires=docker.service
+
+[Service]
+EnvironmentFile=/etc/calico/calico-rr.env
+ExecStartPre=-/usr/bin/docker rm -f calico-rr
+ExecStart=/usr/bin/docker run --net=host --privileged \
+ --name=calico-rr \
+ -e IP=${IP} \
+ -e IP6=${IP6} \
+ -e ETCD_ENDPOINTS=${ETCD_ENDPOINTS} \
+ -e ETCD_CA_CERT_FILE=${ETCD_CA_CERT_FILE} \
+ -e ETCD_CERT_FILE=${ETCD_CERT_FILE} \
+ -e ETCD_KEY_FILE=${ETCD_KEY_FILE} \
+ -v /var/log/calico-rr:/var/log/calico \
+ -v {{ calico_cert_dir }}:{{ calico_cert_dir }}:ro \
+ {{ calico_rr_image_repo }}:{{ calico_rr_image_tag }}
+
+Restart=always
+RestartSec=10s
+
+ExecStop=-/usr/bin/docker stop calico-rr
+
+[Install]
+WantedBy=multi-user.target

roles/network_plugin/calico/tasks/main.yml

@@ -152,6 +152,16 @@
   run_once: true
   tags: facts
 
+- name: Calico | Set global as_num
+  command: "{{ bin_dir}}/calicoctl config set asNumber {{ global_as_num }}"
+  run_once: true
+  when: not legacy_calicoctl
+
+- name: Calico (old) | Set global as_num
+  command: "{{ bin_dir}}/calicoctl bgp default-node-as {{ global_as_num }}"
+  run_once: true
+  when: legacy_calicoctl
+
 - name: Calico | Write /etc/network-environment
   template: src=network-environment.j2 dest=/etc/network-environment
   when: ansible_service_mgr in ["sysvinit","upstart"]
@@ -191,8 +201,9 @@
 
 - name: Calico | Disable node mesh
   shell: "{{ bin_dir }}/calicoctl config set nodeToNodeMesh off"
-  when: (not legacy_calicoctl and
-         peer_with_router|default(false) and inventory_hostname in groups['kube-node'])
+  when: ((peer_with_router|default(false) or peer_with_calico_rr|default(false))
+          and inventory_hostname in groups['kube-node']
+          and not legacy_calicoctl)
   run_once: true
 
 - name: Calico | Configure peering with router(s)
@@ -208,10 +219,27 @@
   when: (not legacy_calicoctl and
          peer_with_router|default(false) and inventory_hostname in groups['kube-node'])
 
+- name: Calico | Configure peering with route reflectors
+  shell: >
+   echo '{
+   "kind": "bgpPeer",
+   "spec": {"asNumber": "{{ local_as | default(global_as_num)}}"},
+   "apiVersion": "v1",
+   "metadata": {"node": "{{ inventory_hostname }}",
+     "scope": "node",
+     "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"]) }}"}
+   }'
+   | {{ bin_dir }}/calicoctl create --skip-exists -f -
+  with_items: "{{ groups['calico-rr'] | default([]) }}"
+  when: (not legacy_calicoctl and
+         peer_with_calico_rr|default(false) and inventory_hostname in groups['kube-node']
+         and hostvars[item]['cluster_id'] == cluster_id)
+
 - name: Calico (old) | Disable node mesh
   shell: "{{ bin_dir }}/calicoctl bgp node-mesh off"
-  when: (legacy_calicoctl and
-         peer_with_router|default(false) and inventory_hostname in groups['kube-node'])
+  when: ((peer_with_router|default(false) or peer_with_calico_rr|default(false))
+          and inventory_hostname in groups['kube-node']
+          and legacy_calicoctl)
   run_once: true
 
 - name: Calico (old) | Configure peering with router(s)
@@ -219,3 +247,10 @@
   with_items: "{{ peers|default([]) }}"
   when: (legacy_calicoctl and
          peer_with_router|default(false) and inventory_hostname in groups['kube-node'])
+
+- name: Calico (old) | Configure peering with route reflectors
+  shell: "{{ bin_dir }}/calicoctl node bgp peer add {{ hostvars[item]['calico_rr_ip']|default(hostvars[item]['ip']) }} as {{ local_as | default(global_as_num) }}"
+  with_items: "{{ groups['calico-rr'] | default([]) }}"
+  when: (legacy_calicoctl and
+         peer_with_calico_rr|default(false) and inventory_hostname in groups['kube-node']
+         and hostvars[item]['cluster_id'] == cluster_id)