[jjo] add kube-router support (#3339)

* [jjo] add kube-router support

Fixes cloudnativelabs/kube-router#147.

* add kube-router as another network_plugin choice
* support most used kube-router flags via
  `kube_router_foo` vars as other plugins
* implement replacing kube-proxy (--run-service-proxy=true) via
  `kube_proxy_mode: none`, verified in a _non kubeadm_enabled_
  install, should also work for recent kubeadm releases via
  `skipKubeProxyInstall: true` config

* [jjo] address PR#3339 review from @woopstar

* add busybox image used by kube-router to downloads

* fix busybox download groups key

* rework kubeadm_enabled + kube_router_run_service_proxy

- verify it working ok w/the kubeadm_enabled and
  kube_router_run_service_proxy true or false

- introduce `kube_proxy_remove` fact, to decouple logic
  from kube_proxy_mode (which affects kubeadm configmap
  settings, thus no-good to ab-use it to 'none')

* improve kube-router.md re: kubeadm_enabled and kube_router_run_service_proxy

* address @woopstar latest review

* add inventory/sample/group_vars/k8s-cluster/k8s-net-kube-router.yml

* fix kube_router_run_service_proxy conditional for kube-proxy removal

* fix kube_proxy_remove fact (w/ |bool), add some needed kube-proxy tags on my and existing changes

* update kube-router tolerations for 1.12 compatibility

* add PriorityClass to kube-router DaemonSet

roles/network_plugin/kube-router/defaults/main.yml

@@ -0,0 +1,36 @@
+---
+# Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP
+kube_router_run_router: true
+
+# Enables Network Policy -- sets up iptables to provide ingress firewall for pods
+kube_router_run_firewall: true
+
+# Enables Service Proxy -- sets up IPVS for Kubernetes Services
+# see docs/kube-router.md "Caveats" section
+kube_router_run_service_proxy: false
+
+# Add Cluster IP of the service to the RIB so that it gets advertises to the BGP peers.
+kube_router_advertise_cluster_ip: false
+
+# Add External IP of service to the RIB so that it gets advertised to the BGP peers.
+kube_router_advertise_external_ip: false
+
+# Add LoadbBalancer IP of service status as set by the LB provider to the RIB so that it gets advertised to the BGP peers.
+kube_router_advertise_loadbalancer_ip: false
+
+# Array of arbitrary extra arguments to kube-router, see
+# https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md
+kube_router_extra_args: []
+
+# ASN numbers of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr.
+kube_router_peer_router_asns: ~
+
+# The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's.
+kube_router_peer_router_ips: ~
+
+# The remote port of the external BGP to which all nodes will peer. If not set, default BGP port (179) will be used.
+kube_router_peer_router_ports: ~
+
+# Setups node CNI to allow hairpin mode, requires node reboots, see
+# https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md#hairpin-mode
+kube_router_support_hairpin_mode: false

roles/network_plugin/kube-router/tasks/main.yml

@@ -0,0 +1,17 @@
+---
+
+- name: kube-router | Copy cni plugins from hyperkube
+  command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -prf /opt/cni/bin/. /cnibindir/"
+  register: cni_task_result
+  until: cni_task_result.rc == 0
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  changed_when: false
+  tags:
+    - hyperkube
+    - upgrade
+
+- name: kube-router | Create manifest
+  template:
+    src: kube-router.yml.j2
+    dest: "{{ kube_config_dir }}/kube-router.yml"

roles/network_plugin/kube-router/tasks/reset.yml

@@ -0,0 +1,9 @@
+---
+- name: reset | check kube-dummy-if network device
+  stat:
+    path: /sys/class/net/kube-dummy-if
+  register: kube_dummy_if
+
+- name: reset | remove the network device created by kube-router
+  command: ip link del kube-dummy-if
+  when: kube_dummy_if.stat.exists

roles/network_plugin/kube-router/templates/kube-router.yml.j2

@@ -0,0 +1,225 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kube-router-cfg
+  namespace: kube-system
+  labels:
+    tier: node
+    k8s-app: kube-router
+data:
+  cni-conf.json: |
+    {
+      "name":"kubernetes",
+      "type":"bridge",
+      "bridge":"kube-bridge",
+      "isDefaultGateway":true,
+{% if kube_router_support_hairpin_mode %}
+      "hairpinMode":true,
+{% endif %}
+      "ipam": {
+        "type":"host-local"
+      }
+    }
+  kubeconfig: |
+    apiVersion: v1
+    kind: Config
+    clusterCIDR: {{ kube_pods_subnet }}
+    clusters:
+    - name: cluster
+      cluster:
+        certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        server: {{ kube_apiserver_endpoint }}
+    users:
+    - name: kube-router
+      user:
+        tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    contexts:
+    - context:
+        cluster: cluster
+        user: kube-router
+      name: kube-router-context
+    current-context: kube-router-context
+
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  labels:
+    k8s-app: kube-router
+    tier: node
+  name: kube-router
+  namespace: kube-system
+spec:
+  minReadySeconds: 3
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-router
+        tier: node
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-cluster-critical
+{% endif %}
+      serviceAccountName: kube-router
+      containers:
+      - name: kube-router
+        image: {{ kube_router_image_repo }}:{{ kube_router_image_tag }}
+        imagePullPolicy: IfNotPresent
+        args:
+        - --run-router={{ kube_router_run_router | bool }}
+        - --run-firewall={{ kube_router_run_firewall | bool }}
+        - --run-service-proxy={{ kube_router_run_service_proxy | bool }}
+        - --kubeconfig=/var/lib/kube-router/kubeconfig
+{% if kube_router_advertise_cluster_ip %}
+        - --advertise-cluster-ip
+{% endif %}
+{% if kube_router_advertise_external_ip %}
+        - --advertise-external-ip
+{% endif %}
+{% if kube_router_advertise_loadbalancer_ip %}
+        - --advertise-loadbalancer-ip
+{% endif %}
+{% if kube_router_peer_router_asns %}
+        - --peer-router-asns {{ kube_router_peer_router_asns }}
+{% endif %}
+{% if kube_router_peer_router_ips %}
+        - --peer-router-ips {{ kube_router_peer_router_ips }}
+{% endif %}
+{% if kube_router_peer_router_ports %}
+        - --peer-router-ports {{ kube_router_peer_router_ports }}
+{% endif %}
+{% for arg in kube_router_extra_args %}
+        - "{{ arg }}"
+{% endfor %}
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 20244
+          initialDelaySeconds: 10
+          periodSeconds: 3
+        resources:
+          requests:
+            cpu: 250m
+            memory: 250Mi
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: cni-conf-dir
+          mountPath: /etc/cni/net.d
+        - name: kubeconfig
+          mountPath: /var/lib/kube-router
+          readOnly: true
+      initContainers:
+      - name: install-cni
+        image: {{ busybox_image_repo }}:{{ busybox_image_tag }}
+        imagePullPolicy: IfNotPresent
+        command:
+        - /bin/sh
+        - -c
+        - set -e -x;
+          if [ ! -f /etc/cni/net.d/10-kuberouter.conf ]; then
+            TMP=/etc/cni/net.d/.tmp-kuberouter-cfg;
+            cp /etc/kube-router/cni-conf.json ${TMP};
+            mv ${TMP} /etc/cni/net.d/10-kuberouter.conf;
+          fi;
+          if [ ! -f /var/lib/kube-router/kubeconfig ]; then
+            TMP=/var/lib/kube-router/.tmp-kubeconfig;
+            cp /etc/kube-router/kubeconfig ${TMP};
+            mv ${TMP} /var/lib/kube-router/kubeconfig;
+          fi
+        volumeMounts:
+        - mountPath: /etc/cni/net.d
+          name: cni-conf-dir
+        - mountPath: /etc/kube-router
+          name: kube-router-cfg
+        - name: kubeconfig
+          mountPath: /var/lib/kube-router
+      hostNetwork: true
+      tolerations:
+      - operator: Exists
+      # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+      - key: CriticalAddonsOnly
+        operator: "Exists"
+      volumes:
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: cni-conf-dir
+        hostPath:
+          path: /etc/cni/net.d
+      - name: kube-router-cfg
+        configMap:
+          name: kube-router-cfg
+      - name: kubeconfig
+        hostPath:
+          path: /var/lib/kube-router
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-router
+  namespace: kube-system
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: kube-router
+  namespace: kube-system
+rules:
+  - apiGroups:
+    - ""
+    resources:
+      - namespaces
+      - pods
+      - services
+      - nodes
+      - endpoints
+    verbs:
+      - list
+      - get
+      - watch
+  - apiGroups:
+    - "networking.k8s.io"
+    resources:
+      - networkpolicies
+    verbs:
+      - list
+      - get
+      - watch
+  - apiGroups:
+    - extensions
+    resources:
+      - networkpolicies
+    verbs:
+      - get
+      - list
+      - watch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: kube-router
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-router
+subjects:
+- kind: ServiceAccount
+  name: kube-router
+  namespace: kube-system

roles/network_plugin/meta/main.yml

@@ -30,5 +30,10 @@ dependencies:
     tags:
       - contiv
 
+  - role: network_plugin/kube-router
+    when: kube_network_plugin == 'kube-router'
+    tags:
+      - kube-router
+
   - role: network_plugin/cloud
     when: kube_network_plugin == 'cloud'