Adds support for Multus (multiple interfaces) CNI plugin (#3166)

* Adds support for Multus (multiple interfaces) CNI plugin

Multus is a latin word for "Multi". As the name suggests, it acts as a
Multi plugin in Kubernetes and provides multiple network interface
support in a pod. Multus uses the concept of invoking delegates by
grouping multiple plugins into delegates and invoking them in the
sequential order of the CNI configuration file provided in json format.

* Change CNI version (0.1.0->0.3.1) of Contiv to be compatible with Multus

roles/download/defaults/main.yml

@@ -59,6 +59,7 @@ pod_infra_version: 3.1
 contiv_version: 1.2.1
 cilium_version: "v1.3.0"
 kube_router_version: "v0.2.1"
+multus_version: "v3.1.autoconf"
 
 # Download URLs
 kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/{{ image_arch }}/kubeadm"
@@ -160,6 +161,8 @@ cilium_image_repo: "docker.io/cilium/cilium"
 cilium_image_tag: "{{ cilium_version }}"
 kube_router_image_repo: "cloudnativelabs/kube-router"
 kube_router_image_tag: "{{ kube_router_version }}"
+multus_image_repo: "docker.io/nfvpe/multus"
+multus_image_tag: "{{ multus_version }}"
 nginx_image_repo: nginx
 nginx_image_tag: 1.13
 dnsmasq_version: 2.78
@@ -290,6 +293,15 @@ downloads:
     groups:
       - k8s-cluster
 
+  multus:
+    enabled: "{{ kube_network_plugin_multus }}"
+    container: true
+    repo: "{{ multus_image_repo }}"
+    tag: "{{ multus_image_tag }}"
+    sha256: "{{ multus_digest_checksum|default(None) }}"
+    groups:
+      - k8s-cluster
+
   flannel:
     enabled: "{{ kube_network_plugin == 'flannel' or kube_network_plugin == 'canal' }}"
     container: true

roles/kubernetes-apps/network_plugin/meta/main.yml

@@ -34,3 +34,8 @@ dependencies:
     when: kube_network_plugin == 'kube-router'
     tags:
       - kube-router
+
+  - role: kubernetes-apps/network_plugin/multus
+    when: kube_network_plugin_multus
+    tags:
+      - multus

roles/kubernetes-apps/network_plugin/multus/tasks/main.yml

@@ -0,0 +1,11 @@
+---
+- name: Multus | Start resources
+  kube:
+    name: "{{item.item.name}}"
+    namespace: "kube-system"
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: "{{item.item.type}}"
+    filename: "{{kube_config_dir}}/{{item.item.file}}"
+    state: "latest"
+  with_items: "{{ multus_manifest_1.results }} + {{multus_manifest_2.results }}"
+  when: inventory_hostname == groups['kube-master'][0] and not item|skipped

roles/kubespray-defaults/defaults/main.yaml

@@ -103,6 +103,7 @@ kube_users:
 # Choose network plugin (cilium, calico, weave or flannel)
 # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
 kube_network_plugin: calico
+kube_network_plugin_multus: false
 
 # Determines if calico-rr group exists
 peer_with_calico_rr: "{{ 'calico-rr' in groups and groups['calico-rr']|length > 0 }}"

roles/network_plugin/contiv/defaults/main.yml

@@ -4,7 +4,7 @@ contiv_config_dir: "{{ kube_config_dir }}/contiv"
 contiv_etcd_conf_dir: "/etc/contiv/etcd"
 contiv_etcd_data_dir: "/var/lib/etcd/contiv-data"
 contiv_netmaster_port: 9999
-contiv_cni_version: 0.1.0
+contiv_cni_version: 0.3.1
 
 # No need to download it by default, but must be defined
 contiv_etcd_image_repo: "{{ etcd_image_repo }}"

roles/network_plugin/meta/main.yml

@@ -37,3 +37,8 @@ dependencies:
 
   - role: network_plugin/cloud
     when: kube_network_plugin == 'cloud'
+
+  - role: network_plugin/multus
+    when: kube_network_plugin_multus
+    tags:
+      - multus

roles/network_plugin/multus/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+multus_conf_file: "auto"
+multus_cni_conf_dir_host: "/etc/cni/net.d"
+multus_cni_bin_dir_host: "/opt/cni/bin"
+multus_cni_conf_dir: "{{ ('/host',  multus_cni_conf_dir_host) | join }}"
+multus_cni_bin_dir: "{{ ('/host', multus_cni_bin_dir_host) | join }}"
+multus_kubeconfig_file_host: "{{ (multus_cni_conf_dir_host, '/multus.d/multus.kubeconfig') | join }}"

roles/network_plugin/multus/files/multus-clusterrole.yml

@@ -0,0 +1,16 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: multus
+rules:
+- apiGroups:
+  - '*'
+  resources:
+  - '*'
+  verbs:
+  - '*'
+- nonResourceURLs:
+  - '*'
+  verbs:
+  - '*'

roles/network_plugin/multus/files/multus-clusterrolebinding.yml

@@ -0,0 +1,13 @@
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system

roles/network_plugin/multus/files/multus-crd.yml

@@ -0,0 +1,22 @@
+---
+kind: CustomResourceDefinition
+apiVersion: apiextensions.k8s.io/v1beta1
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  version: v1
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+    - net-attach-def
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          properties:
+            config:
+              type: string

roles/network_plugin/multus/files/multus-serviceaccount.yml

@@ -0,0 +1,6 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system

roles/network_plugin/multus/tasks/main.yml

@@ -0,0 +1,19 @@
+---
+- name: Multus | Copy manifest files
+  copy:
+    src: "{{ item.file }}"
+    dest: "{{ kube_config_dir }}"
+  with_items:
+    - {name: multus-crd, file: multus-crd.yml, type: customresourcedefinition}
+    - {name: multus-serviceaccount, file: multus-serviceaccount.yml, type: serviceaccount}
+    - {name: multus-clusterrole, file: multus-clusterrole.yml, type: clusterrole}
+    - {name: multus-clusterrolebinding, file: multus-clusterrolebinding.yml, type: clusterrolebinding}
+  register: multus_manifest_1
+
+- name: Multus | Copy manifest templates
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+  with_items:
+    - {name: multus-daemonset, file: multus-daemonset.yml, type: daemonset}
+  register: multus_manifest_2

roles/network_plugin/multus/templates/multus-daemonset.yml.j2

@@ -0,0 +1,54 @@
+---
+kind: DaemonSet
+apiVersion: extensions/v1beta1
+metadata:
+  name: kube-multus-ds-amd64
+  namespace: kube-system
+  labels:
+    tier: node
+    app: multus
+spec:
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: multus
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/arch: amd64
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        image: {{ multus_image_repo }}:{{ multus_image_tag }}
+        command: ["/entrypoint.sh"]
+        args:
+        - "--cni-conf-dir={{ multus_cni_conf_dir }}"
+        - "--cni-bin-dir={{ multus_cni_bin_dir }}"
+        - "--multus-conf-file={{ multus_conf_file }}"
+        - "--multus-kubeconfig-file-host={{ multus_kubeconfig_file_host }}"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: {{  multus_cni_conf_dir }}
+        - name: cnibin
+          mountPath: {{ multus_cni_bin_dir }}
+      volumes:
+      - name: cni
+        hostPath:
+          path: {{ multus_cni_conf_dir_host }}
+      - name: cnibin
+        hostPath:
+          path: {{ multus_cni_bin_dir_host }}