Merge branch 'master' into gpu2

roles/download/defaults/main.yml

@@ -33,6 +33,7 @@ image_arch: "{{host_architecture | default('amd64')}}"
 kube_version: v1.11.3
 kubeadm_version: "{{ kube_version }}"
 etcd_version: v3.2.18
+
 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
 # after migration to container download
 calico_version: "v3.1.3"
@@ -40,12 +41,14 @@ calico_ctl_version: "v3.1.3"
 calico_cni_version: "v3.1.3"
 calico_policy_version: "v3.1.3"
 calico_rr_version: "v0.6.1"
+
 flannel_version: "v0.10.0"
 flannel_cni_version: "v0.3.0"
+
 vault_version: 0.10.1
-weave_version: "2.4.0"
+weave_version: "2.4.1"
 pod_infra_version: 3.1
-contiv_version: 1.1.7
+contiv_version: 1.2.1
 cilium_version: "v1.2.0"
 
 # Download URLs
@@ -55,10 +58,39 @@ etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_ver
 hyperkube_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/amd64/hyperkube"
 
 # Checksums
-etcd_checksum: b729db0732448064271ea6fdcb901773c4fe917763ca07776f22d0e5e0bd4097
-hyperkube_checksum: dac8da16dd6688e52b5dc510f5dd0a20b54350d52fb27ceba2f018ba2c8be692
-kubeadm_checksum: 422a7a32ed9a7b1eaa2a4f9d121674dfbe80eb41e206092c13017d097f75aaec
+hyperkube_checksums:
+  v1.11.3: dac8da16dd6688e52b5dc510f5dd0a20b54350d52fb27ceba2f018ba2c8be692
+  v1.11.2: d727f8cae3fc26b1add9b4ff0d4d9b99605544ff7fb3baeecdca394362adbfb8
+  v1.11.1: 019ce1ecf4c6a70c06a7f4ef107443351458b4d9e6b9ce4a436bfbfbef93feea
+  v1.11.0: 7e191c164dc2c942abd37e4b50846e0be31ca959afffeff6b034beacbc2a106a
+  v1.10.8: f8a68514a6c858089f44ec93b2ffb2d764ea67d3b02b19112348f73ffcfe4386
+  v1.10.7: 13e25eb39467014fd169f38b7cd6bec8ff55525b8001c7abba85957e6470b6cc
+  v1.10.6: 0daa34fa58470e5f20def10d3dd544922c28c558719d3338ad8c524154c91257
+  v1.10.5: 1a53456f9d33a7c07adb1636f20f1d0b92b8e7647063a70d0ce134a238e680fe
+  v1.10.4: 16e36693c15494036d930139a749ec1bc492b7fefa2c3adc1abbe8f38178ae7c
+  v1.10.3: e807753dc309635902a56069ee06fc390944ef034b72c53b2e1e51d0c9ead8a3
+  v1.10.2: 3843fb594a18c4a64d77736bab72000ec4b8c4ddf178e20ec3249f709e9ed9c1
+  v1.10.1: 6e0642ad6bae68dc81b8d1c9efa18e265e17e23da1895862823cafac08c0344c
+  v1.10.0: b5575b2fb4266754c1675b8cd5d9b6cac70f3fee7a05c4e80da3a9e83e58c57e
+kubeadm_checksums:
+  v1.11.3: 422a7a32ed9a7b1eaa2a4f9d121674dfbe80eb41e206092c13017d097f75aaec
+  v1.11.2: 6b17720a65b8ff46efe92a5544f149c39a221910d89939838d75581d4e6924c0
+  v1.11.1: 425ec24b95f7217ee06d1588aba22f206a5829f8c6a5352c2862368552361fe6
+  v1.11.0: 0000478fc59a24ec1727de744188d13c4d702a644954132efa9d9954371b3553
+  v1.10.8: 42660875dd94c93267bd2f567c67d692b362bd143d7502967a62c5474b2b25b8
+  v1.10.7: cdeb07fd3705e973800c4aa0b8a510d5dba1de8e1039428cfebdaf3d93e332b6
+  v1.10.6: e1d49a6b33b384f681468add2e9ee08552069ae0d6b0ad59e1c943ddbaeac3fa
+  v1.10.5: f231d4bcc9f2ed15597272e5359e380cc760c0b57a1f7cb97ce2bbab5df774e0
+  v1.10.4: 7e1169bbbeed973ab402941672dec957638dea5952a1e8bc89a37d5e709cc4b4
+  v1.10.3: b2a6f0764b89a4a13a3da4471af943ce98efeb29e2913c9e7880fe27f4f43a5f
+  v1.10.2: 394d7d340214c91d669186cf4f2110d8eb840ca965399b4d8b22d0545a60e377
+  v1.10.1: 012e48fb92b1c22543b12ab2db7d780777972043287404c98cca4d2c6ec964ec
+  v1.10.0: ebbac985834289037b544523c3e2f39bb44bea938aca9d9e88ef7e880fb8472f
+
+etcd_binary_checksum: b729db0732448064271ea6fdcb901773c4fe917763ca07776f22d0e5e0bd4097
 vault_binary_checksum: 3c4d70ba71619a43229e65c67830e30e050eab7a81ac6b28325ff707e5914188
+hyperkube_binary_checksum: "{{ hyperkube_checksums[kube_version] }}"
+kubeadm_binary_checksum: "{{ kubeadm_checksums[kubeadm_version] }}"
 
 # Containers
 # In some cases, we need a way to set --registry-mirror or --insecure-registry for docker,
@@ -95,16 +127,20 @@ netcheck_agent_img_repo: "mirantis/k8s-netchecker-agent"
 netcheck_agent_tag: "{{ netcheck_version }}"
 netcheck_server_img_repo: "mirantis/k8s-netchecker-server"
 netcheck_server_tag: "{{ netcheck_version }}"
-weave_kube_image_repo: "weaveworks/weave-kube"
+weave_kube_image_repo: "docker.io/weaveworks/weave-kube"
 weave_kube_image_tag: "{{ weave_version }}"
-weave_npc_image_repo: "weaveworks/weave-npc"
+weave_npc_image_repo: "docker.io/weaveworks/weave-npc"
 weave_npc_image_tag: "{{ weave_version }}"
 contiv_image_repo: "contiv/netplugin"
 contiv_image_tag: "{{ contiv_version }}"
+contiv_init_image_repo: "contiv/netplugin-init"
+contiv_init_image_tag: "latest"
 contiv_auth_proxy_image_repo: "contiv/auth_proxy"
 contiv_auth_proxy_image_tag: "{{ contiv_version }}"
 contiv_etcd_init_image_repo: "ferest/etcd-initer"
 contiv_etcd_init_image_tag: latest
+contiv_ovs_image_repo: "contiv/ovs"
+contiv_ovs_image_tag: "latest"
 cilium_image_repo: "docker.io/cilium/cilium"
 cilium_image_tag: "{{ cilium_version }}"
 nginx_image_repo: nginx
@@ -112,7 +148,7 @@ nginx_image_tag: 1.13
 dnsmasq_version: 2.78
 dnsmasq_image_repo: "andyshinn/dnsmasq"
 dnsmasq_image_tag: "{{ dnsmasq_version }}"
-kubedns_version: 1.14.10
+kubedns_version: 1.14.11
 kubedns_image_repo: "gcr.io/google_containers/k8s-dns-kube-dns-{{ image_arch }}"
 kubedns_image_tag: "{{ kubedns_version }}"
 
@@ -161,7 +197,7 @@ ingress_nginx_controller_image_repo: "quay.io/kubernetes-ingress-controller/ngin
 ingress_nginx_controller_image_tag: "0.19.0"
 ingress_nginx_default_backend_image_repo: "gcr.io/google_containers/defaultbackend"
 ingress_nginx_default_backend_image_tag: "1.4"
-cert_manager_version: "v0.4.1"
+cert_manager_version: "v0.5.0"
 cert_manager_controller_image_repo: "quay.io/jetstack/cert-manager-controller"
 cert_manager_controller_image_tag: "{{ cert_manager_version }}"
 
@@ -174,6 +210,7 @@ downloads:
     sha256: "{{ netcheck_server_digest_checksum|default(None) }}"
     groups:
       - k8s-cluster
+
   netcheck_agent:
     enabled: "{{ deploy_netchecker }}"
     container: true
@@ -182,20 +219,16 @@ downloads:
     sha256: "{{ netcheck_agent_digest_checksum|default(None) }}"
     groups:
       - k8s-cluster
+
   etcd:
+    container: "{{ etcd_deployment_type != 'host' }}"
+    file: "{{ etcd_deployment_type == 'host' }}"
     enabled: true
-    container: true
-    repo: "{{ etcd_image_repo }}"
-    tag: "{{ etcd_image_tag }}"
-    sha256: "{{ etcd_digest_checksum|default(None) }}"
-    groups:
-      - etcd
-  etcd_file:
-    enabled: true
-    file: true
     version: "{{ etcd_version }}"
     dest: "etcd-{{ etcd_version }}-linux-amd64.tar.gz"
-    sha256: "{{ etcd_checksum }}"
+    repo: "{{ etcd_image_repo }}"
+    tag: "{{ etcd_image_tag }}"
+    sha256: "{{ etcd_binary_checksum if etcd_deployment_type == 'host' else etcd_digest_checksum|d(None) }}"
     source_url: "{{ etcd_download_url }}"
     url: "{{ etcd_download_url }}"
     unarchive: true
@@ -203,12 +236,13 @@ downloads:
     mode: "0755"
     groups:
       - etcd
+
   kubeadm:
     enabled: "{{ kubeadm_enabled }}"
     file: true
     version: "{{ kubeadm_version }}"
     dest: "kubeadm"
-    sha256: "{{ kubeadm_checksum }}"
+    sha256: "{{ kubeadm_binary_checksum }}"
     source_url: "{{ kubeadm_download_url }}"
     url: "{{ kubeadm_download_url }}"
     unarchive: false
@@ -216,6 +250,7 @@ downloads:
     mode: "0755"
     groups:
       - k8s-cluster
+
   hyperkube:
     enabled: true
     container: true
@@ -224,12 +259,13 @@ downloads:
     sha256: "{{ hyperkube_digest_checksum|default(None) }}"
     groups:
       - k8s-cluster
+
   hyperkube_file:
     enabled: true
     file: true
     version: "{{ kube_version }}"
     dest: "hyperkube"
-    sha256: "{{ hyperkube_checksum }}"
+    sha256: "{{ hyperkube_binary_checksum }}"
     source_url: "{{ hyperkube_download_url }}"
     url: "{{ hyperkube_download_url }}"
     unarchive: false
@@ -237,6 +273,7 @@ downloads:
     mode: "0755"
     groups:
       - k8s-cluster
+
   cilium:
     enabled: "{{ kube_network_plugin == 'cilium' }}"
     container: true
@@ -245,6 +282,7 @@ downloads:
     sha256: "{{ cilium_digest_checksum|default(None) }}"
     groups:
       - k8s-cluster
+
   flannel:
     enabled: "{{ kube_network_plugin == 'flannel' or kube_network_plugin == 'canal' }}"
     container: true
@@ -253,6 +291,7 @@ downloads:
     sha256: "{{ flannel_digest_checksum|default(None) }}"
     groups:
       - k8s-cluster
+
   flannel_cni:
     enabled: "{{ kube_network_plugin == 'flannel' }}"
     container: true
@@ -261,6 +300,7 @@ downloads:
     sha256: "{{ flannel_cni_digest_checksum|default(None) }}"
     groups:
       - k8s-cluster
+
   calicoctl:
     enabled: "{{ kube_network_plugin == 'calico' or kube_network_plugin == 'canal' }}"
     container: true
@@ -269,6 +309,7 @@ downloads:
     sha256: "{{ calicoctl_digest_checksum|default(None) }}"
     groups:
       - k8s-cluster
+
   calico_node:
     enabled: "{{ kube_network_plugin == 'calico' or kube_network_plugin == 'canal' }}"
     container: true
@@ -277,6 +318,7 @@ downloads:
     sha256: "{{ calico_node_digest_checksum|default(None) }}"
     groups:
       - k8s-cluster
+
   calico_cni:
     enabled: "{{ kube_network_plugin == 'calico' or kube_network_plugin == 'canal' }}"
     container: true
@@ -285,6 +327,7 @@ downloads:
     sha256: "{{ calico_cni_digest_checksum|default(None) }}"
     groups:
       - k8s-cluster
+
   calico_policy:
     enabled: "{{ enable_network_policy or kube_network_plugin == 'canal' }}"
     container: true
@@ -293,6 +336,7 @@ downloads:
     sha256: "{{ calico_policy_digest_checksum|default(None) }}"
     groups:
       - k8s-cluster
+
   calico_rr:
     enabled: "{{ peer_with_calico_rr is defined and peer_with_calico_rr and kube_network_plugin == 'calico' }}"
     container: true
@@ -301,6 +345,7 @@ downloads:
     sha256: "{{ calico_rr_digest_checksum|default(None) }}"
     groups:
       - calico-rr
+
   weave_kube:
     enabled: "{{ kube_network_plugin == 'weave' }}"
     container: true
@@ -309,6 +354,7 @@ downloads:
     sha256: "{{ weave_kube_digest_checksum|default(None) }}"
     groups:
       - k8s-cluster
+
   weave_npc:
     enabled: "{{ kube_network_plugin == 'weave' }}"
     container: true
@@ -317,6 +363,7 @@ downloads:
     sha256: "{{ weave_npc_digest_checksum|default(None) }}"
     groups:
       - k8s-cluster
+
   contiv:
     enabled: "{{ kube_network_plugin == 'contiv' }}"
     container: true
@@ -325,6 +372,7 @@ downloads:
     sha256: "{{ contiv_digest_checksum|default(None) }}"
     groups:
       - k8s-cluster
+
   contiv_auth_proxy:
     enabled: "{{ kube_network_plugin == 'contiv' }}"
     container: true
@@ -333,6 +381,7 @@ downloads:
     sha256: "{{ contiv_auth_proxy_digest_checksum|default(None) }}"
     groups:
       - k8s-cluster
+
   contiv_etcd_init:
     enabled: "{{ kube_network_plugin == 'contiv' }}"
     container: true
@@ -341,6 +390,7 @@ downloads:
     sha256: "{{ contiv_etcd_init_digest_checksum|default(None) }}"
     groups:
       - k8s-cluster
+
   pod_infra:
     enabled: true
     container: true
@@ -349,6 +399,7 @@ downloads:
     sha256: "{{ pod_infra_digest_checksum|default(None) }}"
     groups:
       - k8s-cluster
+
   install_socat:
     enabled: "{{ ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] }}"
     container: true
@@ -357,6 +408,7 @@ downloads:
     sha256: "{{ install_socat_digest_checksum|default(None) }}"
     groups:
       - k8s-cluster
+
   nginx:
     enabled: "{{ loadbalancer_apiserver_localhost }}"
     container: true
@@ -365,6 +417,7 @@ downloads:
     sha256: "{{ nginx_digest_checksum|default(None) }}"
     groups:
       - kube-node
+
   dnsmasq:
     enabled: "{{ dns_mode == 'dnsmasq_kubedns' }}"
     container: true
@@ -373,6 +426,7 @@ downloads:
     sha256: "{{ dnsmasq_digest_checksum|default(None) }}"
     groups:
       - kube-node
+
   kubedns:
     enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}"
     container: true
@@ -381,6 +435,7 @@ downloads:
     sha256: "{{ kubedns_digest_checksum|default(None) }}"
     groups:
       - kube-node
+
   coredns:
     enabled: "{{ dns_mode in ['coredns', 'coredns_dual'] }}"
     container: true
@@ -389,6 +444,7 @@ downloads:
     sha256: "{{ coredns_digest_checksum|default(None) }}"
     groups:
       - kube-node
+
   dnsmasq_nanny:
     enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}"
     container: true
@@ -397,6 +453,7 @@ downloads:
     sha256: "{{ dnsmasq_nanny_digest_checksum|default(None) }}"
     groups:
       - kube-node
+
   dnsmasq_sidecar:
     enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}"
     container: true
@@ -405,6 +462,7 @@ downloads:
     sha256: "{{ dnsmasq_sidecar_digest_checksum|default(None) }}"
     groups:
       - kube-node
+
   kubednsautoscaler:
     enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}"
     container: true
@@ -413,12 +471,14 @@ downloads:
     sha256: "{{ kubednsautoscaler_digest_checksum|default(None) }}"
     groups:
       - kube-node
+
   testbox:
     enabled: false
     container: true
     repo: "{{ test_image_repo }}"
     tag: "{{ test_image_tag }}"
     sha256: "{{ testbox_digest_checksum|default(None) }}"
+
   elasticsearch:
     enabled: "{{ efk_enabled }}"
     container: true
@@ -427,6 +487,7 @@ downloads:
     sha256: "{{ elasticsearch_digest_checksum|default(None) }}"
     groups:
       - kube-node
+
   fluentd:
     enabled: "{{ efk_enabled }}"
     container: true
@@ -435,6 +496,7 @@ downloads:
     sha256: "{{ fluentd_digest_checksum|default(None) }}"
     groups:
       - kube-node
+
   kibana:
     enabled: "{{ efk_enabled }}"
     container: true
@@ -443,6 +505,7 @@ downloads:
     sha256: "{{ kibana_digest_checksum|default(None) }}"
     groups:
       - kube-node
+
   helm:
     enabled: "{{ helm_enabled }}"
     container: true
@@ -451,6 +514,7 @@ downloads:
     sha256: "{{ helm_digest_checksum|default(None) }}"
     groups:
       - kube-node
+
   tiller:
     enabled: "{{ helm_enabled }}"
     container: true
@@ -459,6 +523,7 @@ downloads:
     sha256: "{{ tiller_digest_checksum|default(None) }}"
     groups:
       - kube-node
+
   vault:
     enabled: "{{ cert_management == 'vault' }}"
     container: "{{ vault_deployment_type != 'host' }}"
@@ -475,6 +540,7 @@ downloads:
     version: "{{ vault_version }}"
     groups:
       - vault
+
   registry:
     enabled: "{{ registry_enabled }}"
     container: true
@@ -483,6 +549,7 @@ downloads:
     sha256: "{{ registry_digest_checksum|default(None) }}"
     groups:
       - kube-node
+
   registry_proxy:
     enabled: "{{ registry_enabled }}"
     container: true
@@ -491,6 +558,7 @@ downloads:
     sha256: "{{ registry_proxy_digest_checksum|default(None) }}"
     groups:
       - kube-node
+
   local_volume_provisioner:
     enabled: "{{ local_volume_provisioner_enabled }}"
     container: true
@@ -499,6 +567,7 @@ downloads:
     sha256: "{{ local_volume_provisioner_digest_checksum|default(None) }}"
     groups:
       - kube-node
+
   cephfs_provisioner:
     enabled: "{{ cephfs_provisioner_enabled }}"
     container: true
@@ -507,6 +576,7 @@ downloads:
     sha256: "{{ cephfs_provisioner_digest_checksum|default(None) }}"
     groups:
       - kube-node
+
   ingress_nginx_controller:
     enabled: "{{ ingress_nginx_enabled }}"
     container: true
@@ -515,6 +585,7 @@ downloads:
     sha256: "{{ ingress_nginx_controller_digest_checksum|default(None) }}"
     groups:
       - kube-node
+
   ingress_nginx_default_backend:
     enabled: "{{ ingress_nginx_enabled }}"
     container: true
@@ -523,6 +594,7 @@ downloads:
     sha256: "{{ ingress_nginx_default_backend_digest_checksum|default(None) }}"
     groups:
       - kube-node
+
   cert_manager_controller:
     enabled: "{{ cert_manager_enabled }}"
     container: true

roles/etcd/tasks/install_host.yml

@@ -1,21 +1,25 @@
 ---
-- name: install | Copy etcd binary from download dir
-  shell: |
-    rsync -piu "{{ local_release_dir }}/etcd-{{ etcd_version }}-linux-amd64/etcd" "{{ bin_dir }}/etcd"
-    rsync -piu "{{ local_release_dir }}/etcd-{{ etcd_version }}-linux-amd64/etcdctl" "{{ bin_dir }}/etcdctl"
+- name: install | Copy etcd and etcdctl binary from download dir
+  synchronize:
+    src: "{{ local_release_dir }}/etcd-{{ etcd_version }}-linux-amd64/{{ item }}"
+    dest: "{{ bin_dir }}/{{ item }}"
+    compress: no
+    perms: yes
+    owner: no
+    group: no
   changed_when: false
+  delegate_to: "{{ inventory_hostname }}"
+  with_items:
+    - "etcd"
+    - "etcdctl"
   when: etcd_cluster_setup
 
-- name: install | Set etcd binary permissions
+- name: install | Set etcd and etcdctl binary permissions
   file:
-    path: "{{ bin_dir }}/etcd"
+    path: "{{ bin_dir }}/{{ item }}"
     mode: "0755"
     state: file
+  with_items:
+    - "etcd"
+    - "etcdctl"
   when: etcd_cluster_setup
-
-- name: install | Set etcdctl binary permissions
-  file:
-    path: "{{ bin_dir }}/etcdctl"
-    mode: "0755"
-    state: file
-  when: etcd_cluster_setup

roles/kubernetes-apps/ansible/defaults/main.yml

@@ -1,6 +1,6 @@
 ---
 # Versions
-kubedns_version: 1.14.10
+kubedns_version: 1.14.11
 kubednsautoscaler_version: 1.1.2
 
 # Limits for dnsmasq/kubedns apps

roles/kubernetes-apps/ansible/templates/coredns-svc.yml.j2

@@ -9,14 +9,21 @@ metadata:
     kubernetes.io/cluster-service: "true"
     addonmanager.kubernetes.io/mode: Reconcile
     kubernetes.io/name: "CoreDNS"
+  annotations:
+    prometheus.io/path: /metrics
+    prometheus.io/port: "9153"
+    prometheus.io/scrape: "true"
 spec:
   selector:
     k8s-app: coredns{{ coredns_ordinal_suffix | default('') }}
   clusterIP: {{ clusterIP }}
   ports:
-  - name: dns
-    port: 53
-    protocol: UDP
-  - name: dns-tcp
-    port: 53
-    protocol: TCP
+    - name: dns
+      port: 53
+      protocol: UDP
+    - name: dns-tcp
+      port: 53
+      protocol: TCP
+    - name: metrics
+      port: 9153
+      protocol: TCP

roles/kubernetes-apps/helm/tasks/main.yml

@@ -68,6 +68,7 @@
     {% if tiller_max_history is defined %} --history-max={{ tiller_max_history }}{% endif %}
     {% if tiller_enable_tls %} --tiller-tls --tiller-tls-verify --tiller-tls-cert={{ tiller_tls_cert }} --tiller-tls-key={{ tiller_tls_key }} --tls-ca-cert={{ tiller_tls_ca_cert }} {% endif %}
     {% if tiller_secure_release_info %} --override 'spec.template.spec.containers[0].command'='{/tiller,--storage=secret}' {% endif %}
+    --debug --dry-run
     | kubectl apply -f -
   changed_when: false
   when:

roles/kubernetes-apps/ingress_controller/cert_manager/templates/00-namespace.yml.j2

@@ -5,3 +5,4 @@ metadata:
   name: {{ cert_manager_namespace }}
   labels:
     name: {{ cert_manager_namespace }}
+    certmanager.k8s.io/disable-validation: "true"

roles/kubernetes-apps/ingress_controller/cert_manager/templates/clusterrole-cert-manager.yml.j2

@@ -5,7 +5,7 @@ metadata:
   name: cert-manager
   labels:
     app: cert-manager
-    chart: cert-manager-v0.4.1
+    chart: cert-manager-v0.5.0
     release: cert-manager
     heritage: Tiller
 rules:
@@ -13,12 +13,7 @@ rules:
     resources: ["certificates", "issuers", "clusterissuers"]
     verbs: ["*"]
   - apiGroups: [""]
-    # TODO: remove endpoints once 0.4 is released. We include it here in case
-    # users use the 'master' version of the Helm chart with a 0.2.x release of
-    # cert-manager that still performs leader election with Endpoint resources.
-    # We advise users don't do this, but some will anyway and this will reduce
-    # friction.
-    resources: ["endpoints", "configmaps", "secrets", "events", "services", "pods"]
+    resources: ["configmaps", "secrets", "events", "services", "pods"]
     verbs: ["*"]
   - apiGroups: ["extensions"]
     resources: ["ingresses"]

roles/kubernetes-apps/ingress_controller/cert_manager/templates/clusterrolebinding-cert-manager.yml.j2

@@ -5,7 +5,7 @@ metadata:
   name: cert-manager
   labels:
     app: cert-manager
-    chart: cert-manager-v0.4.1
+    chart: cert-manager-v0.5.0
     release: cert-manager
     heritage: Tiller
 roleRef:

roles/kubernetes-apps/ingress_controller/cert_manager/templates/crd-certificate.yml.j2

@@ -3,9 +3,11 @@ apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
   name: certificates.certmanager.k8s.io
+  annotations:
+    "helm.sh/hook": crd-install
   labels:
     app: cert-manager
-    chart: cert-manager-v0.4.1
+    chart: cert-manager-v0.5.0
     release: cert-manager
     heritage: Tiller
 spec:

roles/kubernetes-apps/ingress_controller/cert_manager/templates/crd-clusterissuer.yml.j2

@@ -3,9 +3,11 @@ apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
   name: clusterissuers.certmanager.k8s.io
+  annotations:
+    "helm.sh/hook": crd-install
   labels:
     app: cert-manager
-    chart: cert-manager-v0.4.1
+    chart: cert-manager-v0.5.0
     release: cert-manager
     heritage: Tiller
 spec:

roles/kubernetes-apps/ingress_controller/cert_manager/templates/crd-issuer.yml.j2

@@ -3,9 +3,11 @@ apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
   name: issuers.certmanager.k8s.io
+  annotations:
+    "helm.sh/hook": crd-install
   labels:
     app: cert-manager
-    chart: cert-manager-v0.4.1
+    chart: cert-manager-v0.5.0
     release: cert-manager
     heritage: Tiller
 spec:

roles/kubernetes-apps/ingress_controller/cert_manager/templates/deploy-cert-manager.yml.j2

@@ -6,7 +6,7 @@ metadata:
   namespace: {{ cert_manager_namespace }}
   labels:
     app: cert-manager
-    chart: cert-manager-v0.4.1
+    chart: cert-manager-v0.5.0
     release: cert-manager
     heritage: Tiller
 spec:

roles/kubernetes-apps/ingress_controller/cert_manager/templates/sa-cert-manager.yml.j2

@@ -6,6 +6,6 @@ metadata:
   namespace: {{ cert_manager_namespace }}
   labels:
     app: cert-manager
-    chart: cert-manager-v0.4.1
+    chart: cert-manager-v0.5.0
     release: cert-manager
     heritage: Tiller

roles/kubernetes-apps/network_plugin/contiv/tasks/configure.yml

@@ -33,6 +33,46 @@
   when: "contiv_global_config.networkInfraType != contiv_fabric_mode"
   run_once: true
 
+- name: Contiv | Set peer hostname
+  set_fact:
+    contiv_peer_hostname: >-
+      {%- if override_system_hostname|default(true) -%}
+      {{ contiv_peer_hostname|default({})|combine({item: hostvars[item]['inventory_hostname']}) }}
+      {%- else -%}
+      {{ contiv_peer_hostname|default({})|combine({item: hostvars[item]['ansible_fqdn']}) }}
+      {%- endif -%}
+  with_items: "{{ groups['k8s-cluster'] }}"
+  run_once: true
+  when:
+    - contiv_fwd_mode == 'routing'
+    - contiv_peer_with_uplink_leaf
+
+- name: Contiv | Get BGP configuration
+  command: |
+    {{ bin_dir }}/netctl --netmaster "http://127.0.0.1:{{ contiv_netmaster_port }}" \
+      bgp ls --json
+  register: bgp_config
+  run_once: true
+  changed_when: false
+  when:
+    - contiv_fwd_mode == 'routing'
+    - contiv_peer_with_uplink_leaf
+
+- name: Contiv | Configure peering with router(s)
+  command: |
+    {{ bin_dir }}/netctl --netmaster "http://127.0.0.1:{{ contiv_netmaster_port }}" \
+      bgp create {{ item.value }} \
+      --router-ip="{{ hostvars[item.key]['contiv']['router_ip'] }}" \
+      --as="{{ hostvars[item.key]['contiv']['as'] | default(contiv_global_as) }}" \
+      --neighbor-as="{{ hostvars[item.key]['contiv']['neighbor_as'] | default(contiv_global_neighbor_as) }}" \
+      --neighbor="{{ hostvars[item.key]['contiv']['neighbor'] }}"
+  run_once: true
+  with_dict: "{{ contiv_peer_hostname }}"
+  when:
+    - contiv_fwd_mode == 'routing'
+    - contiv_peer_with_uplink_leaf
+    - bgp_config.stdout|from_json|length == 0 or not item.value in bgp_config.stdout|from_json|map(attribute='key')|list
+
 - name: Contiv | Get existing networks
   command: |
     {{ bin_dir }}/netctl --netmaster "http://127.0.0.1:{{ contiv_netmaster_port }}" \

roles/kubernetes-apps/network_plugin/contiv/tasks/main.yml

@@ -9,7 +9,6 @@
     filename: "{{ contiv_config_dir }}/{{ item.item.file }}"
     state: "{{ item.changed | ternary('latest','present') }}"
   with_items: "{{ contiv_manifests_results.results }}"
-  delegate_to: "{{ groups['kube-master'][0] }}"
   run_once: true
 
 - import_tasks: configure.yml

roles/kubernetes/master/tasks/main.yml

@@ -10,8 +10,15 @@
   when: kube_encrypt_secret_data
 
 - name: install | Copy kubectl binary from download dir
-  command: rsync -piu "{{ local_release_dir }}/hyperkube" "{{ bin_dir }}/kubectl"
+  synchronize:
+    src: "{{ local_release_dir }}/hyperkube"
+    dest: "{{ bin_dir }}/kubectl"
+    compress: no
+    perms: yes
+    owner: no
+    group: no
   changed_when: false
+  delegate_to: "{{ inventory_hostname }}"
   tags:
     - hyperkube
     - kubectl

roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2

@@ -68,9 +68,18 @@ apiServerExtraArgs:
 {% endif %}
   service-node-port-range: {{ kube_apiserver_node_port_range }}
   kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}"
+  profiling: "{{ kube_profiling }}"
+  repair-malformed-updates: "false"
+  enable-aggregator-routing: "{{ kube_api_aggregator_routing }}"
+{% if kube_api_anonymous_auth is defined and kube_version | version_compare('v1.5', '>=')  %}
+  anonymous-auth: "{{ kube_api_anonymous_auth }}"
+{% endif %}
 {% if kube_basic_auth|default(true) %}
   basic-auth-file: {{ kube_users_dir }}/known_users.csv
 {% endif %}
+{% if kube_token_auth|default(true) %}
+  token-auth-file: {{ kube_token_dir }}/known_tokens.csv
+{% endif %}
 {% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %}
   oidc-issuer-url: {{ kube_oidc_url }}
   oidc-client-id: {{ kube_oidc_client_id }}
@@ -102,19 +111,21 @@ controllerManagerExtraArgs:
   node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
   node-monitor-period: {{ kube_controller_node_monitor_period }}
   pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }}
+  profiling: "{{ kube_profiling }}"
 {% if kube_feature_gates %}
   feature-gates: {{ kube_feature_gates|join(',') }}
 {% endif %}
+{% for key in kube_kubeadm_controller_extra_args %}
+  {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
+{% endfor %}
 {% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %}
 controllerManagerExtraVolumes:
 - name: openstackcacert
   hostPath: "{{ kube_config_dir }}/openstack-cacert.pem"
   mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
 {% endif %}
-{% for key in kube_kubeadm_controller_extra_args %}
-  {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
-{% endfor %}
 schedulerExtraArgs:
+  profiling: "{{ kube_profiling }}"
 {% if kube_feature_gates %}
   feature-gates: {{ kube_feature_gates|join(',') }}
 {% endif %}

roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2

@@ -60,9 +60,18 @@ apiServerExtraArgs:
 {% endif %}
   service-node-port-range: {{ kube_apiserver_node_port_range }}
   kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}"
+  profiling: "{{ kube_profiling }}"
+  repair-malformed-updates: "false"
+  enable-aggregator-routing: "{{ kube_api_aggregator_routing }}"
+{% if kube_api_anonymous_auth is defined and kube_version | version_compare('v1.5', '>=')  %}
+  anonymous-auth: "{{ kube_api_anonymous_auth }}"
+{% endif %}
 {% if kube_basic_auth|default(true) %}
   basic-auth-file: {{ kube_users_dir }}/known_users.csv
 {% endif %}
+{% if kube_token_auth|default(true) %}
+  token-auth-file: {{ kube_token_dir }}/known_tokens.csv
+{% endif %}
 {% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %}
   oidc-issuer-url: {{ kube_oidc_url }}
   oidc-client-id: {{ kube_oidc_client_id }}
@@ -101,9 +110,13 @@ controllerManagerExtraArgs:
   node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
   node-monitor-period: {{ kube_controller_node_monitor_period }}
   pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }}
+  profiling: "{{ kube_profiling }}"
 {% if kube_feature_gates %}
   feature-gates: {{ kube_feature_gates|join(',') }}
 {% endif %}
+{% for key in kube_kubeadm_controller_extra_args %}
+  {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
+{% endfor %}
 {% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %}
 controllerManagerExtraVolumes:
 - name: openstackcacert
@@ -122,10 +135,8 @@ apiServerExtraVolumes:
   writable: true
 {% endif %}
 {% endif %}
-{% for key in kube_kubeadm_controller_extra_args %}
-  {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
-{% endfor %}
 schedulerExtraArgs:
+  profiling: "{{ kube_profiling }}"
 {% if kube_feature_gates %}
   feature-gates: {{ kube_feature_gates|join(',') }}
 {% endif %}
@@ -150,3 +161,7 @@ nodeRegistration:
 {% if container_manager == 'crio' %}
   criSocket: /var/run/crio/crio.sock
 {% endif %}
+{% if dynamic_kubelet_configuration %}
+featureGates:
+  DynamicKubeletConfig: true
+{% endif %}

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -33,7 +33,7 @@ spec:
     - --audit-log-maxage={{ audit_log_maxage }}
     - --audit-log-maxbackup={{ audit_log_maxbackups }}
     - --audit-log-maxsize={{ audit_log_maxsize }}
-    - --audit-policy-file={{ audit_policy_file }} 
+    - --audit-policy-file={{ audit_policy_file }}
 {% endif %}
     - --advertise-address={{ ip | default(ansible_default_ipv4.address) }}
     - --etcd-servers={{ etcd_access_addresses }}
@@ -58,16 +58,16 @@ spec:
     - --admission-control={{ kube_apiserver_admission_control | join(',') }}
 {% else %}
 {% if kube_apiserver_enable_admission_plugins|length > 0 %}
-    - --enable-admission-plugins={{ kube_apiserver_enable_admission_plugins | join(',') }} 
+    - --enable-admission-plugins={{ kube_apiserver_enable_admission_plugins | join(',') }}
 {% endif %}
 {% if kube_apiserver_disable_admission_plugins|length > 0 %}
-    - --disable-admission-plugins={{ kube_apiserver_disable_admission_plugins | join(',') }} 
+    - --disable-admission-plugins={{ kube_apiserver_disable_admission_plugins | join(',') }}
 {% endif %}
 {% endif %}
     - --service-cluster-ip-range={{ kube_service_addresses }}
     - --service-node-port-range={{ kube_apiserver_node_port_range }}
     - --client-ca-file={{ kube_cert_dir }}/ca.pem
-    - --profiling=false
+    - --profiling={{ kube_profiling }}
     - --repair-malformed-updates=false
     - --kubelet-client-certificate={{ kube_cert_dir }}/node-{{ inventory_hostname }}.pem
     - --kubelet-client-key={{ kube_cert_dir }}/node-{{ inventory_hostname }}-key.pem

roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2

@@ -37,7 +37,7 @@ spec:
     - --node-monitor-grace-period={{ kube_controller_node_monitor_grace_period }}
     - --node-monitor-period={{ kube_controller_node_monitor_period }}
     - --pod-eviction-timeout={{ kube_controller_pod_eviction_timeout }}
-    - --profiling=false
+    - --profiling={{ kube_profiling }}
     - --terminated-pod-gc-threshold=12500
     - --v={{ kube_log_level }}
 {% if rbac_enabled %}

roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2

@@ -32,7 +32,7 @@ spec:
     - --use-legacy-policy-config
     - --policy-config-file={{ kube_config_dir }}/kube-scheduler-policy.yaml
 {% endif %}
-    - --profiling=false
+    - --profiling={{ kube_profiling }}
     - --v={{ kube_log_level }}
 {% if kube_feature_gates %}
     - --feature-gates={{ kube_feature_gates|join(',') }}

roles/kubernetes/node/defaults/main.yml

@@ -86,6 +86,9 @@ kubelet_max_pods: 110
 ## Support custom flags to be passed to kubelet
 kubelet_custom_flags: []
 
+## Support custom flags to be passed to kubelet only on nodes, not masters
+kubelet_node_custom_flags: []
+
 # This setting is used for rkt based kubelet for deploying hyperkube
 # from a docker based registry ( controls --insecure and docker:// )
 ## Empty vaule for quay.io containers

roles/kubernetes/node/tasks/install.yml

@@ -7,8 +7,14 @@
     - kubeadm
 
 - name: install | Copy kubeadm binary from download dir
-  command: rsync -piu "{{ local_release_dir }}/kubeadm" "{{ bin_dir }}/kubeadm"
-  changed_when: false
+  synchronize:
+    src: "{{ local_release_dir }}/kubeadm"
+    dest: "{{ bin_dir }}/kubeadm"
+    compress: no
+    perms: yes
+    owner: no
+    group: no
+  delegate_to: "{{ inventory_hostname }}"
   when: kubeadm_enabled
   tags:
     - kubeadm

roles/kubernetes/node/tasks/install_host.yml

@@ -1,11 +1,18 @@
 ---
 
 - name: install | Copy kubelet binary from download dir
-  command: rsync -piu "{{ local_release_dir }}/hyperkube" "{{ bin_dir }}/kubelet"
-  changed_when: false
+  synchronize:
+    src: "{{ local_release_dir }}/hyperkube"
+    dest: "{{ bin_dir }}/kubelet"
+    compress: no
+    perms: yes
+    owner: no
+    group: no
+  delegate_to: "{{ inventory_hostname }}"
   tags:
     - hyperkube
     - upgrade
+  notify: restart kubelet
 
 - name: install | Set kubelet binary permissions
   file:
@@ -15,7 +22,6 @@
   tags:
     - hyperkube
     - upgrade
-  notify: restart kubelet
 
 - name: install | Copy socat wrapper for Container Linux
   command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/opt/bin {{ install_socat_image_repo }}:{{ install_socat_image_tag }}"

roles/kubernetes/node/tasks/main.yml

@@ -32,6 +32,13 @@
   tags:
     - kubelet
 
+- name: Make sure dynamic kubelet configuration directory is writeable
+  file:
+    path: "{{ dynamic_kubelet_configuration_dir }}"
+    mode: 0600
+    state: directory
+  when: dynamic_kubelet_configuration
+
 - name: Write kubelet config file (kubeadm)
   template:
     src: kubelet.kubeadm.env.j2
@@ -70,6 +77,8 @@
 
 - name: Verify if br_netfilter module exists
   shell: "modinfo br_netfilter"
+  environment:
+    PATH: "{{ ansible_env.PATH}}:/sbin"  # Make sure we can workaround RH's conservative path management
   register: modinfo_br_netfilter
   failed_when: modinfo_br_netfilter.rc not in [0, 1]
   changed_when: false

roles/kubernetes/node/templates/kubelet.kubeadm.env.j2

@@ -26,6 +26,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% if kubelet_authorization_mode_webhook %}
 --authorization-mode=Webhook \
 {% endif %}
+--enforce-node-allocatable={{ kubelet_enforce_node_allocatable }} \
 --client-ca-file={{ kube_cert_dir }}/ca.crt \
 --pod-manifest-path={{ kube_manifest_dir }} \
 --cadvisor-port={{ kube_cadvisor_port }} \
@@ -48,6 +49,9 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% else %}
 --fail-swap-on={{ kubelet_fail_swap_on|default(true)}} \
 {% endif %}
+{% if dynamic_kubelet_configuration %}
+--dynamic-config-dir={{ dynamic_kubelet_configuration_dir }} \
+{% endif %}
 --runtime-cgroups={{ kubelet_runtime_cgroups }} --kubelet-cgroups={{ kubelet_kubelet_cgroups }} \
 {% endset %}
 
@@ -90,7 +94,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% endif %}
 {% set all_node_labels = role_node_labels + inventory_node_labels %}
 
-KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kube_reserved }} --node-labels={{ all_node_labels | join(',') }} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}"
+KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kube_reserved }} --node-labels={{ all_node_labels | join(',') }} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
 {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium"] %}
 KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
 {% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %}

roles/kubernetes/node/templates/kubelet.standard.env.j2

@@ -120,7 +120,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {%   endif %}
 {% endif %}
 
-KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ kube_reserved }} --node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}"
+KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ kube_reserved }} --node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
 
 {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium"] %}
 KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"

roles/kubernetes/preinstall/tasks/0020-verify-settings.yml

@@ -127,3 +127,21 @@
   tags:
     - cloud-provider
     - facts
+
+- name: "Get current version of calico cluster version"
+  shell: "{{ bin_dir }}/calicoctl version  | grep 'Cluster Version' | awk '{ print $3}'"
+  register: calico_version_on_server
+  run_once: yes
+  delegate_to: "{{ groups['kube-master'][0] }}"
+
+
+- name: "Check that calico version is enought for upgrade"
+  assert:
+    that:
+      - calico_version_on_server.stdout|version_compare('v2.6.5', '>=')
+    msg: "Your version of calico is not fresh enough for upgrade. Minimum version v2.6.5"
+  when:
+    - 'calico_version_on_server.stdout is defined'
+    - 'calico_version_on_server.stdout != ""'
+    - inventory_hostname == groups['kube-master'][0]
+  run_once: yes

roles/kubespray-defaults/defaults/main.yaml

@@ -136,9 +136,20 @@ kube_apiserver_port: 6443
 kube_apiserver_insecure_bind_address: 127.0.0.1
 kube_apiserver_insecure_port: 8080
 
+# dynamic kubelet configuration
+dynamic_kubelet_configuration: false
+
+# define kubelet config dir for dynamic kubelet
+#kubelet_config_dir:
+default_kubelet_config_dir: "{{ kube_config_dir }}/dynamic_kubelet_dir"
+dynamic_kubelet_configuration_dir: "{{ kubelet_config_dir | default(default_kubelet_config_dir) }}"
+
 # Aggregator
 kube_api_aggregator_routing: false
 
+# Profiling
+kube_profiling: false
+
 # Container for runtime
 container_manager: docker
 
@@ -303,6 +314,11 @@ weave_mode_seed: false
 weave_seed: uninitialized
 weave_peers: uninitialized
 
+# Contiv L3 BGP Mode
+contiv_peer_with_uplink_leaf: false
+contiv_global_as: "65002"
+contiv_global_neighbor_as: "500"
+
 ## Set no_proxy to all assigned cluster IPs and hostnames
 no_proxy: >-
   {%- if http_proxy is defined or https_proxy is defined %}

roles/network_plugin/calico/tasks/check.yml

@@ -10,7 +10,7 @@
   run_once: yes
 
 - name: "Get current version of calico cluster version"
-  shell: "{{ bin_dir }}/calicoctl version  | grep 'Cluster Version' | awk '{ print $3}'"
+  shell: "{{ bin_dir }}/calicoctl version  | grep 'Cluster Version:' | awk '{ print $3}'"
   register: calico_version_on_server
   run_once: yes
   delegate_to: "{{ groups['kube-master'][0] }}"
@@ -22,6 +22,7 @@
         that:
           - calico_version_on_server.stdout|version_compare('v2.6.5', '>=')
         msg: "Your version of calico is not fresh enough for upgrade"
+      when: calico_upgrade_enabled
 
     - name: "Set upgrade flag when version needs to be updated"
       set_fact:

roles/network_plugin/calico/tasks/install.yml

@@ -1,5 +1,4 @@
 ---
-
 - name: Calico | Write Calico cni config
   template:
     src: "cni-calico.conflist.j2"
@@ -103,6 +102,24 @@
   delegate_to: "{{ groups['kube-master'][0] }}"
   when:
     - 'calico_conf.stdout == "0"'
+    - calico_version | version_compare("v3.0.0", ">=")
+
+- name: Calico | Configure calico network pool (legacy)
+  shell: >
+    echo '
+      { "kind": "ipPool",
+        "spec": {"disabled": false, "ipip": {"enabled": {{ ipip }}, "mode": "{{ ipip_mode|lower }}"},
+                 "nat-outgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }}},
+        "apiVersion": "v1",
+        "metadata": {"cidr": "{{ kube_pods_subnet }}"}
+      }' | {{ bin_dir }}/calicoctl apply -f -
+  environment:
+    NO_DEFAULT_POOLS: true
+  run_once: true
+  delegate_to: "{{ groups['kube-master'][0] }}"
+  when:
+    - 'calico_conf.stdout == "0"'
+    - calico_version | version_compare("v3.0.0", "<")
 
 - name: "Determine nodeToNodeMesh needed state"
   set_fact:
@@ -112,7 +129,6 @@
     - inventory_hostname in groups['k8s-cluster']
   run_once: yes
 
-
 - name: Calico | Set global as_num
   shell: >
     echo '
@@ -127,19 +143,33 @@
         "asNumber": {{ global_as_num }} }} ' | {{ bin_dir }}/calicoctl --skip-exists create -f -
   run_once: true
   delegate_to: "{{ groups['kube-master'][0] }}"
+  when:
+    - calico_version | version_compare('v3.0.0', '>=')
+
+- name: Calico | Set global as_num (legacy)
+  command: "{{ bin_dir}}/calicoctl config set asNumber {{ global_as_num }}"
+  run_once: true
+  when:
+    - calico_version | version_compare('v3.0.0', '<')
+
+- name: Calico | Disable node mesh (legacy)
+  command: "{{ bin_dir }}/calicoctl config set nodeToNodeMesh off"
+  run_once: yes
+  when:
+    - calico_version | version_compare('v3.0.0', '<')
+    - nodeToMeshEnabled|default(True)
 
 - name: Calico | Configure peering with router(s)
   shell: >
    echo '{
    "apiVersion": "projectcalico.org/v3",
-   "kind": "bgpPeer",
+   "kind": "BGPPeer",
    "metadata": {
-      "name": "{{ inventory_hostname }}-bgp"
+      "name": "{{ inventory_hostname }}-{{ item.router_id }}"
    },
    "spec": {
       "asNumber": "{{ item.as }}",
       "node": "{{ inventory_hostname }}",
-      "scope": "node",
       "peerIP": "{{ item.router_id }}"
    }}' | {{ bin_dir }}/calicoctl create --skip-exists -f -
   retries: 4
@@ -147,7 +177,7 @@
   with_items:
     - "{{ peers|default([]) }}"
   when:
-    - calico_version_on_server.stdout|version_compare('v3.0.0', '>') or calico_upgrade_enabled
+    - calico_version | version_compare('v3.0.0', '>=')
     - peer_with_router|default(false)
     - inventory_hostname in groups['k8s-cluster']
 
@@ -164,8 +194,7 @@
   delay: "{{ retry_stagger | random + 3 }}"
   with_items: "{{ peers|default([]) }}"
   when:
-    - calico_version_on_server.stdout|version_compare('v3.0.0', '<')
-    - not calico_upgrade_enabled
+    - calico_version | version_compare('v3.0.0', '<')
     - peer_with_router|default(false)
     - inventory_hostname in groups['k8s-cluster']
 
@@ -173,13 +202,12 @@
   shell: >
    echo '{
    "apiVersion": "projectcalico.org/v3",
-   "kind": "bgpPeer",
+   "kind": "BGPPeer",
    "metadata": {
-      "name": "{{ inventory_hostname }}"
+      "name": "{{ inventory_hostname }}-{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(hostvars[item]["ansible_default_ipv4"]["address"]) }}"
    },
    "spec": {
       "asNumber": "{{ local_as | default(global_as_num)}}",
-      "scope": "node",
       "node": "{{ inventory_hostname }}",
       "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(hostvars[item]["ansible_default_ipv4"]["address"]) }}"
    }}' | {{ bin_dir }}/calicoctl create --skip-exists -f -
@@ -188,7 +216,7 @@
   with_items:
     - "{{ groups['calico-rr'] | default([]) }}"
   when:
-    - calico_version_on_server.stdout|version_compare('v3.0.0', '>') or calico_upgrade_enabled
+    - calico_version | version_compare('v3.0.0', '>=')
     - peer_with_calico_rr|default(false)
     - inventory_hostname in groups['k8s-cluster']
     - hostvars[item]['cluster_id'] == cluster_id
@@ -208,7 +236,7 @@
   delay: "{{ retry_stagger | random + 3 }}"
   with_items: "{{ groups['calico-rr'] | default([]) }}"
   when:
-    - calico_version_on_server.stdout|version_compare('v3.0.0', '<')
+    - calico_version | version_compare('v3.0.0', '<')
     - not calico_upgrade_enabled
     - peer_with_calico_rr|default(false)
     - hostvars[item]['cluster_id'] == cluster_id

roles/network_plugin/cilium/templates/cilium-ds.yml.j2

@@ -159,9 +159,15 @@ spec:
               mountPath: /host/opt/cni/bin
             - name: etc-cni-netd
               mountPath: /host/etc/cni/net.d
+{% if container_manager == 'crio' %}
+            - name: crio-socket
+              mountPath: /var/run/crio.sock
+              readOnly: true
+{% else %}
             - name: docker-socket
               mountPath: /var/run/docker.sock
               readOnly: true
+{% endif %}
             - name: etcd-config-path
               mountPath: /var/lib/etcd-config
               readOnly: true
@@ -183,10 +189,17 @@ spec:
         - name: bpf-maps
           hostPath:
             path: /sys/fs/bpf
+{% if container_manager == 'crio' %}
+        # To read crio events from the node
+        - name: crio-socket
+          hostPath:
+            path: /var/run/crio/crio.sock
+{% else %}
         # To read docker events from the node
         - name: docker-socket
           hostPath:
             path: /var/run/docker.sock
+{% endif %}
         # To install cilium cni plugin in the host
         - name: cni-path
           hostPath:

roles/network_plugin/contiv/defaults/main.yml

@@ -6,8 +6,10 @@ contiv_etcd_data_dir: "/var/lib/etcd/contiv-data"
 contiv_netmaster_port: 9999
 contiv_cni_version: 0.1.0
 
+# No need to download it by default, but must be defined
 contiv_etcd_image_repo: "{{ etcd_image_repo }}"
 contiv_etcd_image_tag: "{{ etcd_image_tag }}"
+
 contiv_etcd_listen_port: 6666
 contiv_etcd_peer_port: 6667
 contiv_etcd_endpoints: |-
@@ -26,9 +28,21 @@ contiv_fwd_mode: routing
 # Fabric mode: aci, aci-opflex or default
 contiv_fabric_mode: default
 
+# Defaut netmode: vxlan or vlan
+contiv_net_mode: vxlan
+
 # Dataplane interface
 contiv_vlan_interface: ""
 
+# Default loglevels are INFO
+contiv_netmaster_loglevel: "WARN"
+contiv_netplugin_loglevel: "WARN"
+contiv_ovsdb_server_loglevel: "warn"
+contiv_ovs_vswitchd_loglevel: "warn"
+
+# VxLAN port
+contiv_vxlan_port: 4789
+
 # Default network configuration
 contiv_networks:
   - name: contivh1

roles/network_plugin/contiv/files/contiv-cleanup.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+echo "Starting cleanup"
+ovs-vsctl list-br | grep contiv | xargs -I % ovs-vsctl del-br %
+for p in $(ifconfig | grep vport | awk '{print $1}');
+do
+	ip link delete $p type veth
+done
+touch /tmp/cleanup.done
+sleep 60

roles/network_plugin/contiv/tasks/main.yml

@@ -16,8 +16,25 @@
   with_items:
     - "{{ contiv_etcd_conf_dir }}"
     - "{{ contiv_etcd_data_dir }}"
+  when: inventory_hostname in groups['kube-master']
 
-- set_fact:
+- name: Contiv | Workaround https://github.com/contiv/netplugin/issues/1152
+  set_fact:
+    kube_apiserver_endpoint_for_contiv: |-
+      {% if not is_kube_master and loadbalancer_apiserver_localhost -%}
+      https://localhost:{{ nginx_kube_apiserver_port|default(kube_apiserver_port) }}
+      {%- elif loadbalancer_apiserver is defined and loadbalancer_apiserver.port is defined -%}
+      https://{{ apiserver_loadbalancer_domain_name|default('lb-apiserver.kubernetes.local') }}
+      {%-   if loadbalancer_apiserver.port|string != "443" -%}
+      :{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }}
+      {%-   endif -%}
+      {%- else -%}
+      https://{{ first_kube_master }}:{{ kube_apiserver_port }}
+      {%- endif %}
+  when: inventory_hostname in groups['kube-master']
+
+- name: Contiv | Set necessary facts
+  set_fact:
     contiv_config_dir: "{{ contiv_config_dir }}"
     contiv_enable_api_proxy: "{{ contiv_enable_api_proxy }}"
     contiv_fabric_mode: "{{ contiv_fabric_mode }}"
@@ -26,22 +43,26 @@
     contiv_networks: "{{ contiv_networks }}"
     contiv_manifests:
       - {name: contiv-config, file: contiv-config.yml, type: configmap}
+      - {name: contiv-etcd, file: contiv-etcd.yml, type: daemonset}
+      - {name: contiv-etcd-proxy, file: contiv-etcd-proxy.yml, type: daemonset}
+      - {name: contiv-ovs, file: contiv-ovs.yml, type: daemonset}
       - {name: contiv-netmaster, file: contiv-netmaster-clusterrolebinding.yml, type: clusterrolebinding}
       - {name: contiv-netmaster, file: contiv-netmaster-clusterrole.yml, type: clusterrole}
       - {name: contiv-netmaster, file: contiv-netmaster-serviceaccount.yml, type: serviceaccount}
+      - {name: contiv-netmaster, file: contiv-netmaster.yml, type: daemonset}
       - {name: contiv-netplugin, file: contiv-netplugin-clusterrolebinding.yml, type: clusterrolebinding}
       - {name: contiv-netplugin, file: contiv-netplugin-clusterrole.yml, type: clusterrole}
       - {name: contiv-netplugin, file: contiv-netplugin-serviceaccount.yml, type: serviceaccount}
-      - {name: contiv-etcd, file: contiv-etcd.yml, type: daemonset}
-      - {name: contiv-etcd-proxy, file: contiv-etcd-proxy.yml, type: daemonset}
       - {name: contiv-netplugin, file: contiv-netplugin.yml, type: daemonset}
-      - {name: contiv-netmaster, file: contiv-netmaster.yml, type: daemonset}
+  when: inventory_hostname in groups['kube-master']
 
 - set_fact:
     contiv_manifests: |-
       {% set _ = contiv_manifests.append({"name": "contiv-api-proxy", "file": "contiv-api-proxy.yml", "type": "daemonset"}) %}
       {{ contiv_manifests }}
-  when: contiv_enable_api_proxy
+  when:
+    - contiv_enable_api_proxy
+    - inventory_hostname in groups['kube-master']
 
 - name: Contiv | Create /var/contiv
   file:
@@ -55,21 +76,23 @@
     mode: 0755
     owner: root
     group: root
+  when: inventory_hostname in groups['kube-master']
 
 - name: Contiv | Install all Kubernetes resources
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ contiv_config_dir }}/{{ item.file }}"
   with_items: "{{ contiv_manifests }}"
-  delegate_to: "{{ groups['kube-master'][0] }}"
-  run_once: true
   register: contiv_manifests_results
+  when: inventory_hostname in groups['kube-master']
 
 - name: Contiv | Generate contiv-api-proxy certificates
   script: generate-certificate.sh
   args:
     creates: /var/contiv/auth_proxy_key.pem
-  when: "contiv_enable_api_proxy and contiv_generate_certificate"
+  when:
+    - contiv_enable_api_proxy
+    - contiv_generate_certificate
   delegate_to: "{{ groups['kube-master'][0] }}"
   run_once: true
 
@@ -81,7 +104,9 @@
   with_items:
     - auth_proxy_key.pem
     - auth_proxy_cert.pem
-  when: "contiv_enable_api_proxy and contiv_generate_certificate"
+  when:
+    - contiv_enable_api_proxy
+    - contiv_generate_certificate
   delegate_to: "{{ groups['kube-master'][0] }}"
   run_once: true
 
@@ -92,9 +117,11 @@
   with_items:
     - auth_proxy_key.pem
     - auth_proxy_cert.pem
-  when: "inventory_hostname != groups['kube-master'][0]
-         and inventory_hostname in groups['kube-master']
-         and contiv_enable_api_proxy and contiv_generate_certificate"
+  when:
+    - inventory_hostname != groups['kube-master'][0]
+    - inventory_hostname in groups['kube-master']
+    - contiv_enable_api_proxy
+    - contiv_generate_certificate
 
 - name: Contiv | Copy cni plugins from hyperkube
   command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/bash -c '/bin/cp -fa /opt/cni/bin/* /cnibindir/'"

roles/network_plugin/contiv/tasks/pre-reset.yml

@@ -0,0 +1,66 @@
+---
+- name: reset | Check that kubectl is still here
+  stat:
+    path: "{{ bin_dir }}/kubectl"
+  register: contiv_kubectl
+
+- name: reset | Delete contiv netplugin and netmaster daemonsets
+  kube:
+    name: "{{ item }}"
+    namespace: "kube-system"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "ds"
+    state: absent
+  with_items:
+    - contiv-netplugin
+    - contiv-netmaster
+  register: contiv_cleanup_deletion
+  tags:
+    - network
+  when:
+    - contiv_kubectl.stat.exists
+    - inventory_hostname == groups['kube-master'][0]
+
+- name: reset | Copy contiv temporary cleanup script
+  copy:
+    src: ../files/contiv-cleanup.sh  # Not in role_path so we must trick...
+    dest: /opt/cni/bin/cleanup
+    owner: root
+    group: root
+    mode: 0750
+  when:
+    - contiv_kubectl.stat.exists
+
+- name: reset | Lay down contiv cleanup template
+  template:
+    src: ../templates/contiv-cleanup.yml.j2  # Not in role_path so we must trick...
+    dest: "{{ kube_config_dir }}/contiv-cleanup.yml"  # kube_config_dir is used here as contiv_config_dir is not necessarily set at reset
+  register: contiv_cleanup_manifest
+  when:
+    - contiv_kubectl.stat.exists
+    - inventory_hostname == groups['kube-master'][0]
+
+- name: reset | Start contiv cleanup resources
+  kube:
+    name: "contiv-cleanup"
+    namespace: "kube-system"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "ds"
+    state: latest
+    filename: "{{ kube_config_dir }}/contiv-cleanup.yml"
+  when:
+    - contiv_kubectl.stat.exists
+    - inventory_hostname == groups['kube-master'][0]
+  ignore_errors: true
+
+- name: reset | Wait until contiv cleanup is done
+  command: "{{ bin_dir }}/kubectl -n kube-system get ds contiv-cleanup -o jsonpath='{.status.numberReady}'"
+  register: cleanup_done_all_nodes
+  until: cleanup_done_all_nodes.stdout|int == groups['k8s-cluster']|length
+  retries: 5
+  delay: 5
+  ignore_errors: true
+  changed_when: false
+  when:
+    - contiv_kubectl.stat.exists
+    - inventory_hostname == groups['kube-master'][0]

roles/network_plugin/contiv/tasks/reset.yml

@@ -0,0 +1,9 @@
+---
+- name: reset | check contiv vxlan_sys network device
+  stat:
+    path: "/sys/class/net/vxlan_sys_{{ contiv_vxlan_port | default('4789') }}"
+  register: contiv_vxlan_sys
+
+- name: reset | remove the vxlan_sys network device created by contiv
+  command: "ip link del vxlan_sys_{{ contiv_vxlan_port | default('4789') }}"
+  when: contiv_vxlan_sys.stat.exists

roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2

@@ -35,16 +35,19 @@ spec:
             - --listen-address=0.0.0.0:{{ contiv_api_proxy_port }}
             - --tls-key-file=/var/contiv/auth_proxy_key.pem
             - --tls-certificate=/var/contiv/auth_proxy_cert.pem
+            - --data-store-driver=$(STORE_DRIVER)
             - --data-store-address=$(CONTIV_ETCD)
             - --netmaster-address=127.0.0.1:{{ contiv_netmaster_port }}
           env:
             - name: NO_NETMASTER_STARTUP_CHECK
               value: "0"
+            - name: STORE_DRIVER
+              value: etcd
             - name: CONTIV_ETCD
               valueFrom:
                 configMapKeyRef:
                   name: contiv-config
-                  key: cluster_store
+                  key: contiv_etcd
           securityContext:
             privileged: false
           volumeMounts:

roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2

@@ -0,0 +1,57 @@
+---
+kind: DaemonSet
+apiVersion: extensions/v1beta1
+metadata:
+  name: contiv-cleanup
+  namespace: kube-system
+  labels:
+    k8s-app: contiv-cleanup
+spec:
+  selector:
+    matchLabels:
+      k8s-app: contiv-cleanup
+  template:
+    metadata:
+      labels:
+        k8s-app: contiv-cleanup
+    spec:
+      hostNetwork: true
+      hostPID: true
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      serviceAccountName: contiv-netplugin
+      containers:
+      - name: contiv-ovs-cleanup
+        image: {{ contiv_ovs_image_repo }}:{{ contiv_ovs_image_tag }}
+        command: ["/opt/cni/bin/cleanup"]
+        securityContext:
+          privileged: true
+        volumeMounts:
+         - mountPath: /etc/openvswitch
+           name: etc-openvswitch
+           readOnly: false
+         - mountPath: /var/run
+           name: var-run
+           readOnly: false
+         - mountPath: /opt/cni/bin
+           name: cni-bin-dir
+           readOnly: false
+        readinessProbe:
+          exec:
+            command:
+            - cat
+            - /tmp/cleanup.done
+          initialDelaySeconds: 3
+          periodSeconds: 3
+          successThreshold: 1
+      volumes:
+        - name: etc-openvswitch
+          hostPath:
+            path: /etc/openvswitch
+        - name: var-run
+          hostPath:
+            path: /var/run
+        - name: cni-bin-dir
+          hostPath:
+            path: /opt/cni/bin

roles/network_plugin/contiv/templates/contiv-config.yml.j2

@@ -7,20 +7,22 @@ metadata:
   name: contiv-config
   namespace: kube-system
 data:
-  # The location of your cluster store. This is set to the
-  # avdertise-client value below from the contiv-etcd service.
-  # Change it to an external etcd/consul instance if required.
-  cluster_store: "etcd://127.0.0.1:{{ contiv_etcd_listen_port }}"
-  # The CNI network configuration to install on each node.
-  cni_config: |-
+  contiv_netmaster_loglevel: {{ contiv_netmaster_loglevel }}
+  contiv_netplugin_loglevel: {{ contiv_netplugin_loglevel }}
+  contiv_ovsdb_server_extra_flags: "--verbose={{ contiv_ovsdb_server_loglevel }}"
+  contiv_ovs_vswitchd_extra_flags: "--verbose={{ contiv_ovs_vswitchd_loglevel }}"
+  contiv_fwdmode: {{ contiv_fwd_mode }}
+  contiv_netmode: {{ contiv_net_mode }}
+  contiv_etcd: "http://127.0.0.1:{{ contiv_etcd_listen_port }}"
+  contiv_cni_config: |-
     {
       "cniVersion": "{{ contiv_cni_version }}",
       "name": "contiv-net",
       "type": "contivk8s"
     }
-  config: |-
+  contiv_k8s_config: |-
     {
-       "K8S_API_SERVER": "{{ kube_apiserver_endpoint }}",
+       "K8S_API_SERVER": "{{ kube_apiserver_endpoint_for_contiv }}",
        "K8S_CA": "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
        "K8S_KEY": "",
        "K8S_CERT": "",

roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2

@@ -19,6 +19,8 @@ spec:
     spec:
       hostNetwork: true
       hostPID: true
+      nodeSelector:
+        node-role.kubernetes.io/node: "true"
       containers:
         - name: contiv-etcd-proxy
           image: {{ contiv_etcd_image_repo }}:{{ contiv_etcd_image_tag }}

roles/network_plugin/contiv/templates/contiv-netmaster-clusterrole.yml.j2

@@ -13,6 +13,7 @@ rules:
       - namespaces
       - networkpolicies
     verbs:
+      - get
       - watch
       - list
       - update

roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2

@@ -1,4 +1,4 @@
-# This manifest deploys the Contiv API Server on Kubernetes.
+---
 kind: DaemonSet
 apiVersion: extensions/v1beta1
 metadata:
@@ -31,20 +31,31 @@ spec:
       containers:
         - name: contiv-netmaster
           image: {{ contiv_image_repo }}:{{ contiv_image_tag }}
-          args:
-            - -m
-            - -pkubernetes
           env:
-            - name: CONTIV_ETCD
+            - name: CONTIV_ROLE
+              value: netmaster
+            - name: CONTIV_NETMASTER_MODE
+              value: kubernetes
+            - name: CONTIV_NETMASTER_ETCD_ENDPOINTS
               valueFrom:
                 configMapKeyRef:
                   name: contiv-config
-                  key: cluster_store
-            - name: CONTIV_CONFIG
+                  key: contiv_etcd
+            - name: CONTIV_NETMASTER_FORWARD_MODE
               valueFrom:
                 configMapKeyRef:
                   name: contiv-config
-                  key: config
+                  key: contiv_fwdmode
+            - name: CONTIV_NETMASTER_NET_MODE
+              valueFrom:
+                configMapKeyRef:
+                  name: contiv-config
+                  key: contiv_netmode
+            - name: CONTIV_NETMASTER_LOG_LEVEL
+              valueFrom:
+                configMapKeyRef:
+                  name: contiv-config
+                  key: contiv_netmaster_loglevel
           securityContext:
             privileged: true
           volumeMounts:

roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2

@@ -1,3 +1,4 @@
+---
 # This manifest installs contiv-netplugin container, as well
 # as the Contiv CNI plugins and network config on
 # each master and worker node in a Kubernetes cluster.
@@ -27,73 +28,99 @@ spec:
       - key: node-role.kubernetes.io/master
         effect: NoSchedule
       serviceAccountName: contiv-netplugin
+      initContainers:
+        - name: contiv-netplugin-init
+          image: {{ contiv_init_image_repo }}:{{ contiv_init_image_tag }}
+          env:
+            - name: CONTIV_ROLE
+              value: netplugin
+            - name: CONTIV_MODE
+              value: kubernetes
+            - name: CONTIV_K8S_CONFIG
+              valueFrom:
+                configMapKeyRef:
+                  name: contiv-config
+                  key: contiv_k8s_config
+            - name: CONTIV_CNI_CONFIG
+              valueFrom:
+                configMapKeyRef:
+                  name: contiv-config
+                  key: contiv_cni_config
+          volumeMounts:
+            - mountPath: /var/contiv
+              name: var-contiv
+              readOnly: false
+            - mountPath: /etc/cni/net.d/
+              name: etc-cni-dir
+              readOnly: false
+        - name: contiv-cni
+          image: {{ contiv_image_repo }}:{{ contiv_version }}
+          command: ["cp", "/contiv/bin/contivk8s", "/opt/cni/bin/contivk8s"]
+          volumeMounts:
+            - mountPath: /opt/cni/bin
+              name: cni-bin-dir
+              readOnly: false
       containers:
         # Runs netplugin container on each Kubernetes node. This
         # container programs network policy and routes on each
         # host.
         - name: contiv-netplugin
           image: {{ contiv_image_repo }}:{{ contiv_image_tag }}
-          args:
-            - -pkubernetes
-            - -x
           env:
             - name: VLAN_IF
               value: {{ contiv_vlan_interface }}
-            - name: VTEP_IP
+            - name: CONTIV_NETPLUGIN_VLAN_UPLINKS
+              value: {{ contiv_vlan_interface }}
+            - name: CONTIV_NETPLUGIN_VXLAN_PORT
+              value: "{{ contiv_vxlan_port }}"
+            - name: CONTIV_ROLE
+              value: netplugin
+            - name: CONTIV_NETPLUGIN_MODE
+              value: kubernetes
+            - name: CONTIV_NETPLUGIN_VTEP_IP
               valueFrom:
                  fieldRef:
                     fieldPath: status.podIP
-            - name: CONTIV_ETCD
+            - name: CONTIV_NETPLUGIN_ETCD_ENDPOINTS
               valueFrom:
                 configMapKeyRef:
                   name: contiv-config
-                  key: cluster_store
-            - name: CONTIV_CNI_CONFIG
+                  key: contiv_etcd
+            - name: CONTIV_NETPLUGIN_FORWARD_MODE
               valueFrom:
                 configMapKeyRef:
                   name: contiv-config
-                  key: cni_config
-            - name: CONTIV_CONFIG
+                  key: contiv_fwdmode
+            - name: CONTIV_NETPLUGIN_NET_MODE
               valueFrom:
                 configMapKeyRef:
                   name: contiv-config
-                  key: config
+                  key: contiv_netmode
+            - name: CONTIV_NETPLUGIN_LOG_LEVEL
+              valueFrom:
+                configMapKeyRef:
+                  name: contiv-config
+                  key: contiv_netplugin_loglevel
+          resources:
+            requests:
+              cpu: 250m
           securityContext:
             privileged: true
           volumeMounts:
-            - mountPath: /etc/openvswitch
-              name: etc-openvswitch
-              readOnly: false
-            - mountPath: /lib/modules
-              name: lib-modules
-              readOnly: false
             - mountPath: /var/run
               name: var-run
               readOnly: false
             - mountPath: /var/contiv
               name: var-contiv
               readOnly: false
-            - mountPath: /opt/cni/bin
-              name: cni-bin-dir
-              readOnly: false
-            - mountPath: /etc/cni/net.d/
-              name: etc-cni-dir
-              readOnly: false
       volumes:
         # Used by contiv-netplugin
-        - name: etc-openvswitch
-          hostPath:
-            path: /etc/openvswitch
-        - name: lib-modules
-          hostPath:
-            path: /lib/modules
         - name: var-run
           hostPath:
             path: /var/run
         - name: var-contiv
           hostPath:
             path: /var/contiv
-        # Used to install CNI.
         - name: cni-bin-dir
           hostPath:
             path: /opt/cni/bin

roles/network_plugin/contiv/templates/contiv-ovs.yml.j2

@@ -0,0 +1,80 @@
+---
+apiVersion: apps/v1
+# This manifest deploys the contiv-ovs pod.
+kind: DaemonSet
+apiVersion: extensions/v1beta1
+metadata:
+  name: contiv-ovs
+  namespace: kube-system
+  labels:
+    k8s-app: contiv-ovs
+spec:
+  selector:
+    matchLabels:
+      k8s-app: contiv-ovs
+  template:
+    metadata:
+      labels:
+        k8s-app: contiv-ovs
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      hostNetwork: true
+      hostPID: true
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      containers:
+      # Runs ovs containers on each Kubernetes node.
+      - name: contiv-ovsdb-server
+        image: {{ contiv_ovs_image_repo }}:{{ contiv_ovs_image_tag }}
+        command: ["/scripts/start-ovsdb-server.sh"]
+        securityContext:
+          privileged: false
+        # Won't work until https://github.com/contiv/ovs-docker/pull/4 is merged and image is built again
+        env:
+          - name: OVSDBSERVER_EXTRA_FLAGS
+            valueFrom:
+              configMapKeyRef:
+                name: contiv-config
+                key: contiv_ovsdb_server_extra_flags
+        volumeMounts:
+          - mountPath: /etc/openvswitch
+            name: etc-openvswitch
+            readOnly: false
+          - mountPath: /var/run
+            name: var-run
+            readOnly: false
+      - name: contiv-ovs-vswitchd
+        image: {{ contiv_ovs_image_repo }}:{{ contiv_ovs_image_tag }}
+        command: ["/scripts/start-ovs-vswitchd.sh"]
+        securityContext:
+          privileged: true
+        # Won't work until https://github.com/contiv/ovs-docker/pull/4 is merged and image is built again
+        env:
+          - name: OVSVSWITCHD_EXTRA_FLAGS
+            valueFrom:
+              configMapKeyRef:
+                name: contiv-config
+                key: contiv_ovs_vswitchd_extra_flags
+        volumeMounts:
+         - mountPath: /etc/openvswitch
+           name: etc-openvswitch
+           readOnly: false
+         - mountPath: /lib/modules
+           name: lib-modules
+           readOnly: true
+         - mountPath: /var/run
+           name: var-run
+           readOnly: false
+      volumes:
+        # Used by contiv-ovs
+        - name: etc-openvswitch
+          hostPath:
+            path: /etc/openvswitch
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: var-run
+          hostPath:
+            path: /var/run

roles/network_plugin/weave/templates/weave-net.yml.j2

@@ -42,13 +42,13 @@ items:
           - patch
           - update
       - apiGroups:
-        - policy
+          - policy
         resourceNames:
-        - privileged
+          - privileged
         resources:
-        - podsecuritypolicies
+          - podsecuritypolicies
         verbs:
-        - use
+          - use
   - apiVersion: rbac.authorization.k8s.io/v1beta1
     kind: ClusterRoleBinding
     metadata:

roles/reset/tasks/main.yml

@@ -1,5 +1,12 @@
 ---
 
+- name: reset | include file with pre-reset tasks specific to the network_plugin if exists
+  include_tasks: "{{ (role_path + '/../network_plugin/' + kube_network_plugin + '/tasks/pre-reset.yml') | realpath  }}"
+  when:
+    - kube_network_plugin in ['contiv']
+  tags:
+    - network
+
 - name: reset | stop services
   service:
     name: "{{ item }}"
@@ -150,6 +157,11 @@
     - "{{ bin_dir }}/weave"
     - /var/lib/rkt
     - /etc/vault
+    - /etc/contiv
+    - /var/contiv
+    - /run/contiv
+    - /etc/openvswitch
+    - /run/openvswitch
   ignore_errors: yes
   tags:
     - files
@@ -181,7 +193,7 @@
 - name: reset | include file with reset tasks specific to the network_plugin if exists
   include_tasks: "{{ (role_path + '/../network_plugin/' + kube_network_plugin + '/tasks/reset.yml') | realpath  }}"
   when:
-    - kube_network_plugin in ['flannel', 'cilium']
+    - kube_network_plugin in ['flannel', 'cilium', 'contiv']
   tags:
     - network
 

roles/upgrade/pre-upgrade/defaults/main.yml

@@ -1,5 +1,5 @@
 ---
 drain_grace_period: 300
 drain_timeout: 360s
-drain_label_selector: ""
+drain_pod_selector: ""
 drain_nodes: true

roles/upgrade/pre-upgrade/tasks/main.yml

@@ -32,6 +32,24 @@
   delegate_to: "{{ groups['kube-master'][0] }}"
   when: needs_cordoning
 
+- name: Check kubectl version
+  command: kubectl version --client --short
+  register: kubectl_version
+  delegate_to: "{{ groups['kube-master'][0] }}"
+  run_once: yes
+  when:
+    - drain_nodes
+    - needs_cordoning
+    - 'drain_pod_selector != ""'
+
+- name: Ensure minimum version for drain label selector if necessary
+  assert:
+    that: "kubectl_version.stdout.split(' ')[-1] | version_compare('v1.10.0', '>=')"
+  when:
+    - drain_nodes
+    - needs_cordoning
+    - 'drain_pod_selector != ""'
+
 - name: Drain node
   command: >-
     {{ bin_dir }}/kubectl drain
@@ -40,7 +58,7 @@
     --grace-period {{ drain_grace_period }}
     --timeout {{ drain_timeout }}
     --delete-local-data {{ inventory_hostname }}
-    {% if drain_label_selector != "" %}--selector '{{ drain_label_selector }}'{% endif %}
+    {% if drain_pod_selector != "" %}--pod-selector '{{ drain_pod_selector }}'{% endif %}
   delegate_to: "{{ groups['kube-master'][0] }}"
   when:
     - drain_nodes

roles/win_nodes/kubernetes_patch/tasks/main.yml

@@ -16,11 +16,11 @@
 
     # Due to https://github.com/kubernetes/kubernetes/issues/58212 we cannot rely on exit code for "kubectl patch"
     - name: Check current nodeselector for kube-proxy daemonset
-      shell: kubectl get ds kube-proxy --namespace=kube-system -o jsonpath='{.spec.template.spec.nodeSelector.beta\.kubernetes\.io/os}'
+      shell: "{{bin_dir}}/kubectl get ds kube-proxy --namespace=kube-system -o jsonpath='{.spec.template.spec.nodeSelector.beta.kubernetes.io/os}'"
       register: current_kube_proxy_state
 
     - name: Apply nodeselector patch for kube-proxy daemonset
-      shell: kubectl patch ds kube-proxy --namespace=kube-system --type=strategic -p "$(cat nodeselector-os-linux-patch.json)"
+      shell: "{{bin_dir}}/kubectl patch ds kube-proxy --namespace=kube-system --type=strategic -p \"$(cat nodeselector-os-linux-patch.json)\""
       args:
         chdir: "{{ kubernetes_user_manifests_path }}"
       register: patch_kube_proxy_state