All CNIs: support ANY toleration. (#3391)

Before, Nodes tainted with NoExecute policy did not have calico/weave Pod.
Network pod should run on all nodes whatever happens on a specific node.

Also always set the Pods to be critical.
Also remove deprecated scheduler.alpha.kubernetes.io/tolerations annotations.

roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2

@@ -16,6 +16,7 @@ spec:
       labels:
         k8s-app: contiv-api-proxy
       annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
 {% if kube_version|version_compare('v1.11.1', '>=') %}
@@ -28,8 +29,10 @@ spec:
       nodeSelector:
         node-role.kubernetes.io/master: "true"
       tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
+        - operator: Exists
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
       serviceAccountName: contiv-netmaster
       containers:
         - name: contiv-api-proxy

roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2

@@ -14,6 +14,9 @@ spec:
     metadata:
       labels:
         k8s-app: contiv-cleanup
+      annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
 {% if kube_version|version_compare('v1.11.1', '>=') %}
       priorityClassName: system-node-critical
@@ -21,8 +24,10 @@ spec:
       hostNetwork: true
       hostPID: true
       tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
+        - operator: Exists
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
       serviceAccountName: contiv-netplugin
       containers:
       - name: contiv-ovs-cleanup

roles/network_plugin/contiv/templates/contiv-etcd.yml.j2

@@ -25,8 +25,10 @@ spec:
       nodeSelector:
         node-role.kubernetes.io/master: "true"
       tolerations:
-        - key: node-role.kubernetes.io/master
-          effect: NoSchedule
+        - operator: Exists
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
       initContainers:
         - name: contiv-etcd-init
           image: {{ contiv_etcd_init_image_repo }}:{{ contiv_etcd_init_image_tag }}

roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2

@@ -16,6 +16,7 @@ spec:
       labels:
         k8s-app: contiv-netmaster
       annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
 {% if kube_version|version_compare('v1.11.1', '>=') %}
@@ -28,8 +29,10 @@ spec:
       nodeSelector:
         node-role.kubernetes.io/master: "true"
       tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
+        - operator: Exists
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
       serviceAccountName: contiv-netmaster
       containers:
         - name: contiv-netmaster

roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2

@@ -20,6 +20,7 @@ spec:
       labels:
         k8s-app: contiv-netplugin
       annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
 {% if kube_version|version_compare('v1.11.1', '>=') %}
@@ -28,8 +29,10 @@ spec:
       hostNetwork: true
       hostPID: true
       tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
+        - operator: Exists
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
       serviceAccountName: contiv-netplugin
       initContainers:
         - name: contiv-netplugin-init

roles/network_plugin/contiv/templates/contiv-ovs.yml.j2

@@ -17,6 +17,7 @@ spec:
       labels:
         k8s-app: contiv-ovs
       annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
 {% if kube_version|version_compare('v1.11.1', '>=') %}
@@ -25,8 +26,10 @@ spec:
       hostNetwork: true
       hostPID: true
       tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
+        - operator: Exists
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
       containers:
       # Runs ovs containers on each Kubernetes node.
       - name: contiv-ovsdb-server