[jjo] add kube-router support (#3339)

* [jjo] add kube-router support

Fixes cloudnativelabs/kube-router#147.

* add kube-router as another network_plugin choice
* support most used kube-router flags via
  `kube_router_foo` vars as other plugins
* implement replacing kube-proxy (--run-service-proxy=true) via
  `kube_proxy_mode: none`, verified in a _non kubeadm_enabled_
  install, should also work for recent kubeadm releases via
  `skipKubeProxyInstall: true` config

* [jjo] address PR#3339 review from @woopstar

* add busybox image used by kube-router to downloads

* fix busybox download groups key

* rework kubeadm_enabled + kube_router_run_service_proxy

- verify it working ok w/the kubeadm_enabled and
  kube_router_run_service_proxy true or false

- introduce `kube_proxy_remove` fact, to decouple logic
  from kube_proxy_mode (which affects kubeadm configmap
  settings, thus no-good to ab-use it to 'none')

* improve kube-router.md re: kubeadm_enabled and kube_router_run_service_proxy

* address @woopstar latest review

* add inventory/sample/group_vars/k8s-cluster/k8s-net-kube-router.yml

* fix kube_router_run_service_proxy conditional for kube-proxy removal

* fix kube_proxy_remove fact (w/ |bool), add some needed kube-proxy tags on my and existing changes

* update kube-router tolerations for 1.12 compatibility

* add PriorityClass to kube-router DaemonSet

roles/download/defaults/main.yml

@@ -55,6 +55,7 @@ weave_version: "2.4.1"
 pod_infra_version: 3.1
 contiv_version: 1.2.1
 cilium_version: "v1.2.0"
+kube_router_version: "v0.2.0"
 
 # Download URLs
 kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/{{ image_arch }}/kubeadm"
@@ -152,6 +153,8 @@ contiv_ovs_image_repo: "contiv/ovs"
 contiv_ovs_image_tag: "latest"
 cilium_image_repo: "docker.io/cilium/cilium"
 cilium_image_tag: "{{ cilium_version }}"
+kube_router_image_repo: "cloudnativelabs/kube-router"
+kube_router_image_tag: "{{ kube_router_version }}"
 nginx_image_repo: nginx
 nginx_image_tag: 1.13
 dnsmasq_version: 2.78
@@ -178,6 +181,8 @@ kubednsautoscaler_image_repo: "gcr.io/google_containers/cluster-proportional-aut
 kubednsautoscaler_image_tag: "{{ kubednsautoscaler_version }}"
 test_image_repo: busybox
 test_image_tag: latest
+busybox_image_repo: busybox
+busybox_image_tag: 1.29.2
 helm_version: "v2.9.1"
 helm_image_repo: "lachlanevenson/k8s-helm"
 helm_image_tag: "{{ helm_version }}"
@@ -391,6 +396,15 @@ downloads:
     groups:
       - k8s-cluster
 
+  kube_router:
+    enabled: "{{ kube_network_plugin == 'kube-router' }}"
+    container: true
+    repo: "{{ kube_router_image_repo }}"
+    tag: "{{ kube_router_image_tag }}"
+    sha256: "{{ kube_router_digest_checksum|default(None) }}"
+    groups:
+      - k8s-cluster
+
   pod_infra:
     enabled: true
     container: true
@@ -472,6 +486,15 @@ downloads:
     groups:
       - kube-node
 
+  busybox:
+    enabled: "{{ kube_network_plugin in ['kube-router'] }}"
+    container: true
+    repo: "{{ busybox_image_repo }}"
+    tag: "{{ busybox_image_tag }}"
+    sha256: "{{ busybox_digest_checksum|default(None) }}"
+    groups:
+      - k8s-cluster
+
   testbox:
     enabled: false
     container: true

roles/kubernetes-apps/network_plugin/kube-router/tasks/main.yml

@@ -0,0 +1,22 @@
+---
+
+- name: kube-router | Start Resources
+  kube:
+    name: "kube-router"
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/kube-router.yml"
+    resource: "ds"
+    namespace: "kube-system"
+    state: "latest"
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+
+- name: kube-router | Wait for kube-router pods to be ready
+  command: "{{bin_dir}}/kubectl -n kube-system get pods -l k8s-app=kube-router -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'"
+  register: pods_not_ready
+  until: pods_not_ready.stdout.find("kube-router")==-1
+  retries: 30
+  delay: 10
+  ignore_errors: yes
+  when:
+    - inventory_hostname == groups['kube-master'][0]

roles/kubernetes-apps/network_plugin/meta/main.yml

@@ -29,3 +29,8 @@ dependencies:
     when: kube_network_plugin == 'weave'
     tags:
       - weave
+
+  - role: kubernetes-apps/network_plugin/kube-router
+    when: kube_network_plugin == 'kube-router'
+    tags:
+      - kube-router

roles/kubernetes/kubeadm/tasks/main.yml

@@ -96,6 +96,9 @@
     - kubeadm_config_api_fqdn is not defined
     - is_kube_master
     - kubeadm_discovery_address != kube_apiserver_endpoint
+    - not kube_proxy_remove
+  tags:
+    - kube-proxy
 
 # FIXME(mattymo): Reconcile kubelet kubeconfig filename for both deploy modes
 - name: Symlink kubelet kubeconfig for calico/canal
@@ -114,3 +117,19 @@
     - kubeadm_config_api_fqdn is not defined
     - is_kube_master
     - kubeadm_discovery_address != kube_apiserver_endpoint
+    - not kube_proxy_remove
+  tags:
+    - kube-proxy
+
+# FIXME(jjo): need to post-remove kube-proxy until https://github.com/kubernetes/kubeadm/issues/776
+# is fixed
+- name: Delete kube-proxy daemonset if kube_proxy_remove set, e.g. kube_network_plugin providing proxy services
+  shell: "{{ bin_dir }}/kubectl delete daemonset -n kube-system kube-proxy"
+  delegate_to: "{{groups['kube-master']|first}}"
+  run_once: true
+  when:
+    - kube_proxy_remove
+    - is_kube_master
+    - kubeadm_discovery_address != kube_apiserver_endpoint
+  tags:
+    - kube-proxy

roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2

@@ -55,7 +55,7 @@ spec:
 {% if kube_network_plugin is defined and kube_network_plugin == 'cloud' %}
     - --configure-cloud-routes=true
 {% endif %}
-{% if kube_network_plugin is defined and kube_network_plugin in ["cloud", "flannel", "canal", "cilium"] %}
+{% if kube_network_plugin is defined and kube_network_plugin in ["cloud", "flannel", "canal", "cilium", "kube-router"] %}
     - --allocate-node-cidrs=true
     - --cluster-cidr={{ kube_pods_subnet }}
     - --service-cluster-ip-range={{ kube_service_addresses }}

roles/kubernetes/node/tasks/main.yml

@@ -146,15 +146,26 @@
   template:
     src: manifests/kube-proxy.manifest.j2
     dest: "{{ kube_manifest_dir }}/kube-proxy.manifest"
-  when: not kubeadm_enabled
+  when:
+    - not (kubeadm_enabled or kube_proxy_remove)
   tags:
     - kube-proxy
 
-- name: Purge proxy manifest for kubeadm
+- name: Purge proxy manifest for kubeadm or if proxy services being provided by other means, e.g. network_plugin
   file:
     path: "{{ kube_manifest_dir }}/kube-proxy.manifest"
     state: absent
-  when: kubeadm_enabled
+  when:
+    - kubeadm_enabled or kube_proxy_remove
+  tags:
+    - kube-proxy
+
+- name: Cleanup kube-proxy leftovers from node
+  command: "{{ docker_bin_dir }}/docker run --rm --privileged -v /lib/modules:/lib/modules --net=host {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} kube-proxy --cleanup"
+  when:
+    - kube_proxy_remove
+  # `kube-proxy --cleanup`, being Ok as per shown WARNING, still returns 255 from above run (?)
+  ignore_errors: true
   tags:
     - kube-proxy
 

roles/kubernetes/node/templates/kubelet.kubeadm.env.j2

@@ -97,7 +97,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% set all_node_labels = role_node_labels + inventory_node_labels %}
 
 KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kube_reserved }} --node-labels={{ all_node_labels | join(',') }} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
-{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium"] %}
+{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium", "kube-router"] %}
 KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
 {% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %}
 KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet"

roles/kubernetes/node/templates/kubelet.rkt.service.j2

@@ -33,7 +33,7 @@ ExecStart=/usr/bin/rkt run \
         --volume var-lib-docker,kind=host,source={{ docker_daemon_graph }},readOnly=false \
         --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,readOnly=false,recursive=true \
         --volume var-log,kind=host,source=/var/log \
-{% if kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium"] %}
+{% if kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium", "kube-router"] %}
         --volume etc-cni,kind=host,source=/etc/cni,readOnly=true \
         --volume opt-cni,kind=host,source=/opt/cni,readOnly=true \
         --volume var-lib-cni,kind=host,source=/var/lib/cni,readOnly=false \

roles/kubernetes/node/templates/kubelet.standard.env.j2

@@ -124,7 +124,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 
 KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ kube_reserved }} --node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
 
-{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium"] %}
+{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium", "kube-router"] %}
 KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
 {% elif kube_network_plugin is defined and kube_network_plugin == "weave" %}
 DOCKER_SOCKET="--docker-endpoint=unix:/var/run/weave/weave.sock"

roles/kubernetes/preinstall/tasks/0020-verify-settings.yml

@@ -26,7 +26,7 @@
 
 - name: Stop if unknown network plugin
   assert:
-    that: kube_network_plugin in ['calico', 'canal', 'flannel', 'weave', 'cloud', 'cilium', 'contiv']
+    that: kube_network_plugin in ['calico', 'canal', 'flannel', 'weave', 'cloud', 'cilium', 'contiv', 'kube-router']
   when: kube_network_plugin is defined
   ignore_errors: "{{ ignore_assert_errors }}"
 

roles/kubernetes/preinstall/tasks/0040-set_facts.yml

@@ -158,3 +158,20 @@
       paths:
         - ../vars
       skip: true
+
+- name: force kube_proxy removal if proxy services are replaced by other means
+  set_fact:
+    kube_proxy_remove: "{{ (kube_network_plugin == 'kube-router') and (kube_router_run_service_proxy is defined and kube_router_run_service_proxy)| bool }}"
+  tags:
+    - facts
+    - kube-proxy
+
+- name: override kube_proxy_mode to ipvs if kube_proxy_remove is set, as ipvs won't require kube-proxy cleanup when kube-proxy daemonset gets deleted
+  set_fact:
+    kube_proxy_mode: 'ipvs'
+  when:
+    - kubeadm_enabled
+    - kube_proxy_remove
+  tags:
+    - facts
+    - kube-proxy

roles/kubernetes/preinstall/tasks/0050-create_directories.yml

@@ -33,7 +33,7 @@
     - "/opt/cni/bin"
     - "/var/lib/calico"
   when:
-    - kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium"]
+    - kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium", "kube-router"]
     - inventory_hostname in groups['k8s-cluster']
   tags:
     - network
@@ -42,6 +42,7 @@
     - weave
     - canal
     - contiv
+    - kube-router
     - bootstrap-os
 
 - name: Create local volume provisioner directories

roles/kubernetes/secrets/tasks/main.yml

@@ -113,7 +113,11 @@
   with_items:
     - "node-{{ inventory_hostname }}.pem"
     - "kube-proxy-{{ inventory_hostname }}.pem"
-  when: inventory_hostname in groups['k8s-cluster']
+  when:
+    - inventory_hostname in groups['k8s-cluster']
+  tags:
+    - node
+    - kube-proxy
 
 - name: "Gen_certs | set kube node certificate serial facts"
   set_fact:
@@ -123,6 +127,7 @@
   tags:
     - kubelet
     - node
+    - kube-proxy
 
 - import_tasks: gen_tokens.yml
   tags:

roles/network_plugin/kube-router/defaults/main.yml

@@ -0,0 +1,36 @@
+---
+# Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP
+kube_router_run_router: true
+
+# Enables Network Policy -- sets up iptables to provide ingress firewall for pods
+kube_router_run_firewall: true
+
+# Enables Service Proxy -- sets up IPVS for Kubernetes Services
+# see docs/kube-router.md "Caveats" section
+kube_router_run_service_proxy: false
+
+# Add Cluster IP of the service to the RIB so that it gets advertises to the BGP peers.
+kube_router_advertise_cluster_ip: false
+
+# Add External IP of service to the RIB so that it gets advertised to the BGP peers.
+kube_router_advertise_external_ip: false
+
+# Add LoadbBalancer IP of service status as set by the LB provider to the RIB so that it gets advertised to the BGP peers.
+kube_router_advertise_loadbalancer_ip: false
+
+# Array of arbitrary extra arguments to kube-router, see
+# https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md
+kube_router_extra_args: []
+
+# ASN numbers of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr.
+kube_router_peer_router_asns: ~
+
+# The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's.
+kube_router_peer_router_ips: ~
+
+# The remote port of the external BGP to which all nodes will peer. If not set, default BGP port (179) will be used.
+kube_router_peer_router_ports: ~
+
+# Setups node CNI to allow hairpin mode, requires node reboots, see
+# https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md#hairpin-mode
+kube_router_support_hairpin_mode: false

roles/network_plugin/kube-router/tasks/main.yml

@@ -0,0 +1,17 @@
+---
+
+- name: kube-router | Copy cni plugins from hyperkube
+  command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -prf /opt/cni/bin/. /cnibindir/"
+  register: cni_task_result
+  until: cni_task_result.rc == 0
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  changed_when: false
+  tags:
+    - hyperkube
+    - upgrade
+
+- name: kube-router | Create manifest
+  template:
+    src: kube-router.yml.j2
+    dest: "{{ kube_config_dir }}/kube-router.yml"

roles/network_plugin/kube-router/tasks/reset.yml

@@ -0,0 +1,9 @@
+---
+- name: reset | check kube-dummy-if network device
+  stat:
+    path: /sys/class/net/kube-dummy-if
+  register: kube_dummy_if
+
+- name: reset | remove the network device created by kube-router
+  command: ip link del kube-dummy-if
+  when: kube_dummy_if.stat.exists

roles/network_plugin/kube-router/templates/kube-router.yml.j2

@@ -0,0 +1,225 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kube-router-cfg
+  namespace: kube-system
+  labels:
+    tier: node
+    k8s-app: kube-router
+data:
+  cni-conf.json: |
+    {
+      "name":"kubernetes",
+      "type":"bridge",
+      "bridge":"kube-bridge",
+      "isDefaultGateway":true,
+{% if kube_router_support_hairpin_mode %}
+      "hairpinMode":true,
+{% endif %}
+      "ipam": {
+        "type":"host-local"
+      }
+    }
+  kubeconfig: |
+    apiVersion: v1
+    kind: Config
+    clusterCIDR: {{ kube_pods_subnet }}
+    clusters:
+    - name: cluster
+      cluster:
+        certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+        server: {{ kube_apiserver_endpoint }}
+    users:
+    - name: kube-router
+      user:
+        tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    contexts:
+    - context:
+        cluster: cluster
+        user: kube-router
+      name: kube-router-context
+    current-context: kube-router-context
+
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  labels:
+    k8s-app: kube-router
+    tier: node
+  name: kube-router
+  namespace: kube-system
+spec:
+  minReadySeconds: 3
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-router
+        tier: node
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-cluster-critical
+{% endif %}
+      serviceAccountName: kube-router
+      containers:
+      - name: kube-router
+        image: {{ kube_router_image_repo }}:{{ kube_router_image_tag }}
+        imagePullPolicy: IfNotPresent
+        args:
+        - --run-router={{ kube_router_run_router | bool }}
+        - --run-firewall={{ kube_router_run_firewall | bool }}
+        - --run-service-proxy={{ kube_router_run_service_proxy | bool }}
+        - --kubeconfig=/var/lib/kube-router/kubeconfig
+{% if kube_router_advertise_cluster_ip %}
+        - --advertise-cluster-ip
+{% endif %}
+{% if kube_router_advertise_external_ip %}
+        - --advertise-external-ip
+{% endif %}
+{% if kube_router_advertise_loadbalancer_ip %}
+        - --advertise-loadbalancer-ip
+{% endif %}
+{% if kube_router_peer_router_asns %}
+        - --peer-router-asns {{ kube_router_peer_router_asns }}
+{% endif %}
+{% if kube_router_peer_router_ips %}
+        - --peer-router-ips {{ kube_router_peer_router_ips }}
+{% endif %}
+{% if kube_router_peer_router_ports %}
+        - --peer-router-ports {{ kube_router_peer_router_ports }}
+{% endif %}
+{% for arg in kube_router_extra_args %}
+        - "{{ arg }}"
+{% endfor %}
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 20244
+          initialDelaySeconds: 10
+          periodSeconds: 3
+        resources:
+          requests:
+            cpu: 250m
+            memory: 250Mi
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: cni-conf-dir
+          mountPath: /etc/cni/net.d
+        - name: kubeconfig
+          mountPath: /var/lib/kube-router
+          readOnly: true
+      initContainers:
+      - name: install-cni
+        image: {{ busybox_image_repo }}:{{ busybox_image_tag }}
+        imagePullPolicy: IfNotPresent
+        command:
+        - /bin/sh
+        - -c
+        - set -e -x;
+          if [ ! -f /etc/cni/net.d/10-kuberouter.conf ]; then
+            TMP=/etc/cni/net.d/.tmp-kuberouter-cfg;
+            cp /etc/kube-router/cni-conf.json ${TMP};
+            mv ${TMP} /etc/cni/net.d/10-kuberouter.conf;
+          fi;
+          if [ ! -f /var/lib/kube-router/kubeconfig ]; then
+            TMP=/var/lib/kube-router/.tmp-kubeconfig;
+            cp /etc/kube-router/kubeconfig ${TMP};
+            mv ${TMP} /var/lib/kube-router/kubeconfig;
+          fi
+        volumeMounts:
+        - mountPath: /etc/cni/net.d
+          name: cni-conf-dir
+        - mountPath: /etc/kube-router
+          name: kube-router-cfg
+        - name: kubeconfig
+          mountPath: /var/lib/kube-router
+      hostNetwork: true
+      tolerations:
+      - operator: Exists
+      # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+      - key: CriticalAddonsOnly
+        operator: "Exists"
+      volumes:
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: cni-conf-dir
+        hostPath:
+          path: /etc/cni/net.d
+      - name: kube-router-cfg
+        configMap:
+          name: kube-router-cfg
+      - name: kubeconfig
+        hostPath:
+          path: /var/lib/kube-router
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-router
+  namespace: kube-system
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: kube-router
+  namespace: kube-system
+rules:
+  - apiGroups:
+    - ""
+    resources:
+      - namespaces
+      - pods
+      - services
+      - nodes
+      - endpoints
+    verbs:
+      - list
+      - get
+      - watch
+  - apiGroups:
+    - "networking.k8s.io"
+    resources:
+      - networkpolicies
+    verbs:
+      - list
+      - get
+      - watch
+  - apiGroups:
+    - extensions
+    resources:
+      - networkpolicies
+    verbs:
+      - get
+      - list
+      - watch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: kube-router
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-router
+subjects:
+- kind: ServiceAccount
+  name: kube-router
+  namespace: kube-system

roles/network_plugin/meta/main.yml

@@ -30,5 +30,10 @@ dependencies:
     tags:
       - contiv
 
+  - role: network_plugin/kube-router
+    when: kube_network_plugin == 'kube-router'
+    tags:
+      - kube-router
+
   - role: network_plugin/cloud
     when: kube_network_plugin == 'cloud'

roles/reset/tasks/main.yml

@@ -165,6 +165,7 @@
     - /run/contiv
     - /etc/openvswitch
     - /run/openvswitch
+    - /var/lib/kube-router
   ignore_errors: yes
   tags:
     - files
@@ -196,7 +197,7 @@
 - name: reset | include file with reset tasks specific to the network_plugin if exists
   include_tasks: "{{ (role_path + '/../network_plugin/' + kube_network_plugin + '/tasks/reset.yml') | realpath  }}"
   when:
-    - kube_network_plugin in ['flannel', 'cilium', 'contiv']
+    - kube_network_plugin in ['flannel', 'cilium', 'contiv', 'kube-router']
   tags:
     - network
 

roles/win_nodes/kubernetes_patch/tasks/main.yml

@@ -32,3 +32,5 @@
     - debug: msg={{ patch_kube_proxy_state.stderr_lines }}
       when: patch_kube_proxy_state is not skipped
   tags: init
+  when:
+    - not kube_proxy_remove