Systemd units, limits, and bin path fixes

* Add restart for weave service unit
* Reuse docker_bin_dir everythere
* Limit systemd managed docker containers by CPU/RAM. Do not configure native
  systemd limits due to the lack of consensus in the kernel community
  requires out-of-tree kernel patches.

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>

roles/network_plugin/calico/defaults/main.yml

@@ -19,3 +19,17 @@ global_as_num: "64512"
 # not be specified in calico CNI config, so Calico will use built-in
 # defaults. The value should be a number, not a string.
 # calico_mtu: 1500
+
+# Limits for apps
+calico_rr_memory_limit: 1000M
+calico_rr_cpu_limit: 300m
+calico_rr_memory_requests: 500M
+calico_rr_cpu_requests: 150m
+calico_node_memory_limit: 500M
+calico_node_cpu_limit: 300m
+calico_node_memory_requests: 256M
+calico_node_cpu_requests: 150m
+calicoctl_memory_limit: 170M
+calicoctl_cpu_limit: 100m
+calicoctl_memory_requests: 70M
+calicoctl_cpu_requests: 50m

roles/network_plugin/calico/rr/templates/calico-rr.service.j2

@@ -5,8 +5,8 @@ Requires=docker.service
 
 [Service]
 EnvironmentFile=/etc/calico/calico-rr.env
-ExecStartPre=-/usr/bin/docker rm -f calico-rr
-ExecStart=/usr/bin/docker run --net=host --privileged \
+ExecStartPre=-{{ docker_bin_dir }}/docker rm -f calico-rr
+ExecStart={{ docker_bin_dir }}/docker run --net=host --privileged \
  --name=calico-rr \
  -e IP=${IP} \
  -e IP6=${IP6} \
@@ -16,12 +16,13 @@ ExecStart=/usr/bin/docker run --net=host --privileged \
  -e ETCD_KEY_FILE=${ETCD_KEY_FILE} \
  -v /var/log/calico-rr:/var/log/calico \
  -v {{ calico_cert_dir }}:{{ calico_cert_dir }}:ro \
+ --memory={{ calico_rr_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ calico_rr_cpu_limit|regex_replace('m', '') }} \
  {{ calico_rr_image_repo }}:{{ calico_rr_image_tag }}
 
 Restart=always
 RestartSec=10s
 
-ExecStop=-/usr/bin/docker stop calico-rr
+ExecStop=-{{ docker_bin_dir }}/docker stop calico-rr
 
 [Install]
 WantedBy=multi-user.target

roles/network_plugin/calico/tasks/main.yml

@@ -41,7 +41,7 @@
   notify: restart calico-node
 
 - name: Calico | Copy cni plugins from hyperkube
-  command: "/usr/bin/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -a /opt/cni/bin/ /cnibindir/"
+  command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -a /opt/cni/bin/ /cnibindir/"
   register: cni_task_result
   until: cni_task_result.rc == 0
   retries: 4
@@ -50,7 +50,7 @@
   tags: [hyperkube, upgrade]
 
 - name: Calico | Copy cni plugins from calico/cni container
-  command: "/usr/bin/docker run --rm -v /opt/cni/bin:/cnibindir {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} sh -c 'cp -a /opt/cni/bin/* /cnibindir/'"
+  command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} sh -c 'cp -a /opt/cni/bin/* /cnibindir/'"
   register: cni_task_result
   until: cni_task_result.rc == 0
   retries: 4

roles/network_plugin/calico/templates/calico-node.service.j2

@@ -5,8 +5,8 @@ Requires=docker.service
 
 [Service]
 EnvironmentFile=/etc/calico/calico.env
-ExecStartPre=-/usr/bin/docker rm -f calico-node
-ExecStart=/usr/bin/docker run --net=host --privileged \
+ExecStartPre=-{{ docker_bin_dir }}/docker rm -f calico-node
+ExecStart={{ docker_bin_dir }}/docker run --net=host --privileged \
  --name=calico-node \
  -e HOSTNAME=${CALICO_HOSTNAME} \
  -e IP=${CALICO_IP} \
@@ -24,12 +24,13 @@ ExecStart=/usr/bin/docker run --net=host --privileged \
  -v /lib/modules:/lib/modules \
  -v /var/run/calico:/var/run/calico \
  -v {{ calico_cert_dir }}:{{ calico_cert_dir }}:ro \
+ --memory={{ calico_node_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ calico_node_cpu_limit|regex_replace('m', '') }} \
  {{ calico_node_image_repo }}:{{ calico_node_image_tag }}
 
 Restart=always
 RestartSec=10s
 
-ExecStop=-/usr/bin/docker stop calico-node
+ExecStop=-{{ docker_bin_dir }}/docker stop calico-node
 
 [Install]
 WantedBy=multi-user.target

roles/network_plugin/calico/templates/calicoctl-container.j2

@@ -1,13 +1,14 @@
 #!/bin/bash
-/usr/bin/docker run -i --privileged --rm \
+{{ docker_bin_dir }}/docker run -i --privileged --rm \
 --net=host --pid=host \
 -e ETCD_ENDPOINTS={{ etcd_access_endpoint }} \
 -e ETCD_CA_CERT_FILE=/etc/calico/certs/ca_cert.crt \
 -e ETCD_CERT_FILE=/etc/calico/certs/cert.crt \
 -e ETCD_KEY_FILE=/etc/calico/certs/key.pem \
--v /usr/bin/docker:/usr/bin/docker \
+-v {{ docker_bin_dir }}/docker:{{ docker_bin_dir }}/docker \
 -v /var/run/docker.sock:/var/run/docker.sock \
 -v /var/run/calico:/var/run/calico \
 -v /etc/calico/certs:/etc/calico/certs:ro \
+--memory={{ calicoctl_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ calicoctl_cpu_limit|regex_replace('m', '') }} \
 {{ calicoctl_image_repo }}:{{ calicoctl_image_tag}} \
 $@

roles/network_plugin/canal/defaults/main.yml

@@ -13,3 +13,13 @@ canal_log_level: "info"
 # Etcd SSL dirs
 canal_cert_dir: /etc/canal/certs
 etcd_cert_dir: /etc/ssl/etcd/ssl
+
+# Limits for apps
+calico_node_memory_limit: 500M
+calico_node_cpu_limit: 200m
+calico_node_memory_requests: 256M
+calico_node_cpu_requests: 100m
+flannel_memory_limit: 500M
+flannel_cpu_limit: 200m
+flannel_memory_requests: 256M
+flannel_cpu_requests: 100m

roles/network_plugin/canal/tasks/main.yml

@@ -43,7 +43,7 @@
     dest: "{{kube_config_dir}}/canal-node.yaml"
 
 - name: Canal | Copy cni plugins from hyperkube
-  command: "/usr/bin/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -a /opt/cni/bin/ /cnibindir/"
+  command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -a /opt/cni/bin/ /cnibindir/"
   register: cni_task_result
   until: cni_task_result.rc == 0
   retries: 4
@@ -52,7 +52,7 @@
   tags: [hyperkube, upgrade]
 
 - name: Canal | Copy cni plugins from calico/cni
-  command: "/usr/bin/docker run --rm -v /opt/cni/bin:/cnibindir {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} sh -c 'cp -a /opt/cni/bin/* /cnibindir/'"
+  command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} sh -c 'cp -a /opt/cni/bin/* /cnibindir/'"
   register: cni_task_result
   until: cni_task_result.rc == 0
   retries: 4

roles/network_plugin/canal/templates/canal-node.yml.j2

@@ -49,6 +49,13 @@ spec:
         - name: flannel
           image: "{{ flannel_image_repo }}:{{ flannel_image_tag }}"
           imagePullPolicy: {{ k8s_image_pull_policy }}
+          resources:
+            limits:
+              cpu: {{ flannel_cpu_limit }}
+              memory: {{ flannel_memory_limit }}
+            requests:
+              cpu: {{ flannel_cpu_requests }}
+              memory: {{ flannel_memory_requests }}
           env:
             # Cluster name
             - name: CLUSTER_NAME
@@ -119,6 +126,13 @@ spec:
         - name: calico-node
           image: "{{ calico_node_image_repo }}:{{ calico_node_image_tag }}"
           imagePullPolicy: {{ k8s_image_pull_policy }}
+          resources:
+            limits:
+              cpu: {{ calico_node_cpu_limit }}
+              memory: {{ calico_node_memory_limit }}
+            requests:
+              cpu: {{ calico_node_cpu_requests }}
+              memory: {{ calico_node_memory_requests }}
           env:
             # The location of the etcd cluster.
             - name: ETCD_ENDPOINTS

roles/network_plugin/cloud/tasks/main.yml

@@ -1,7 +1,7 @@
 ---
 
 - name: Cloud | Copy cni plugins from hyperkube
-  command: "/usr/bin/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -r /opt/cni/bin/. /cnibindir/"
+  command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -r /opt/cni/bin/. /cnibindir/"
   register: cni_task_result
   until: cni_task_result.rc == 0
   retries: 4

roles/network_plugin/flannel/defaults/main.yml

@@ -10,3 +10,9 @@ flannel_public_ip: "{{ access_ip|default(ip|default(ansible_default_ipv4.address
 # You can choose what type of flannel backend to use
 # please refer to flannel's docs : https://github.com/coreos/flannel/blob/master/README.md
 flannel_backend_type: "vxlan"
+
+# Limits for apps
+flannel_memory_limit: 500M
+flannel_cpu_limit: 300m
+flannel_memory_requests: 256M
+flannel_cpu_requests: 150m

roles/network_plugin/flannel/handlers/main.yml

@@ -32,7 +32,7 @@
   pause: seconds=10 prompt="Waiting for docker restart"
 
 - name: Flannel | wait for docker
-  command: /usr/bin/docker images
+  command: "{{ docker_bin_dir }}/docker images"
   register: docker_ready
   retries: 10
   delay: 5

roles/network_plugin/flannel/templates/flannel-pod.yml

@@ -19,6 +19,13 @@
       - name: "flannel-container"
         image: "{{ flannel_image_repo }}:{{ flannel_image_tag }}"
         imagePullPolicy: {{ k8s_image_pull_policy }}
+        resources:
+          limits:
+            cpu: {{ flannel_cpu_limit }}
+            memory: {{ flannel_memory_limit }}
+          requests:
+            cpu: {{ flannel_cpu_requests }}
+            memory: {{ flannel_memory_requests }}
         command:
           - "/bin/sh"
           - "-c"
@@ -26,9 +33,6 @@
         ports:
           - hostPort: 10253
             containerPort: 10253
-        resources:
-          limits:
-            cpu: "100m"
         volumeMounts:
           - name: "subnetenv"
             mountPath: "/run/flannel"

roles/network_plugin/weave/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+# Limits
+weave_memory_limit: 500M
+weave_cpu_limit: 300m

roles/network_plugin/weave/tasks/main.yml

@@ -1,6 +1,6 @@
 ---
 - name: Weave | Copy cni plugins from hyperkube
-  command: "/usr/bin/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -r /opt/cni/bin/. /cnibindir/"
+  command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -r /opt/cni/bin/. /cnibindir/"
   register: cni_task_result
   until: cni_task_result.rc == 0
   retries: 4

roles/network_plugin/weave/templates/weave.j2

@@ -1,3 +1,4 @@
+WEAVE_DOCKER_ARGS="--memory={{ weave_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ weave_cpu_limit|regex_replace('m', '') }}"
 WEAVE_PEERS="{% for host in groups['k8s-cluster'] %}{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}{% if not loop.last %} {% endif %}{% endfor %}"
 WEAVEPROXY_ARGS="--rewrite-inspect --without-dns"
 WEAVE_SUBNET="--ipalloc-range {{ kube_pods_subnet }}"

roles/network_plugin/weave/templates/weave.service.j2

@@ -6,12 +6,13 @@ After=docker.service docker.socket
 
 [Service]
 EnvironmentFile=-/etc/weave.env
-ExecStartPre=-/usr/bin/docker rm -f weave
+ExecStartPre=-{{ docker_bin_dir }}/docker rm -f weave
 ExecStartPre={{ bin_dir }}/weave launch-router \
             $WEAVE_SUBNET \
             $WEAVE_PEERS
-ExecStart=/usr/bin/docker attach weave
+ExecStart={{ docker_bin_dir }}/docker attach weave
 ExecStop={{ bin_dir }}/weave stop
+Restart=on-failure
 
 [Install]
 WantedBy=multi-user.target

roles/network_plugin/weave/templates/weaveproxy.service.j2

@@ -7,11 +7,11 @@ After=docker.service docker.socket
 [Service]
 EnvironmentFile=-/etc/weave.%H.env
 EnvironmentFile=-/etc/weave.env
-ExecStartPre=-/usr/bin/docker rm -f weaveproxy
+ExecStartPre=-{{ docker_bin_dir }}/docker rm -f weaveproxy
 ExecStartPre={{ bin_dir }}/weave launch-proxy $WEAVEPROXY_ARGS
-ExecStart=/usr/bin/docker attach weaveproxy
+ExecStart={{ docker_bin_dir }}/docker attach weaveproxy
 Restart=on-failure
-ExecStop=/opt/bin/weave stop-proxy
+ExecStop={{ bin_dir }}/weave stop-proxy
 
 [Install]
 WantedBy=weave-network.target