Drop linux capabilities and rework users/groups

* Drop linux capabilities for unprivileged containerized
  worlkoads Kargo configures for deployments.
* Configure required securityContext/user/group/groups for kube
  components' static manifests, etcd, calico-rr and k8s apps,
  like dnsmasq daemonset.
* Rework cloud-init (etcd) users creation for CoreOS.
* Fix nologin paths, adjust defaults for addusers role and ensure
  supplementary groups membership added for users.
* Add netplug user for network plugins (yet unused by privileged
  networking containers though).
* Grant the kube and netplug users read access for etcd certs via
  the etcd certs group.
* Grant group read access to kube certs via the kube cert group.
* Remove priveleged mode for calico-rr and run it under its uid/gid
  and supplementary etcd_cert group.
* Adjust docs.
* Align cpu/memory limits and dropped caps with added rkt support
  for control plane.

Signed-off-by: Bogdan Dobrelya <bogdando@mail.ru>

roles/kubernetes/master/defaults/main.yml

@@ -13,6 +13,21 @@ kube_apiserver_node_port_range: "30000-32767"
 etcd_config_dir: /etc/ssl/etcd
 etcd_cert_dir: "{{ etcd_config_dir }}/ssl"
 
+# Linux capabilities to be dropped for k8s apps ran by container engines
+apps_drop_cap:
+  - chown
+  - dac_override
+  - fowner
+  - fsetid
+  - kill
+  - setgid
+  - setuid
+  - setpcap
+  - sys_chroot
+  - mknod
+  - audit_write
+  - setfcap
+
 # Limits for kube components
 kube_controller_memory_limit: 512M
 kube_controller_cpu_limit: 250m

roles/kubernetes/master/tasks/main.yml

@@ -2,6 +2,9 @@
 - include: pre-upgrade.yml
   tags: k8s-pre-upgrade
 
+- include: set_facts.yml
+  tags: facts
+
 - name: Copy kubectl from hyperkube container
   command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp /hyperkube /systembindir/kubectl"
   register: kube_task_result

roles/kubernetes/master/tasks/set_facts.yml

@@ -0,0 +1,22 @@
+---
+- name: Master | get kube user ID
+  shell: /usr/bin/id -u {{ kubelet_user }} || echo 0
+  register: kube_uid
+
+- name: Master | get kube group ID
+  shell: /usr/bin/getent group {{ kubelet_group }} | cut -d':' -f3 || echo 0
+  register: kube_gid
+
+- name: Master | get kube cert group ID
+  shell: /usr/bin/getent group {{ kube_cert_group }} | cut -d':' -f3 || echo 0
+  register: kube_cert_gid
+
+- name: Master | get etcd cert group ID
+  shell: /usr/bin/getent group {{ etcd_cert_group }} | cut -d':' -f3 || echo 0
+  register: etcd_cert_gid
+
+- set_fact:
+    kubelet_user_id: "{{ kube_uid.stdout }}"
+    kubelet_group_id: "{{ kube_gid.stdout }}"
+    kube_cert_group_id: "{{ kube_cert_gid.stdout }}"
+    etcd_cert_group_id: "{{ etcd_cert_gid.stdout }}"

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -12,6 +12,14 @@ spec:
   - name: kube-apiserver
     image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}
     imagePullPolicy: {{ k8s_image_pull_policy }}
+    securityContext:
+      capabilities:
+        drop:
+{% for c in apps_drop_cap %}
+          - {{ c.upper() }}
+{% endfor %}
+        add:
+          - DAC_OVERRIDE
     resources:
       limits:
         cpu: {{ kube_apiserver_cpu_limit }}

roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2

@@ -11,6 +11,17 @@ spec:
   - name: kube-controller-manager
     image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}
     imagePullPolicy: {{ k8s_image_pull_policy }}
+    securityContext:
+      runAsUser: {{ kubelet_user_id }}
+      fsGroup: {{ kubelet_group_id }}
+      supplementalGroups:
+        - {{ kube_cert_group_id }}
+        - {{ etcd_cert_group_id }}
+      capabilities:
+        drop:
+{% for c in apps_drop_cap %}
+          - {{ c.upper() }}
+{% endfor %}
     resources:
       limits:
         cpu: {{ kube_controller_cpu_limit }}

roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2

@@ -11,6 +11,17 @@ spec:
   - name: kube-scheduler
     image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}
     imagePullPolicy: {{ k8s_image_pull_policy }}
+    securityContext:
+      runAsUser: {{ kubelet_user_id }}
+      fsGroup: {{ kubelet_group_id }}
+      supplementalGroups:
+        - {{ kube_cert_group_id }}
+        - {{ etcd_cert_group_id }}
+      capabilities:
+        drop:
+{% for c in apps_drop_cap %}
+          - {{ c.upper() }}
+{% endfor %}
     resources:
       limits:
         cpu: {{ kube_scheduler_cpu_limit }}