Drop linux capabilities and rework users/groups

* Drop linux capabilities for unprivileged containerized
  worlkoads Kargo configures for deployments.
* Configure required securityContext/user/group/groups for kube
  components' static manifests, etcd, calico-rr and k8s apps,
  like dnsmasq daemonset.
* Rework cloud-init (etcd) users creation for CoreOS.
* Fix nologin paths, adjust defaults for addusers role and ensure
  supplementary groups membership added for users.
* Add netplug user for network plugins (yet unused by privileged
  networking containers though).
* Grant the kube and netplug users read access for etcd certs via
  the etcd certs group.
* Grant group read access to kube certs via the kube cert group.
* Remove priveleged mode for calico-rr and run it under its uid/gid
  and supplementary etcd_cert group.
* Adjust docs.
* Align cpu/memory limits and dropped caps with added rkt support
  for control plane.

Signed-off-by: Bogdan Dobrelya <bogdando@mail.ru>

roles/kubernetes-apps/ansible/defaults/main.yml

@@ -51,3 +51,18 @@ netchecker_kubectl_memory_requests: 64M
 etcd_cert_dir: "/etc/ssl/etcd/ssl"
 calico_cert_dir: "/etc/calico/certs"
 canal_cert_dir: "/etc/canal/certs"
+
+# Linux capabilities to be dropped for k8s apps ran by container engines
+apps_drop_cap:
+  - chown
+  - dac_override
+  - fowner
+  - fsetid
+  - kill
+  - setgid
+  - setuid
+  - setpcap
+  - sys_chroot
+  - mknod
+  - audit_write
+  - setfcap

roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2

@@ -25,6 +25,12 @@ spec:
         - name: calico-policy-controller
           image: {{ calico_policy_image_repo }}:{{ calico_policy_image_tag }}
           imagePullPolicy: {{ k8s_image_pull_policy }}
+          securityContext:
+            capabilities:
+              drop:
+{% for c in apps_drop_cap %}
+                - {{ c.upper() }}
+{% endfor %}
           resources:
             limits:
               cpu: {{ calico_policy_controller_cpu_limit }}

roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml

@@ -23,6 +23,12 @@ spec:
             - name: REPORT_INTERVAL
               value: '{{ agent_report_interval }}'
           imagePullPolicy: {{ k8s_image_pull_policy }}
+          securityContext:
+            capabilities:
+              drop:
+{% for c in apps_drop_cap %}
+                - {{ c.upper() }}
+{% endfor %}
           resources:
             limits:
               cpu: {{ netchecker_agent_cpu_limit }}

roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml

@@ -24,6 +24,12 @@ spec:
             - name: REPORT_INTERVAL
               value: '{{ agent_report_interval }}'
           imagePullPolicy: {{ k8s_image_pull_policy }}
+          securityContext:
+            capabilities:
+              drop:
+{% for c in apps_drop_cap %}
+                - {{ c.upper() }}
+{% endfor %}
           resources:
             limits:
               cpu: {{ netchecker_agent_cpu_limit }}

roles/kubernetes-apps/ansible/templates/netchecker-server-pod.yml

@@ -33,3 +33,9 @@ spec:
           memory: {{ netchecker_kubectl_memory_requests }}
       args:
         - proxy
+      securityContext:
+        capabilities:
+          drop:
+{% for c in apps_drop_cap %}
+            - {{ c.upper() }}
+{% endfor %}