Drop linux capabilities and rework users/groups

* Drop linux capabilities for unprivileged containerized
  worlkoads Kargo configures for deployments.
* Configure required securityContext/user/group/groups for kube
  components' static manifests, etcd, calico-rr and k8s apps,
  like dnsmasq daemonset.
* Rework cloud-init (etcd) users creation for CoreOS.
* Fix nologin paths, adjust defaults for addusers role and ensure
  supplementary groups membership added for users.
* Add netplug user for network plugins (yet unused by privileged
  networking containers though).
* Grant the kube and netplug users read access for etcd certs via
  the etcd certs group.
* Grant group read access to kube certs via the kube cert group.
* Remove priveleged mode for calico-rr and run it under its uid/gid
  and supplementary etcd_cert group.
* Adjust docs.
* Align cpu/memory limits and dropped caps with added rkt support
  for control plane.

Signed-off-by: Bogdan Dobrelya <bogdando@mail.ru>

roles/etcd/defaults/main.yml

@@ -3,10 +3,26 @@ etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/
 
 etcd_config_dir: /etc/ssl/etcd
 etcd_cert_dir: "{{ etcd_config_dir }}/ssl"
-etcd_cert_group: root
 
 etcd_script_dir: "{{ bin_dir }}/etcd-scripts"
 
+# Linux capabilities to be dropped for container engines
+etcd_drop_cap:
+  - chown
+  - dac_override
+  - fowner
+  - fsetid
+  - kill
+  - setgid
+  - setuid
+  - setpcap
+  - net_bind_service
+  - net_raw
+  - sys_chroot
+  - mknod
+  - audit_write
+  - setfcap
+
 # Limits
 etcd_memory_limit: 512M
 etcd_cpu_limit: 300m

roles/etcd/files/make-ssl-etcd.sh

@@ -94,5 +94,8 @@ if [ -n "$HOSTS" ]; then
     done
 fi
 
+# Grant the group read access
+chmod g+r *.pem
+
 # Install certs
 mv *.pem ${SSLDIR}/

roles/etcd/meta/main.yml

@@ -2,7 +2,7 @@
 dependencies:
   - role: adduser
     user: "{{ addusers.etcd }}"
-    when: not ansible_os_family in ['CoreOS', 'Container Linux by CoreOS']
+    tags: bootstrap-os
   - role: download
     file: "{{ downloads.etcd }}"
     tags: download

roles/etcd/tasks/gen_certs.yml

@@ -4,14 +4,15 @@
     path={{ etcd_cert_dir }}
     group={{ etcd_cert_group }}
     state=directory
-    owner=root
+    mode=0750
+    owner={{ etcd_user }}
     recurse=yes
 
 - name: "Gen_certs | create etcd script dir (on {{groups['etcd'][0]}})"
   file:
     path: "{{ etcd_script_dir }}"
     state: directory
-    owner: root
+    owner: "{{ etcd_user }}"
   run_once: yes
   delegate_to: "{{groups['etcd'][0]}}"
 
@@ -20,7 +21,8 @@
     path={{ etcd_cert_dir }}
     group={{ etcd_cert_group }}
     state=directory
-    owner=root
+    mode=0750
+    owner={{ etcd_user }}
     recurse=yes
   run_once: yes
   delegate_to: "{{groups['etcd'][0]}}"
@@ -124,12 +126,12 @@
     path={{ etcd_cert_dir }}
     group={{ etcd_cert_group }}
     state=directory
-    owner=kube
+    owner={{ etcd_user }}
     recurse=yes
   tags: facts
 
-- name: Gen_certs | set permissions on keys
-  shell: chmod 0600 {{ etcd_cert_dir}}/*key.pem
+- name: Gen_certs | set shared group permissions on keys
+  shell: chmod 0640 {{ etcd_cert_dir}}/*.pem
   when: inventory_hostname in groups['etcd']
   changed_when: false
 

roles/etcd/tasks/main.yml

@@ -1,6 +1,8 @@
 ---
 - include: pre_upgrade.yml
   tags: etcd-pre-upgrade
+- include: set_facts.yml
+  tags: [bootstrap-os, facts]
 - include: check_certs.yml
   tags: [etcd-secrets, facts]
 - include: gen_certs.yml

roles/etcd/tasks/pre_upgrade.yml

@@ -1,3 +1,4 @@
+---
 - name: "Pre-upgrade | check for etcd-proxy unit file"
   stat:
     path: /etc/systemd/system/etcd-proxy.service
@@ -49,3 +50,7 @@
     awk -F"[: =]" '{print "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses | regex_replace('https','http') }} member update "$1" https:"$7":"$8}' | bash
   run_once: true
   when: 'etcd_member_list.rc == 0 and "http://" in etcd_member_list.stdout'
+
+- name: "Pre-upgrade | share access to etcd certs for its users"
+  shell: chmod g+r {{ etcd_cert_dir }}/*.pem
+  failed_when: false

roles/etcd/tasks/set_facts.yml

@@ -0,0 +1,17 @@
+---
+- name: Etcd | get etcd user ID
+  shell: /usr/bin/id -u {{ etcd_user }} || echo 0
+  register: etcd_uid
+
+- name: Etcd | get etcd group ID
+  shell: /usr/bin/getent group {{ etcd_group }} | cut -d':' -f3 || echo 0
+  register: etcd_gid
+
+- name: Etcd | get etcd cert group ID
+  shell: /usr/bin/getent group {{ etcd_cert_group }} | cut -d':' -f3 || echo 0
+  register: etcd_cert_gid
+
+- set_fact:
+    etcd_user_id: "{{ etcd_uid.stdout }}"
+    etcd_group_id: "{{ etcd_gid.stdout }}"
+    etcd_cert_group_id: "{{ etcd_cert_gid.stdout }}"

roles/etcd/templates/etcd-docker.service.j2

@@ -14,8 +14,12 @@ ExecStart={{ docker_bin_dir }}/docker run --restart=on-failure:5 \
 -v /etc/ssl/certs:/etc/ssl/certs:ro \
 -v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \
 -v /var/lib/etcd:/var/lib/etcd:rw \
+{% for c in etcd_drop_cap %}
+--cap-drop={{ c }} \
+{% endfor %}
 --memory={{ etcd_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ etcd_cpu_limit|regex_replace('m', '') }} \
 --name={{ etcd_member_name | default("etcd") }} \
+-u {{ etcd_user_id }}:{{ etcd_group_id }} --group-add {{ etcd_cert_group_id }} \
 {{ etcd_image_repo }}:{{ etcd_image_tag }} \
 {% if etcd_after_v3 %}
 {{ etcd_container_bin_dir }}etcd

roles/etcd/templates/etcd-rkt.service.j2

@@ -8,6 +8,9 @@ Restart=on-failure
 RestartSec=10s
 TimeoutStartSec=0
 LimitNOFILE=40000
+User=root
+Group={{ etcd_group_id }}
+SupplementaryGroups={{ etcd_cert_group_id }}
 
 ExecStart=/usr/bin/rkt run \
 --uuid-file-save=/var/run/etcd.uuid \
@@ -20,6 +23,11 @@ ExecStart=/usr/bin/rkt run \
 --set-env-file=/etc/etcd.env \
 --stage1-from-dir=stage1-fly.aci \
 {{ etcd_image_repo }}:{{ etcd_image_tag }} \
+{% for c in etcd_drop_cap %}
+--caps-remove=CAP_{{ c.upper() }} \
+{% endfor %}
+--memory={{ etcd_memory_limit }} --cpu={{ etcd_cpu_limit }} \
+--user={{ etcd_user_id }} --group={{ etcd_group_id }} \
 --name={{ etcd_member_name | default("etcd") }}
 
 ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/etcd.uuid