Merge pull request #1382 from jwfang/rbac

basic rbac support
2026-02-24 14:36:04 -03:30 · 2017-08-07 08:01:51 -05:00
parent 6eacedc443 805d9f22ce
commit 383d582b47
35 changed files with 450 additions and 105 deletions
--- a/roles/kubernetes/master/defaults/main.yml
+++ b/roles/kubernetes/master/defaults/main.yml
@@ -64,4 +64,4 @@ apiserver_custom_flags: []

 controller_mgr_custom_flags: []

-scheduler_custom_flags: []
+scheduler_custom_flags: []
--- a/roles/kubernetes/master/tasks/main.yml
+++ b/roles/kubernetes/master/tasks/main.yml
@@ -60,12 +60,11 @@
  when: kubesystem|failed and inventory_hostname == groups['kube-master'][0]
  tags: apps

- name: Write kube-controller-manager manifest
+- name: Write kube-scheduler kubeconfig
  template:
-    src: manifests/kube-controller-manager.manifest.j2
-    dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest"
-  notify: Master | wait for kube-controller-manager
-  tags: kube-controller-manager
+    src: kube-scheduler-kubeconfig.yaml.j2
+    dest: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"
+  tags: kube-scheduler

 - name: Write kube-scheduler manifest
  template:
@@ -74,6 +73,19 @@
  notify: Master | wait for kube-scheduler
  tags: kube-scheduler

+- name: Write kube-controller-manager kubeconfig
+  template:
+    src: kube-controller-manager-kubeconfig.yaml.j2
+    dest: "{{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml"
+  tags: kube-controller-manager
+
+- name: Write kube-controller-manager manifest
+  template:
+    src: manifests/kube-controller-manager.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest"
+  notify: Master | wait for kube-controller-manager
+  tags: kube-controller-manager
+
 - include: post-upgrade.yml
  tags: k8s-post-upgrade

--- a/roles/kubernetes/master/templates/kube-controller-manager-kubeconfig.yaml.j2
+++ b/roles/kubernetes/master/templates/kube-controller-manager-kubeconfig.yaml.j2
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.pem
+    server: {{ kube_apiserver_endpoint }}
+users:
+- name: kube-controller-manager
+  user:
+    client-certificate: {{ kube_cert_dir }}/kube-controller-manager.pem
+    client-key: {{ kube_cert_dir }}/kube-controller-manager-key.pem
+contexts:
+- context:
+    cluster: local
+    user: kube-controller-manager
+  name: kube-controller-manager-{{ cluster_name }}
+current-context: kube-controller-manager-{{ cluster_name }}
--- a/roles/kubernetes/master/templates/kube-scheduler-kubeconfig.yaml.j2
+++ b/roles/kubernetes/master/templates/kube-scheduler-kubeconfig.yaml.j2
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.pem
+    server: {{ kube_apiserver_endpoint }}
+users:
+- name: kube-scheduler
+  user:
+    client-certificate: {{ kube_cert_dir }}/kube-scheduler.pem
+    client-key: {{ kube_cert_dir }}/kube-scheduler-key.pem
+contexts:
+- context:
+    cluster: local
+    user: kube-scheduler
+  name: kube-scheduler-{{ cluster_name }}
+current-context: kube-scheduler-{{ cluster_name }}
--- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
@@ -81,6 +81,9 @@ spec:
 {% if kube_api_anonymous_auth is defined and kube_version | version_compare('v1.5', '>=')  %}
    - --anonymous-auth={{ kube_api_anonymous_auth }}
 {% endif %}
+{% if authorization_modes %}
+    - --authorization-mode={{ authorization_modes|join(',') }}
+{% endif %}
 {% if apiserver_custom_flags is string %}
    - {{ apiserver_custom_flags }}
 {% else %}
--- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
@@ -24,7 +24,7 @@ spec:
    command:
    - /hyperkube
    - controller-manager
-    - --master={{ kube_apiserver_endpoint }}
+    - --kubeconfig={{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml
    - --leader-elect=true
    - --service-account-private-key-file={{ kube_cert_dir }}/apiserver-key.pem
    - --root-ca-file={{ kube_cert_dir }}/ca.pem
@@ -35,6 +35,9 @@ spec:
    - --node-monitor-period={{ kube_controller_node_monitor_period }}
    - --pod-eviction-timeout={{ kube_controller_pod_eviction_timeout }}
    - --v={{ kube_log_level }}
+{% if rbac_enabled %}
+    - --use-service-account-credentials
+{% endif %}
 {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere"] %}
    - --cloud-provider={{cloud_provider}}
    - --cloud-config={{ kube_config_dir }}/cloud_config
@@ -61,20 +64,36 @@ spec:
      initialDelaySeconds: 30
      timeoutSeconds: 10
    volumeMounts:
-    - mountPath: {{ kube_cert_dir }}
-      name: ssl-certs-kubernetes
+    - mountPath: /etc/ssl/certs
+      name: ssl-certs-host
+      readOnly: true
+    - mountPath: "{{kube_config_dir}}/ssl"
+      name: etc-kube-ssl
+      readOnly: true
+    - mountPath: "{{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml"
+      name: kubeconfig
      readOnly: true
 {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere" ] %}
-    - mountPath: {{ kube_config_dir }}/cloud_config
+    - mountPath: "{{ kube_config_dir }}/cloud_config"
      name: cloudconfig
      readOnly: true
 {% endif %}
  volumes:
-  - hostPath:
-      path: {{ kube_cert_dir }}
-    name: ssl-certs-kubernetes
+  - name: ssl-certs-host
+    hostPath:
+{% if ansible_os_family == 'RedHat' %}
+      path: /etc/pki/tls
+{% else %}
+      path: /usr/share/ca-certificates
+{% endif %}
+  - name: etc-kube-ssl
+    hostPath:
+      path: "{{ kube_config_dir }}/ssl"
+  - name: kubeconfig
+    hostPath:
+      path: "{{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml"
 {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere"] %}
  - hostPath:
-      path: {{ kube_config_dir }}/cloud_config
+      path: "{{ kube_config_dir }}/cloud_config"
    name: cloudconfig
 {% endif %}
--- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2
@@ -2,7 +2,7 @@ apiVersion: v1
 kind: Pod
 metadata:
  name: kube-scheduler
-  namespace: kube-system
+  namespace: {{ system_namespace }}
  labels:
    k8s-app: kube-scheduler
 spec:
@@ -25,7 +25,7 @@ spec:
    - /hyperkube
    - scheduler
    - --leader-elect=true
-    - --master={{ kube_apiserver_endpoint }}
+    - --kubeconfig={{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml
    - --v={{ kube_log_level }}
 {% if scheduler_custom_flags is string %}
    - {{ scheduler_custom_flags }}
@@ -41,3 +41,27 @@ spec:
        port: 10251
      initialDelaySeconds: 30
      timeoutSeconds: 10
+    volumeMounts:
+    - mountPath: /etc/ssl/certs
+      name: ssl-certs-host
+      readOnly: true
+    - mountPath: "{{ kube_config_dir }}/ssl"
+      name: etc-kube-ssl
+      readOnly: true
+    - mountPath: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"
+      name: kubeconfig
+      readOnly: true
+  volumes:
+  - name: ssl-certs-host
+    hostPath:
+{% if ansible_os_family == 'RedHat' %}
+      path: /etc/pki/tls
+{% else %}
+      path: /usr/share/ca-certificates
+{% endif %}
+  - name: etc-kube-ssl
+    hostPath:
+      path: "{{ kube_config_dir }}/ssl"
+  - name: kubeconfig
+    hostPath:
+      path: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"
--- a/roles/kubernetes/node/tasks/install.yml
+++ b/roles/kubernetes/node/tasks/install.yml
@@ -16,7 +16,7 @@
 - include: "install_{{ kubelet_deployment_type }}.yml"

 - name: install | Write kubelet systemd init file
-  template: 
+  template:
    src: "kubelet.{{ kubelet_deployment_type }}.service.j2"
    dest: "/etc/systemd/system/kubelet.service"
    backup: "yes"
--- a/roles/kubernetes/node/tasks/main.yml
+++ b/roles/kubernetes/node/tasks/main.yml
@@ -30,9 +30,12 @@

 - name: write the kubecfg (auth) file for kubelet
  template:
-    src: node-kubeconfig.yaml.j2
-    dest: "{{ kube_config_dir }}/node-kubeconfig.yaml"
+    src: "{{ item }}-kubeconfig.yaml.j2"
+    dest: "{{ kube_config_dir }}/{{ item }}-kubeconfig.yaml"
    backup: yes
+  with_items:
+    - node
+    - kube-proxy
  notify: restart kubelet
  tags: kubelet

--- a/roles/kubernetes/node/templates/kube-proxy-kubeconfig.yaml.j2
+++ b/roles/kubernetes/node/templates/kube-proxy-kubeconfig.yaml.j2
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.pem
+    server: {{ kube_apiserver_endpoint }}
+users:
+- name: kube-proxy
+  user:
+    client-certificate: {{ kube_cert_dir }}/kube-proxy-{{ inventory_hostname }}.pem
+    client-key: {{ kube_cert_dir }}/kube-proxy-{{ inventory_hostname }}-key.pem
+contexts:
+- context:
+    cluster: local
+    user: kube-proxy
+  name: kube-proxy-{{ cluster_name }}
+current-context: kube-proxy-{{ cluster_name }}
--- a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2
+++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2
@@ -25,10 +25,7 @@ spec:
    - /hyperkube
    - proxy
    - --v={{ kube_log_level }}
-    - --master={{ kube_apiserver_endpoint }}
-{% if not is_kube_master %}
-    - --kubeconfig={{kube_config_dir}}/node-kubeconfig.yaml
-{% endif %}
+    - --kubeconfig={{kube_config_dir}}/kube-proxy-kubeconfig.yaml
    - --bind-address={{ ip | default(ansible_default_ipv4.address) }}
    - --cluster-cidr={{ kube_pods_subnet }}
    - --proxy-mode={{ kube_proxy_mode }}
@@ -41,14 +38,14 @@ spec:
    - mountPath: /etc/ssl/certs
      name: ssl-certs-host
      readOnly: true
-    - mountPath: {{kube_config_dir}}/node-kubeconfig.yaml
-      name: "kubeconfig"
+    - mountPath: "{{ kube_config_dir }}/ssl"
+      name: etc-kube-ssl
      readOnly: true
-    - mountPath: {{kube_config_dir}}/ssl
-      name: "etc-kube-ssl"
+    - mountPath: "{{ kube_config_dir }}/kube-proxy-kubeconfig.yaml"
+      name: kubeconfig
      readOnly: true
    - mountPath: /var/run/dbus
-      name: "var-run-dbus"
+      name: var-run-dbus
      readOnly: false
  volumes:
  - name: ssl-certs-host
@@ -58,12 +55,12 @@ spec:
 {% else %}
      path: /usr/share/ca-certificates
 {% endif %}
-  - name: "kubeconfig"
+  - name: etc-kube-ssl
    hostPath:
-      path: "{{kube_config_dir}}/node-kubeconfig.yaml"
-  - name: "etc-kube-ssl"
+      path: "{{ kube_config_dir }}/ssl"
+  - name: kubeconfig
    hostPath:
-      path: "{{kube_config_dir}}/ssl"
-  - name: "var-run-dbus"
+      path: "{{ kube_config_dir }}/kube-proxy-kubeconfig.yaml"
+  - name: var-run-dbus
    hostPath:
-      path: "/var/run/dbus"
+      path: /var/run/dbus
--- a/roles/kubernetes/preinstall/tasks/set_facts.yml
+++ b/roles/kubernetes/preinstall/tasks/set_facts.yml
@@ -23,7 +23,7 @@
      {% if not is_kube_master and loadbalancer_apiserver_localhost|default(true) -%}
           https://localhost:{{ nginx_kube_apiserver_port|default(kube_apiserver_port) }}
      {%- elif is_kube_master -%}
-           http://127.0.0.1:{{ kube_apiserver_insecure_port }}
+           https://127.0.0.1:{{ kube_apiserver_port }}
      {%- else -%}
      {%-   if loadbalancer_apiserver is defined and loadbalancer_apiserver.port is defined -%}
           https://{{ apiserver_loadbalancer_domain_name|default('lb-apiserver.kubernetes.local') }}:{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }}
--- a/roles/kubernetes/secrets/files/make-ssl.sh
+++ b/roles/kubernetes/secrets/files/make-ssl.sh
@@ -72,32 +72,47 @@ else
    openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca" > /dev/null 2>&1
 fi

+gen_key_and_cert() {
+    local name=$1
+    local subject=$2
+    openssl genrsa -out ${name}-key.pem 2048 > /dev/null 2>&1
+    openssl req -new -key ${name}-key.pem -out ${name}.csr -subj "${subject}" -config ${CONFIG} > /dev/null 2>&1
+    openssl x509 -req -in ${name}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${name}.pem -days 3650 -extensions v3_req -extfile ${CONFIG} > /dev/null 2>&1
+}
+
 if [ ! -e "$SSLDIR/ca-key.pem" ]; then
-    # kube-apiserver key
-    openssl genrsa -out apiserver-key.pem 2048 > /dev/null 2>&1
-    openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config ${CONFIG} > /dev/null 2>&1
-    openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 3650 -extensions v3_req -extfile ${CONFIG} > /dev/null 2>&1
+    # kube-apiserver
+    gen_key_and_cert "apiserver" "/CN=kube-apiserver"
    cat ca.pem >> apiserver.pem
+    # kube-scheduler
+    gen_key_and_cert "kube-scheduler" "/CN=system:kube-scheduler"
+    # kube-controller-manager
+    gen_key_and_cert "kube-controller-manager" "/CN=system:kube-controller-manager"
 fi

+# Admins
 if [ -n "$MASTERS" ]; then
    for host in $MASTERS; do
        cn="${host%%.*}"
-        # admin key
-        openssl genrsa -out admin-${host}-key.pem 2048 > /dev/null 2>&1
-        openssl req -new -key admin-${host}-key.pem -out admin-${host}.csr -subj "/CN=kube-admin-${cn}/O=system:masters" > /dev/null 2>&1
-        openssl x509 -req -in admin-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin-${host}.pem -days 3650 > /dev/null 2>&1
+        # admin
+        gen_key_and_cert "admin-${host}" "/CN=kube-admin-${cn}/O=system:masters"
    done
 fi

-# Nodes and Admin
+# Nodes
 if [ -n "$HOSTS" ]; then
    for host in $HOSTS; do
        cn="${host%%.*}"
-        # node key
-        openssl genrsa -out node-${host}-key.pem 2048 > /dev/null 2>&1
-        openssl req -new -key node-${host}-key.pem -out node-${host}.csr -subj "/CN=kube-node-${cn}" > /dev/null 2>&1
-        openssl x509 -req -in node-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out node-${host}.pem -days 3650 > /dev/null 2>&1
+        gen_key_and_cert "node-${host}" "/CN=system:node:${cn}/O=system:nodes"
+    done
+fi
+
+# system:kube-proxy
+if [ -n "$HOSTS" ]; then
+    for host in $HOSTS; do
+        cn="${host%%.*}"
+        # kube-proxy
+        gen_key_and_cert "kube-proxy-${host}" "/CN=system:kube-proxy"
    done
 fi

--- a/roles/kubernetes/secrets/tasks/gen_certs_script.yml
+++ b/roles/kubernetes/secrets/tasks/gen_certs_script.yml
@@ -56,24 +56,39 @@

 - set_fact:
    all_master_certs: "['ca-key.pem',
+                      'apiserver.pem',
+                      'apiserver-key.pem',
+                      'kube-scheduler.pem',
+                      'kube-scheduler-key.pem',
+                      'kube-controller-manager.pem',
+                      'kube-controller-manager-key.pem',
                      {% for node in groups['kube-master'] %}
                      'admin-{{ node }}.pem',
                      'admin-{{ node }}-key.pem',
-                      'apiserver.pem',
-                      'apiserver-key.pem',
                      {% endfor %}]"
    my_master_certs: ['ca-key.pem',
                     'admin-{{ inventory_hostname }}.pem',
                     'admin-{{ inventory_hostname }}-key.pem',
                     'apiserver.pem',
-                     'apiserver-key.pem'
+                     'apiserver-key.pem',
+                     'kube-scheduler.pem',
+                     'kube-scheduler-key.pem',
+                     'kube-controller-manager.pem',
+                     'kube-controller-manager-key.pem',
                     ]
    all_node_certs: "['ca.pem',
                    {% for node in groups['k8s-cluster'] %}
                    'node-{{ node }}.pem',
                    'node-{{ node }}-key.pem',
+                    'kube-proxy-{{ node }}.pem',
+                    'kube-proxy-{{ node }}-key.pem',
                    {% endfor %}]"
-    my_node_certs: ['ca.pem', 'node-{{ inventory_hostname }}.pem', 'node-{{ inventory_hostname }}-key.pem']
+    my_node_certs: ['ca.pem',
+                   'node-{{ inventory_hostname }}.pem',
+                   'node-{{ inventory_hostname }}-key.pem',
+                   'kube-proxy-{{ inventory_hostname }}.pem',
+                   'kube-proxy-{{ inventory_hostname }}-key.pem',
+                   ]
  tags: facts

 - name: Gen_certs | Gather master certs