Merge branch 'master' of https://github.com/kubernetes-incubator/kubespray into etcd-fix-4

roles/kubernetes/client/tasks/main.yml

@@ -55,7 +55,7 @@
 - name: Copy kubectl binary to ansible host
   fetch:
     src: "{{ bin_dir }}/kubectl"
-    dest: "{{ bin_dir }}/kubectl"
+    dest: "{{ artifacts_dir }}/kubectl"
     flat: yes
     validate_checksum: no
   become: no
@@ -68,8 +68,6 @@
       #!/bin/bash
       kubectl --kubeconfig=admin.conf $@
     dest: "{{ artifacts_dir }}/kubectl.sh"
-    owner: root
-    group: root
     mode: 0755
   become: no
   run_once: yes

roles/kubernetes/master/defaults/main.yml

@@ -52,7 +52,7 @@ kube_apiserver_admission_control:
       {%- if kube_version | version_compare('v1.9', '<') -%}
       GenericAdmissionWebhook
       {%- else -%}
-      ValidatingAdmissionWebhook
+      MutatingAdmissionWebhook,ValidatingAdmissionWebhook
       {%- endif -%}
   - ResourceQuota
 
@@ -73,7 +73,9 @@ kube_oidc_auth: false
 ## Optional settings for OIDC
 # kube_oidc_ca_file: {{ kube_cert_dir }}/ca.pem
 # kube_oidc_username_claim: sub
+# kube_oidc_username_prefix: oidc:
 # kube_oidc_groups_claim: groups
+# kube_oidc_groups_prefix: oidc:
 
 ## Variables for custom flags
 apiserver_custom_flags: []

roles/kubernetes/master/tasks/kubeadm-migrate-certs.yml

@@ -9,6 +9,10 @@
     - {src: apiserver-key.pem, dest: apiserver.key}
     - {src: ca.pem, dest: ca.crt}
     - {src: ca-key.pem, dest: ca.key}
+    - {src: front-proxy-ca.pem, dest: front-proxy-ca.crt}
+    - {src: front-proxy-ca-key.pem, dest: front-proxy-ca.key}
+    - {src: front-proxy-client.pem, dest: front-proxy-client.crt}
+    - {src: front-proxy-client-key.pem, dest: front-proxy-client.key}
     - {src: service-account-key.pem, dest: sa.pub}
     - {src: service-account-key.pem, dest: sa.key}
   register: kubeadm_copy_old_certs

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -73,9 +73,15 @@ spec:
 {%   if kube_oidc_username_claim is defined %}
     - --oidc-username-claim={{ kube_oidc_username_claim }}
 {%   endif %}
+{%   if kube_oidc_username_prefix is defined %}
+    - "--oidc-username-prefix={{ kube_oidc_username_prefix }}"
+{%   endif %}
 {%   if kube_oidc_groups_claim is defined %}
     - --oidc-groups-claim={{ kube_oidc_groups_claim }}
 {%   endif %}
+{%   if kube_oidc_groups_prefix is defined %}
+    - "--oidc-groups-prefix={{ kube_oidc_groups_prefix }}"
+{%   endif %}
 {% endif %}
     - --secure-port={{ kube_apiserver_port }}
     - --insecure-port={{ kube_apiserver_insecure_port }}
@@ -111,7 +117,7 @@ spec:
     - --feature-gates={{ kube_feature_gates|join(',') }}
 {% endif %}
 {% if kube_version | version_compare('v1.9', '>=') %}
-    - --requestheader-client-ca-file={{ kube_cert_dir }}/ca.pem
+    - --requestheader-client-ca-file={{ kube_cert_dir }}/front-proxy-ca.pem
     - --requestheader-allowed-names=front-proxy-client
     - --requestheader-extra-headers-prefix=X-Remote-Extra-
     - --requestheader-group-headers=X-Remote-Group

roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2

@@ -29,6 +29,7 @@ spec:
     - --leader-elect=true
     - --kubeconfig={{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml
 {% if volume_cross_zone_attachment %}
+    - --use-legacy-policy-config
     - --policy-config-file={{ kube_config_dir }}/kube-scheduler-policy.yaml
 {% endif %}
     - --profiling=false

roles/kubernetes/node/defaults/main.yml

@@ -92,3 +92,48 @@ kube_cadvisor_port: 0
 
 # The read-only port for the Kubelet to serve on with no authentication/authorization.
 kube_read_only_port: 0
+
+# sysctl_file_path to add sysctl conf to
+sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf"
+
+# For the openstack integration kubelet will need credentials to access
+# openstack apis like nova and cinder. Per default this values will be
+# read from the environment.
+openstack_auth_url: "{{ lookup('env','OS_AUTH_URL')  }}"
+openstack_username: "{{ lookup('env','OS_USERNAME')  }}"
+openstack_password: "{{ lookup('env','OS_PASSWORD')  }}"
+openstack_region: "{{ lookup('env','OS_REGION_NAME')  }}"
+openstack_tenant_id: "{{ lookup('env','OS_TENANT_ID')| default(lookup('env','OS_PROJECT_ID')|default(lookup('env','OS_PROJECT_NAME'),true)) }}"
+openstack_domain_name: "{{ lookup('env','OS_USER_DOMAIN_NAME') }}"
+openstack_domain_id: "{{ lookup('env','OS_USER_DOMAIN_ID') }}"
+
+# For the vsphere integration, kubelet will need credentials to access
+# vsphere apis
+# Documentation regarding these values can be found
+# https://github.com/kubernetes/kubernetes/blob/master/pkg/cloudprovider/providers/vsphere/vsphere.go#L105
+vsphere_vcenter_ip: "{{ lookup('env', 'VSPHERE_VCENTER') }}"
+vsphere_vcenter_port: "{{ lookup('env', 'VSPHERE_VCENTER_PORT') }}"
+vsphere_user: "{{ lookup('env', 'VSPHERE_USER') }}"
+vsphere_password: "{{ lookup('env', 'VSPHERE_PASSWORD') }}"
+vsphere_datacenter: "{{ lookup('env', 'VSPHERE_DATACENTER') }}"
+vsphere_datastore: "{{ lookup('env', 'VSPHERE_DATASTORE') }}"
+vsphere_working_dir: "{{ lookup('env', 'VSPHERE_WORKING_DIR') }}"
+vsphere_insecure: "{{ lookup('env', 'VSPHERE_INSECURE') }}"
+vsphere_resource_pool: "{{ lookup('env', 'VSPHERE_RESOURCE_POOL') }}"
+
+vsphere_scsi_controller_type: pvscsi
+# vsphere_public_network is name of the network the VMs are joined to
+vsphere_public_network: "{{ lookup('env', 'VSPHERE_PUBLIC_NETWORK')|default('') }}"
+
+## When azure is used, you need to also set the following variables.
+## see docs/azure.md for details on how to get these values
+#azure_tenant_id:
+#azure_subscription_id:
+#azure_aad_client_id:
+#azure_aad_client_secret:
+#azure_resource_group:
+#azure_location:
+#azure_subnet_name:
+#azure_security_group_name:
+#azure_vnet_name:
+#azure_route_table_name:

roles/kubernetes/node/tasks/main.yml

@@ -61,6 +61,7 @@
     name: net.ipv4.ip_local_reserved_ports
     value: "{{ kube_apiserver_node_port_range }}"
     sysctl_set: yes
+    sysctl_file: "{{ sysctl_file_path }}"
     state: present
     reload: yes
   when: kube_apiserver_node_port_range is defined
@@ -96,6 +97,7 @@
   sysctl:
     name: "{{ item }}"
     state: present
+    sysctl_file: "{{ sysctl_file_path }}"
     value: 1
     reload: yes
   when: sysctl_bridge_nf_call_iptables.rc == 0
@@ -118,6 +120,19 @@
   tags:
     - kube-proxy
 
+- name: Persist ip_vs modules
+  copy:
+    dest: /etc/modules-load.d/kube_proxy-ipvs.conf
+    content: |
+      ip_vs
+      ip_vs_rr
+      ip_vs_wrr
+      ip_vs_sh
+      nf_conntrack_ipv4
+  when: kube_proxy_mode == 'ipvs'
+  tags:
+    - kube-proxy
+
 - name: Write proxy manifest
   template:
     src: manifests/kube-proxy.manifest.j2
@@ -134,6 +149,14 @@
   tags:
     - kube-proxy
 
+- include_tasks: "{{ cloud_provider }}-credential-check.yml"
+  when:
+    - cloud_provider is defined
+    - cloud_provider in [ 'openstack', 'azure', 'vsphere' ]
+  tags:
+    - cloud-provider
+    - facts
+
 - name: Write cloud-config
   template:
     src: "{{ cloud_provider }}-cloud-config.j2"

roles/kubernetes/node/templates/kubelet-container.j2

@@ -24,6 +24,15 @@
   -v /var/lib/kubelet:/var/lib/kubelet:shared \
   -v /var/lib/cni:/var/lib/cni:shared \
   -v /var/run:/var/run:rw \
+  {# we can run into issues with double mounting /var/lib/kubelet #}
+  {# surely there's a better way to do this #}
+  {% if '/var/lib/kubelet' not in kubelet_flexvolumes_plugins_dir %}
+  -v {{ kubelet_flexvolumes_plugins_dir }}:{{ kubelet_flexvolumes_plugins_dir }}:rw \
+  {% endif -%}
+  {% if local_volume_provisioner_enabled -%}
+  -v {{ local_volume_provisioner_base_dir }}:{{ local_volume_provisioner_base_dir }}:rw \
+  -v {{ local_volume_provisioner_mount_dir }}:{{ local_volume_provisioner_mount_dir }}:rw \
+  {% endif %}
   -v {{kube_config_dir}}:{{kube_config_dir}}:ro \
   -v /etc/os-release:/etc/os-release:ro \
   {{ hyperkube_image_repo }}:{{ hyperkube_image_tag}} \

roles/kubernetes/node/templates/kubelet.docker.service.j2

@@ -23,9 +23,7 @@ ExecStart={{ bin_dir }}/kubelet \
 Restart=always
 RestartSec=10s
 ExecStartPre=-{{ docker_bin_dir }}/docker rm -f kubelet
-{% if kubelet_flexvolumes_plugins_dir is defined %}
 ExecStartPre=-/bin/mkdir -p {{ kubelet_flexvolumes_plugins_dir }}
-{% endif %}
 ExecReload={{ docker_bin_dir }}/docker restart kubelet
 
 

roles/kubernetes/node/templates/kubelet.host.service.j2

@@ -7,9 +7,7 @@ Wants=docker.socket
 [Service]
 User=root
 EnvironmentFile=-{{kube_config_dir}}/kubelet.env
-{% if kubelet_flexvolumes_plugins_dir is defined %}
 ExecStartPre=-/bin/mkdir -p {{ kubelet_flexvolumes_plugins_dir }}
-{% endif %}
 ExecStart={{ bin_dir }}/kubelet \
 		$KUBE_LOGTOSTDERR \
 		$KUBE_LOG_LEVEL \

roles/kubernetes/node/templates/kubelet.rkt.service.j2

@@ -12,10 +12,7 @@ LimitNOFILE=40000
 
 ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet.uuid
 ExecStartPre=-/bin/mkdir -p /var/lib/kubelet
-
-{% if kubelet_flexvolumes_plugins_dir is defined %}
 ExecStartPre=-/bin/mkdir -p {{ kubelet_flexvolumes_plugins_dir }}
-{% endif %}
 
 EnvironmentFile={{kube_config_dir}}/kubelet.env
 # stage1-fly mounts /proc /sys /dev so no need to duplicate the mounts
@@ -41,8 +38,17 @@ ExecStart=/usr/bin/rkt run \
         --volume etc-cni,kind=host,source=/etc/cni,readOnly=true \
         --volume opt-cni,kind=host,source=/opt/cni,readOnly=true \
         --volume var-lib-cni,kind=host,source=/var/lib/cni,readOnly=false \
-{% if kubelet_flexvolumes_plugins_dir is defined %}
+{# we can run into issues with double mounting /var/lib/kubelet #}
+{# surely there's a better way to do this #}
+{% if '/var/lib/kubelet' not in kubelet_flexvolumes_plugins_dir %}
         --volume flexvolumes,kind=host,source={{ kubelet_flexvolumes_plugins_dir }},readOnly=false \
+{% endif -%}
+{% if local_volume_provisioner_enabled %}
+        --volume local-volume-provisioner-base-dir,kind=host,source={{ local_volume_provisioner_base_dir }},readOnly=false \
+{# Not pretty, but needed to avoid double mount #}
+{% if local_volume_provisioner_base_dir not in local_volume_provisioner_mount_dir and local_volume_provisioner_mount_dir not in local_volume_provisioner_base_dir %}
+        --volume local-volume-provisioner-mount-dir,kind=host,source={{ local_volume_provisioner_mount_dir }},readOnly=false \
+{% endif %}
 {% endif %}
 {% if kubelet_load_modules == true %}
         --mount volume=modprobe,target=/usr/sbin/modprobe \
@@ -65,8 +71,17 @@ ExecStart=/usr/bin/rkt run \
         --mount volume=var-lib-kubelet,target=/var/lib/kubelet \
         --mount volume=var-log,target=/var/log \
         --mount volume=hosts,target=/etc/hosts \
-{% if kubelet_flexvolumes_plugins_dir is defined %}
+{# we can run into issues with double mounting /var/lib/kubelet #}
+{# surely there's a better way to do this #}
+{% if '/var/lib/kubelet' not in kubelet_flexvolumes_plugins_dir %}
         --mount volume=flexvolumes,target={{ kubelet_flexvolumes_plugins_dir }} \
+{% endif -%}
+{% if local_volume_provisioner_enabled %}
+        --mount volume=local-volume-provisioner-base-dir,target={{ local_volume_provisioner_base_dir }} \
+{# Not pretty, but needed to avoid double mount #}
+{% if local_volume_provisioner_base_dir not in local_volume_provisioner_mount_dir and local_volume_provisioner_mount_dir not in local_volume_provisioner_base_dir %}
+        --mount volume=local-volume-provisioner-mount-dir,target={{ local_volume_provisioner_mount_dir }} \
+{% endif %}
 {% endif %}
         --stage1-from-dir=stage1-fly.aci \
 {% if kube_hyperkube_image_repo == "docker" %}

roles/kubernetes/node/templates/kubelet.standard.env.j2

@@ -83,21 +83,21 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {# Kubelet node labels #}
 {% set role_node_labels = [] %}
 {% if inventory_hostname in groups['kube-master'] %}
-{%   do role_node_labels.append('node-role.kubernetes.io/master=true') %}
+{%   set dummy = role_node_labels.append('node-role.kubernetes.io/master=true') %}
 {%   if not standalone_kubelet|bool %}
-{%     do role_node_labels.append('node-role.kubernetes.io/node=true') %}
+{%     set dummy = role_node_labels.append('node-role.kubernetes.io/node=true') %}
 {%   endif %}
 {% else %}
-{%   do role_node_labels.append('node-role.kubernetes.io/node=true') %}
+{%   set dummy = role_node_labels.append('node-role.kubernetes.io/node=true') %}
 {% endif %}
 {% if inventory_hostname in groups['kube-ingress']|default([]) %}
-{%   do role_node_labels.append('node-role.kubernetes.io/ingress=true') %}
+{%   set dummy = role_node_labels.append('node-role.kubernetes.io/ingress=true') %}
 {% endif %}
 {% set inventory_node_labels = [] %}
 {% if node_labels is defined %}
-{% for labelname, labelvalue in node_labels.iteritems() %}
-{% do inventory_node_labels.append(labelname + '=' + labelvalue) %}
-{% endfor %}
+{%   for labelname, labelvalue in node_labels.iteritems() %}
+{%     set dummy = inventory_node_labels.append('%s=%s'|format(labelname, labelvalue)) %}
+{%   endfor %}
 {% endif %}
 {% set all_node_labels = role_node_labels + inventory_node_labels %}
 
@@ -110,9 +110,7 @@ DOCKER_SOCKET="--docker-endpoint=unix:/var/run/weave/weave.sock"
 KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet"
 {% endif %}
 
-{% if kubelet_flexvolumes_plugins_dir is defined %}
 KUBELET_VOLUME_PLUGIN="--volume-plugin-dir={{ kubelet_flexvolumes_plugins_dir }}"
-{% endif %}
 
 # Should this cluster be allowed to run privileged docker containers
 KUBE_ALLOW_PRIV="--allow-privileged=true"

roles/kubernetes/preinstall/defaults/main.yml

@@ -8,7 +8,7 @@ epel_enabled: false
 
 common_required_pkgs:
   - python-httplib2
-  - openssl
+  - "{{ (ansible_distribution == 'openSUSE Tumbleweed') | ternary('openssl-1_1_0', 'openssl') }}"
   - curl
   - rsync
   - bash-completion
@@ -23,35 +23,6 @@ disable_ipv6_dns: false
 kube_cert_group: kube-cert
 kube_config_dir: /etc/kubernetes
 
-# For the openstack integration kubelet will need credentials to access
-# openstack apis like nova and cinder. Per default this values will be
-# read from the environment.
-openstack_auth_url: "{{ lookup('env','OS_AUTH_URL')  }}"
-openstack_username: "{{ lookup('env','OS_USERNAME')  }}"
-openstack_password: "{{ lookup('env','OS_PASSWORD')  }}"
-openstack_region: "{{ lookup('env','OS_REGION_NAME')  }}"
-openstack_tenant_id: "{{ lookup('env','OS_TENANT_ID')|default(lookup('env','OS_PROJECT_ID'),true)  }}"
-openstack_domain_name: "{{ lookup('env','OS_USER_DOMAIN_NAME') }}"
-openstack_domain_id: "{{ lookup('env','OS_USER_DOMAIN_ID') }}"
-
-# For the vsphere integration, kubelet will need credentials to access
-# vsphere apis
-# Documentation regarding these values can be found
-# https://github.com/kubernetes/kubernetes/blob/master/pkg/cloudprovider/providers/vsphere/vsphere.go#L105
-vsphere_vcenter_ip: "{{ lookup('env', 'VSPHERE_VCENTER') }}"
-vsphere_vcenter_port: "{{ lookup('env', 'VSPHERE_VCENTER_PORT') }}"
-vsphere_user: "{{ lookup('env', 'VSPHERE_USER') }}"
-vsphere_password: "{{ lookup('env', 'VSPHERE_PASSWORD') }}"
-vsphere_datacenter: "{{ lookup('env', 'VSPHERE_DATACENTER') }}"
-vsphere_datastore: "{{ lookup('env', 'VSPHERE_DATASTORE') }}"
-vsphere_working_dir: "{{ lookup('env', 'VSPHERE_WORKING_DIR') }}"
-vsphere_insecure: "{{ lookup('env', 'VSPHERE_INSECURE') }}"
-vsphere_resource_pool: "{{ lookup('env', 'VSPHERE_RESOURCE_POOL') }}"
-
-vsphere_scsi_controller_type: pvscsi
-# vsphere_public_network is name of the network the VMs are joined to
-vsphere_public_network: "{{ lookup('env', 'VSPHERE_PUBLIC_NETWORK')|default('') }}"
-
 # Container Linux by CoreOS cloud init config file to define /etc/resolv.conf content
 # for hostnet pods and infra needs
 resolveconf_cloud_init_conf: /etc/resolveconf_cloud_init.conf
@@ -60,3 +31,5 @@ resolveconf_cloud_init_conf: /etc/resolveconf_cloud_init.conf
 populate_inventory_to_hosts_file: true
 
 preinstall_selinux_state: permissive
+
+sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf"

roles/kubernetes/preinstall/tasks/dhclient-hooks.yml

@@ -15,7 +15,7 @@
   notify: Preinstall | restart network
   when: dhclientconffile is defined
 
-- name: Configue dhclient hooks for resolv.conf (non-RH)
+- name: Configure dhclient hooks for resolv.conf (non-RH)
   template:
     src: dhclient_dnsupdate.sh.j2
     dest: "{{ dhclienthookfile }}"
@@ -24,7 +24,7 @@
   notify: Preinstall | restart network
   when: ansible_os_family != "RedHat"
 
-- name: Configue dhclient hooks for resolv.conf (RH-only)
+- name: Configure dhclient hooks for resolv.conf (RH-only)
   template:
     src: dhclient_dnsupdate_rh.sh.j2
     dest: "{{ dhclienthookfile }}"

roles/kubernetes/preinstall/tasks/main.yml

@@ -3,6 +3,12 @@
   tags:
     - asserts
 
+# This is run before bin_dir is pinned because these tasks are run on localhost
+- import_tasks: pre_upgrade.yml
+  run_once: true
+  tags:
+    - upgrade
+
 - name: Force binaries directory for Container Linux by CoreOS
   set_fact:
     bin_dir: "/opt/bin"
@@ -71,14 +77,6 @@
     - cloud-provider
     - facts
 
-- include_tasks: "{{ cloud_provider }}-credential-check.yml"
-  when:
-    - cloud_provider is defined
-    - cloud_provider in [ 'openstack', 'azure', 'vsphere' ]
-  tags:
-    - cloud-provider
-    - facts
-
 - name: Create cni directories
   file:
     path: "{{ item }}"
@@ -99,6 +97,20 @@
     - contiv
     - bootstrap-os
 
+- name: Create local volume provisioner directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: kube
+  with_items:
+    - "{{ local_volume_provisioner_base_dir }}"
+    - "{{ local_volume_provisioner_mount_dir }}"
+  when:
+    - inventory_hostname in groups['k8s-cluster']
+    - local_volume_provisioner_enabled
+  tags:
+    - persistent_volumes
+
 - import_tasks: resolvconf.yml
   when:
     - dns_mode != 'none'
@@ -146,6 +158,15 @@
     - not is_atomic
   tags: bootstrap-os
 
+- name: Update package management cache (zypper) - SUSE
+  shell: zypper -n --gpg-auto-import-keys ref
+  register: make_cache_output
+  until: make_cache_output|succeeded
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  when:
+    - ansible_pkg_mgr == 'zypper'
+  tags: bootstrap-os
 
 - name: Update package management cache (APT)
   apt:
@@ -224,12 +245,6 @@
   tags:
     - bootstrap-os
 
-- name: set default sysctl file path
-  set_fact:
-    sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf"
-  tags:
-    - bootstrap-os
-
 - name: Stat sysctl file configuration
   stat:
     path: "{{sysctl_file_path}}"

roles/kubernetes/preinstall/tasks/pre_upgrade.yml

@@ -0,0 +1,28 @@
+---
+- name: "Pre-upgrade | check if old credential dir exists"
+  local_action:
+    module: stat
+    path: "{{ inventory_dir }}/../credentials"
+  vars:
+    ansible_python_interpreter: "/usr/bin/env python"
+  register: old_credential_dir
+  become: no
+
+- name: "Pre-upgrade | check if new credential dir exists"
+  local_action:
+    module: stat
+    path: "{{ inventory_dir }}/credentials"
+  vars:
+    ansible_python_interpreter: "/usr/bin/env python"
+  register: new_credential_dir
+  become: no
+  when: old_credential_dir.stat.exists
+
+- name: "Pre-upgrade | move data from old credential dir to new"
+  local_action: command mv {{ inventory_dir }}/../credentials {{ inventory_dir }}/credentials
+  args:
+    creates: "{{ inventory_dir }}/credentials"
+  vars:
+    ansible_python_interpreter: "/usr/bin/env python"
+  become: no
+  when: old_credential_dir.stat.exists and not new_credential_dir.stat.exists

roles/kubernetes/preinstall/tasks/verify-settings.yml

@@ -12,7 +12,7 @@
 
 - name: Stop if unknown OS
   assert:
-    that: ansible_distribution in ['RedHat', 'CentOS', 'Fedora', 'Ubuntu', 'Debian', 'CoreOS', 'Container Linux by CoreOS']
+    that: ansible_distribution in ['RedHat', 'CentOS', 'Fedora', 'Ubuntu', 'Debian', 'CoreOS', 'Container Linux by CoreOS', 'openSUSE Leap', 'openSUSE Tumbleweed']
   ignore_errors: "{{ ignore_assert_errors }}"
 
 - name: Stop if unknown network plugin
@@ -94,4 +94,4 @@
   assert:
     that: ansible_kernel.split('-')[0]|version_compare('4.8', '>=')
   when: kube_network_plugin == 'cilium'
-  ignore_errors: "{{ ignore_assert_errors }}"
+  ignore_errors: "{{ ignore_assert_errors }}"

roles/kubernetes/preinstall/vars/suse.yml

@@ -0,0 +1,4 @@
+---
+required_pkgs:
+  - device-mapper
+  - ebtables

roles/kubernetes/secrets/defaults/main.yml

@@ -1,3 +1,4 @@
 ---
 kube_cert_group: kube-cert
 kube_vault_mount_path: kube
+front_proxy_vault_mount_path: front-proxy

roles/kubernetes/secrets/files/make-ssl.sh

@@ -72,6 +72,15 @@ else
     openssl req -x509 -new -nodes -key ca-key.pem -days 36500 -out ca.pem -subj "/CN=kube-ca" > /dev/null 2>&1
 fi
 
+# Front proxy client CA
+if [ -e "$SSLDIR/front-proxy-ca-key.pem" ]; then
+    # Reuse existing front proxy CA
+    cp $SSLDIR/{front-proxy-ca.pem,front-proxy-ca-key.pem} .
+else
+    openssl genrsa -out front-proxy-ca-key.pem 2048 > /dev/null 2>&1
+    openssl req -x509 -new -nodes -key front-proxy-ca-key.pem -days 36500 -out front-proxy-ca.pem -subj "/CN=front-proxy-ca" > /dev/null 2>&1
+fi
+
 gen_key_and_cert() {
     local name=$1
     local subject=$2
@@ -80,6 +89,14 @@ gen_key_and_cert() {
     openssl x509 -req -in ${name}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${name}.pem -days 36500 -extensions v3_req -extfile ${CONFIG} > /dev/null 2>&1
 }
 
+gen_key_and_cert_front_proxy() {
+    local name=$1
+    local subject=$2
+    openssl genrsa -out ${name}-key.pem 2048 > /dev/null 2>&1
+    openssl req -new -key ${name}-key.pem -out ${name}.csr -subj "${subject}" -config ${CONFIG} > /dev/null 2>&1
+    openssl x509 -req -in ${name}.csr -CA front-proxy-ca.pem -CAkey front-proxy-ca-key.pem -CAcreateserial -out ${name}.pem -days 36500 -extensions v3_req -extfile ${CONFIG} > /dev/null 2>&1
+}
+
 # Admins
 if [ -n "$MASTERS" ]; then
 
@@ -105,7 +122,7 @@ if [ -n "$MASTERS" ]; then
     # kube-controller-manager
     gen_key_and_cert "kube-controller-manager" "/CN=system:kube-controller-manager"
     # metrics aggregator
-    gen_key_and_cert "front-proxy-client" "/CN=front-proxy-client"
+    gen_key_and_cert_front_proxy "front-proxy-client" "/CN=front-proxy-client"
 
     for host in $MASTERS; do
         cn="${host%%.*}"

roles/kubernetes/secrets/tasks/check-certs.yml

@@ -48,8 +48,11 @@
        '{{ kube_cert_dir }}/kube-scheduler-key.pem',
        '{{ kube_cert_dir }}/kube-controller-manager.pem',
        '{{ kube_cert_dir }}/kube-controller-manager-key.pem',
+       '{{ kube_cert_dir }}/front-proxy-ca.pem',
+       '{{ kube_cert_dir }}/front-proxy-ca-key.pem',
        '{{ kube_cert_dir }}/front-proxy-client.pem',
        '{{ kube_cert_dir }}/front-proxy-client-key.pem',
+       '{{ kube_cert_dir }}/service-account-key.pem',
        {% for host in groups['kube-master'] %}
        '{{ kube_cert_dir }}/admin-{{ host }}.pem'
        '{{ kube_cert_dir }}/admin-{{ host }}-key.pem'
@@ -71,7 +74,9 @@
       {% for cert in ['apiserver.pem', 'apiserver-key.pem',
                       'kube-scheduler.pem','kube-scheduler-key.pem',
                       'kube-controller-manager.pem','kube-controller-manager-key.pem',
-                      'front-proxy-client.pem','front-proxy-client-key.pem'] -%}
+                      'front-proxy-ca.pem','front-proxy-ca-key.pem',
+                      'front-proxy-client.pem','front-proxy-client-key.pem',
+                      'service-account-key.pem'] -%}
         {% set cert_file = "%s/%s.pem"|format(kube_cert_dir, cert) %}
         {% if not cert_file in existing_certs -%}
         {%- set gen = True -%}

roles/kubernetes/secrets/tasks/gen_certs_script.yml

@@ -73,6 +73,8 @@
                        'kube-scheduler-key.pem',
                        'kube-controller-manager.pem',
                        'kube-controller-manager-key.pem',
+                       'front-proxy-ca.pem',
+                       'front-proxy-ca-key.pem',
                        'front-proxy-client.pem',
                        'front-proxy-client-key.pem',
                        'service-account-key.pem',
@@ -85,6 +87,8 @@
                       'admin-{{ inventory_hostname }}-key.pem',
                       'apiserver.pem',
                       'apiserver-key.pem',
+                      'front-proxy-ca.pem',
+                      'front-proxy-ca-key.pem',
                       'front-proxy-client.pem',
                       'front-proxy-client-key.pem',
                       'service-account-key.pem',

roles/kubernetes/secrets/tasks/gen_certs_vault.yml

@@ -52,6 +52,11 @@
         "{{ hostvars[host]['ip'] }}",
         {%- endif -%}
         {%- endfor -%}
+        {%- if supplementary_addresses_in_ssl_keys is defined -%}
+        {%- for ip_item in supplementary_addresses_in_ssl_keys -%}
+        "{{ ip_item }}",
+        {%- endfor -%}
+        {%- endif -%}
         "127.0.0.1","::1","{{ kube_apiserver_ip }}"
         ]
     issue_cert_path: "{{ item }}"
@@ -98,6 +103,8 @@
 - include_tasks: ../../../vault/tasks/shared/issue_cert.yml
   vars:
     issue_cert_common_name: "front-proxy-client"
+    issue_cert_copy_ca: "{{ item == kube_front_proxy_clients_certs_needed|first }}"
+    issue_cert_ca_filename: front-proxy-ca.pem
     issue_cert_alt_names: "{{ kube_cert_alt_names }}"
     issue_cert_file_group: "{{ kube_cert_group }}"
     issue_cert_file_owner: kube
@@ -110,12 +117,17 @@
         "{{ hostvars[host]['ip'] }}",
         {%- endif -%}
         {%- endfor -%}
+        {%- if supplementary_addresses_in_ssl_keys is defined -%}
+        {%- for ip_item in supplementary_addresses_in_ssl_keys -%}
+        "{{ ip_item }}",
+        {%- endfor -%}
+        {%- endif -%}
         "127.0.0.1","::1","{{ kube_apiserver_ip }}"
         ]
     issue_cert_path: "{{ item }}"
     issue_cert_role: front-proxy-client
     issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
-    issue_cert_mount_path: "{{ kube_vault_mount_path }}"
+    issue_cert_mount_path: "{{ front_proxy_vault_mount_path }}"
   with_items: "{{ kube_front_proxy_clients_certs_needed|d([]) }}"
   when: inventory_hostname in groups['kube-master']
   notify: set secret_changed

roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml

@@ -32,7 +32,7 @@
     sync_file_hosts: "{{ groups['kube-master'] }}"
     sync_file_is_cert: true
     sync_file_owner: kube
-  with_items: ["apiserver.pem", "kube-scheduler.pem", "kube-controller-manager.pem"]
+  with_items: ["apiserver.pem", "kube-scheduler.pem", "kube-controller-manager.pem", "service-account.pem"]
 
 - name: sync_kube_master_certs | Set facts for kube master components sync_file results
   set_fact:
@@ -44,6 +44,18 @@
   set_fact:
     sync_file_results: []
 
+- include_tasks: ../../../vault/tasks/shared/sync_file.yml
+  vars:
+    sync_file: front-proxy-ca.pem
+    sync_file_dir: "{{ kube_cert_dir }}"
+    sync_file_group: "{{ kube_cert_group }}"
+    sync_file_hosts: "{{ groups['kube-master'] }}"
+    sync_file_owner: kube
+
+- name: sync_kube_master_certs | Unset sync_file_results after front-proxy-ca.pem
+  set_fact:
+    sync_file_results: []
+
 - include_tasks: ../../../vault/tasks/shared/sync_file.yml
   vars:
     sync_file: "{{ item }}"

roles/kubernetes/secrets/tasks/upd_ca_trust.yml

@@ -8,6 +8,8 @@
       /etc/pki/ca-trust/source/anchors/kube-ca.crt
       {%- elif ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] -%}
       /etc/ssl/certs/kube-ca.pem
+      {%- elif ansible_os_family == "Suse" -%}
+      /etc/pki/trust/anchors/kube-ca.pem
       {%- endif %}
   tags:
     - facts
@@ -19,9 +21,9 @@
     remote_src: true
   register: kube_ca_cert
 
-- name: Gen_certs | update ca-certificates (Debian/Ubuntu/Container Linux by CoreOS)
+- name: Gen_certs | update ca-certificates (Debian/Ubuntu/SUSE/Container Linux by CoreOS)
   command: update-ca-certificates
-  when: kube_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS", "Container Linux by CoreOS"]
+  when: kube_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS", "Container Linux by CoreOS", "Suse"]
 
 - name: Gen_certs | update ca-certificates (RedHat)
   command: update-ca-trust extract