Merge branch 'master' into gpu2

roles/kubernetes/master/tasks/main.yml

@@ -10,8 +10,15 @@
   when: kube_encrypt_secret_data
 
 - name: install | Copy kubectl binary from download dir
-  command: rsync -piu "{{ local_release_dir }}/hyperkube" "{{ bin_dir }}/kubectl"
+  synchronize:
+    src: "{{ local_release_dir }}/hyperkube"
+    dest: "{{ bin_dir }}/kubectl"
+    compress: no
+    perms: yes
+    owner: no
+    group: no
   changed_when: false
+  delegate_to: "{{ inventory_hostname }}"
   tags:
     - hyperkube
     - kubectl

roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2

@@ -68,9 +68,18 @@ apiServerExtraArgs:
 {% endif %}
   service-node-port-range: {{ kube_apiserver_node_port_range }}
   kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}"
+  profiling: "{{ kube_profiling }}"
+  repair-malformed-updates: "false"
+  enable-aggregator-routing: "{{ kube_api_aggregator_routing }}"
+{% if kube_api_anonymous_auth is defined and kube_version | version_compare('v1.5', '>=')  %}
+  anonymous-auth: "{{ kube_api_anonymous_auth }}"
+{% endif %}
 {% if kube_basic_auth|default(true) %}
   basic-auth-file: {{ kube_users_dir }}/known_users.csv
 {% endif %}
+{% if kube_token_auth|default(true) %}
+  token-auth-file: {{ kube_token_dir }}/known_tokens.csv
+{% endif %}
 {% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %}
   oidc-issuer-url: {{ kube_oidc_url }}
   oidc-client-id: {{ kube_oidc_client_id }}
@@ -102,19 +111,21 @@ controllerManagerExtraArgs:
   node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
   node-monitor-period: {{ kube_controller_node_monitor_period }}
   pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }}
+  profiling: "{{ kube_profiling }}"
 {% if kube_feature_gates %}
   feature-gates: {{ kube_feature_gates|join(',') }}
 {% endif %}
+{% for key in kube_kubeadm_controller_extra_args %}
+  {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
+{% endfor %}
 {% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %}
 controllerManagerExtraVolumes:
 - name: openstackcacert
   hostPath: "{{ kube_config_dir }}/openstack-cacert.pem"
   mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
 {% endif %}
-{% for key in kube_kubeadm_controller_extra_args %}
-  {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
-{% endfor %}
 schedulerExtraArgs:
+  profiling: "{{ kube_profiling }}"
 {% if kube_feature_gates %}
   feature-gates: {{ kube_feature_gates|join(',') }}
 {% endif %}

roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2

@@ -60,9 +60,18 @@ apiServerExtraArgs:
 {% endif %}
   service-node-port-range: {{ kube_apiserver_node_port_range }}
   kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}"
+  profiling: "{{ kube_profiling }}"
+  repair-malformed-updates: "false"
+  enable-aggregator-routing: "{{ kube_api_aggregator_routing }}"
+{% if kube_api_anonymous_auth is defined and kube_version | version_compare('v1.5', '>=')  %}
+  anonymous-auth: "{{ kube_api_anonymous_auth }}"
+{% endif %}
 {% if kube_basic_auth|default(true) %}
   basic-auth-file: {{ kube_users_dir }}/known_users.csv
 {% endif %}
+{% if kube_token_auth|default(true) %}
+  token-auth-file: {{ kube_token_dir }}/known_tokens.csv
+{% endif %}
 {% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %}
   oidc-issuer-url: {{ kube_oidc_url }}
   oidc-client-id: {{ kube_oidc_client_id }}
@@ -101,9 +110,13 @@ controllerManagerExtraArgs:
   node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
   node-monitor-period: {{ kube_controller_node_monitor_period }}
   pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }}
+  profiling: "{{ kube_profiling }}"
 {% if kube_feature_gates %}
   feature-gates: {{ kube_feature_gates|join(',') }}
 {% endif %}
+{% for key in kube_kubeadm_controller_extra_args %}
+  {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
+{% endfor %}
 {% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %}
 controllerManagerExtraVolumes:
 - name: openstackcacert
@@ -122,10 +135,8 @@ apiServerExtraVolumes:
   writable: true
 {% endif %}
 {% endif %}
-{% for key in kube_kubeadm_controller_extra_args %}
-  {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
-{% endfor %}
 schedulerExtraArgs:
+  profiling: "{{ kube_profiling }}"
 {% if kube_feature_gates %}
   feature-gates: {{ kube_feature_gates|join(',') }}
 {% endif %}
@@ -150,3 +161,7 @@ nodeRegistration:
 {% if container_manager == 'crio' %}
   criSocket: /var/run/crio/crio.sock
 {% endif %}
+{% if dynamic_kubelet_configuration %}
+featureGates:
+  DynamicKubeletConfig: true
+{% endif %}

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -33,7 +33,7 @@ spec:
     - --audit-log-maxage={{ audit_log_maxage }}
     - --audit-log-maxbackup={{ audit_log_maxbackups }}
     - --audit-log-maxsize={{ audit_log_maxsize }}
-    - --audit-policy-file={{ audit_policy_file }} 
+    - --audit-policy-file={{ audit_policy_file }}
 {% endif %}
     - --advertise-address={{ ip | default(ansible_default_ipv4.address) }}
     - --etcd-servers={{ etcd_access_addresses }}
@@ -58,16 +58,16 @@ spec:
     - --admission-control={{ kube_apiserver_admission_control | join(',') }}
 {% else %}
 {% if kube_apiserver_enable_admission_plugins|length > 0 %}
-    - --enable-admission-plugins={{ kube_apiserver_enable_admission_plugins | join(',') }} 
+    - --enable-admission-plugins={{ kube_apiserver_enable_admission_plugins | join(',') }}
 {% endif %}
 {% if kube_apiserver_disable_admission_plugins|length > 0 %}
-    - --disable-admission-plugins={{ kube_apiserver_disable_admission_plugins | join(',') }} 
+    - --disable-admission-plugins={{ kube_apiserver_disable_admission_plugins | join(',') }}
 {% endif %}
 {% endif %}
     - --service-cluster-ip-range={{ kube_service_addresses }}
     - --service-node-port-range={{ kube_apiserver_node_port_range }}
     - --client-ca-file={{ kube_cert_dir }}/ca.pem
-    - --profiling=false
+    - --profiling={{ kube_profiling }}
     - --repair-malformed-updates=false
     - --kubelet-client-certificate={{ kube_cert_dir }}/node-{{ inventory_hostname }}.pem
     - --kubelet-client-key={{ kube_cert_dir }}/node-{{ inventory_hostname }}-key.pem

roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2

@@ -37,7 +37,7 @@ spec:
     - --node-monitor-grace-period={{ kube_controller_node_monitor_grace_period }}
     - --node-monitor-period={{ kube_controller_node_monitor_period }}
     - --pod-eviction-timeout={{ kube_controller_pod_eviction_timeout }}
-    - --profiling=false
+    - --profiling={{ kube_profiling }}
     - --terminated-pod-gc-threshold=12500
     - --v={{ kube_log_level }}
 {% if rbac_enabled %}

roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2

@@ -32,7 +32,7 @@ spec:
     - --use-legacy-policy-config
     - --policy-config-file={{ kube_config_dir }}/kube-scheduler-policy.yaml
 {% endif %}
-    - --profiling=false
+    - --profiling={{ kube_profiling }}
     - --v={{ kube_log_level }}
 {% if kube_feature_gates %}
     - --feature-gates={{ kube_feature_gates|join(',') }}

roles/kubernetes/node/defaults/main.yml

@@ -86,6 +86,9 @@ kubelet_max_pods: 110
 ## Support custom flags to be passed to kubelet
 kubelet_custom_flags: []
 
+## Support custom flags to be passed to kubelet only on nodes, not masters
+kubelet_node_custom_flags: []
+
 # This setting is used for rkt based kubelet for deploying hyperkube
 # from a docker based registry ( controls --insecure and docker:// )
 ## Empty vaule for quay.io containers

roles/kubernetes/node/tasks/install.yml

@@ -7,8 +7,14 @@
     - kubeadm
 
 - name: install | Copy kubeadm binary from download dir
-  command: rsync -piu "{{ local_release_dir }}/kubeadm" "{{ bin_dir }}/kubeadm"
-  changed_when: false
+  synchronize:
+    src: "{{ local_release_dir }}/kubeadm"
+    dest: "{{ bin_dir }}/kubeadm"
+    compress: no
+    perms: yes
+    owner: no
+    group: no
+  delegate_to: "{{ inventory_hostname }}"
   when: kubeadm_enabled
   tags:
     - kubeadm

roles/kubernetes/node/tasks/install_host.yml

@@ -1,11 +1,18 @@
 ---
 
 - name: install | Copy kubelet binary from download dir
-  command: rsync -piu "{{ local_release_dir }}/hyperkube" "{{ bin_dir }}/kubelet"
-  changed_when: false
+  synchronize:
+    src: "{{ local_release_dir }}/hyperkube"
+    dest: "{{ bin_dir }}/kubelet"
+    compress: no
+    perms: yes
+    owner: no
+    group: no
+  delegate_to: "{{ inventory_hostname }}"
   tags:
     - hyperkube
     - upgrade
+  notify: restart kubelet
 
 - name: install | Set kubelet binary permissions
   file:
@@ -15,7 +22,6 @@
   tags:
     - hyperkube
     - upgrade
-  notify: restart kubelet
 
 - name: install | Copy socat wrapper for Container Linux
   command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/opt/bin {{ install_socat_image_repo }}:{{ install_socat_image_tag }}"

roles/kubernetes/node/tasks/main.yml

@@ -32,6 +32,13 @@
   tags:
     - kubelet
 
+- name: Make sure dynamic kubelet configuration directory is writeable
+  file:
+    path: "{{ dynamic_kubelet_configuration_dir }}"
+    mode: 0600
+    state: directory
+  when: dynamic_kubelet_configuration
+
 - name: Write kubelet config file (kubeadm)
   template:
     src: kubelet.kubeadm.env.j2
@@ -70,6 +77,8 @@
 
 - name: Verify if br_netfilter module exists
   shell: "modinfo br_netfilter"
+  environment:
+    PATH: "{{ ansible_env.PATH}}:/sbin"  # Make sure we can workaround RH's conservative path management
   register: modinfo_br_netfilter
   failed_when: modinfo_br_netfilter.rc not in [0, 1]
   changed_when: false

roles/kubernetes/node/templates/kubelet.kubeadm.env.j2

@@ -26,6 +26,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% if kubelet_authorization_mode_webhook %}
 --authorization-mode=Webhook \
 {% endif %}
+--enforce-node-allocatable={{ kubelet_enforce_node_allocatable }} \
 --client-ca-file={{ kube_cert_dir }}/ca.crt \
 --pod-manifest-path={{ kube_manifest_dir }} \
 --cadvisor-port={{ kube_cadvisor_port }} \
@@ -48,6 +49,9 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% else %}
 --fail-swap-on={{ kubelet_fail_swap_on|default(true)}} \
 {% endif %}
+{% if dynamic_kubelet_configuration %}
+--dynamic-config-dir={{ dynamic_kubelet_configuration_dir }} \
+{% endif %}
 --runtime-cgroups={{ kubelet_runtime_cgroups }} --kubelet-cgroups={{ kubelet_kubelet_cgroups }} \
 {% endset %}
 
@@ -90,7 +94,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% endif %}
 {% set all_node_labels = role_node_labels + inventory_node_labels %}
 
-KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kube_reserved }} --node-labels={{ all_node_labels | join(',') }} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}"
+KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kube_reserved }} --node-labels={{ all_node_labels | join(',') }} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
 {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium"] %}
 KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
 {% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %}

roles/kubernetes/node/templates/kubelet.standard.env.j2

@@ -120,7 +120,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {%   endif %}
 {% endif %}
 
-KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ kube_reserved }} --node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}"
+KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ kube_reserved }} --node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
 
 {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium"] %}
 KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"

roles/kubernetes/preinstall/tasks/0020-verify-settings.yml

@@ -127,3 +127,21 @@
   tags:
     - cloud-provider
     - facts
+
+- name: "Get current version of calico cluster version"
+  shell: "{{ bin_dir }}/calicoctl version  | grep 'Cluster Version' | awk '{ print $3}'"
+  register: calico_version_on_server
+  run_once: yes
+  delegate_to: "{{ groups['kube-master'][0] }}"
+
+
+- name: "Check that calico version is enought for upgrade"
+  assert:
+    that:
+      - calico_version_on_server.stdout|version_compare('v2.6.5', '>=')
+    msg: "Your version of calico is not fresh enough for upgrade. Minimum version v2.6.5"
+  when:
+    - 'calico_version_on_server.stdout is defined'
+    - 'calico_version_on_server.stdout != ""'
+    - inventory_hostname == groups['kube-master'][0]
+  run_once: yes