From 95e2bde15bd86b9607e98529b6e4eaf4d88642c4 Mon Sep 17 00:00:00 2001 From: Dann Bohn Date: Fri, 16 Feb 2018 16:20:08 -0500 Subject: [PATCH 1/9] set nodeName to "{{ inventory_hostname }}" in kubeadm-config --- roles/kubernetes/master/templates/kubeadm-config.yaml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 index ed1cc7add..dc842a5e6 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 @@ -83,3 +83,5 @@ apiServerCertSANs: {% endfor %} certificatesDir: {{ kube_config_dir }}/ssl unifiedControlPlaneImage: "{{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}" +nodeName: {{ inventory_hostname }} + From aa30fa8009e1b41176dbc1be3f0c99703171a7e6 Mon Sep 17 00:00:00 2001 From: gorazio Date: Tue, 20 Mar 2018 08:47:36 +0300 Subject: [PATCH 2/9] Add prometheus annotations to spec in ingress Added annotations from metadata to spec.template.metadata. Without it, pod does not get any annotations, and Prometheus didn't see it --- .../ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 index 7fd3a946c..3a4c7860b 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 @@ -20,6 +20,9 @@ spec: labels: k8s-app: ingress-nginx version: v{{ ingress_nginx_controller_image_tag }} + annotations: + prometheus.io/port: '10254' + prometheus.io/scrape: 'true' spec: containers: - name: ingress-nginx-controller From 96e46c4209003bfa61decf9c40eed670d6eed704 Mon Sep 17 00:00:00 2001 From: gorazio Date: Tue, 20 Mar 2018 10:23:50 +0300 Subject: [PATCH 3/9] bump after CLA signing --- .../ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 index 3a4c7860b..f8fac3b09 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 @@ -73,3 +73,4 @@ spec: {% if rbac_enabled %} serviceAccountName: ingress-nginx {% endif %} + From 8b71ef8ceb46bcc93ad547f4ccfd452c53d40bee Mon Sep 17 00:00:00 2001 From: Erwan Miran Date: Wed, 21 Mar 2018 09:19:05 +0100 Subject: [PATCH 4/9] Labels from role (node-role.k8s.io/node) and labels from inventory are merged into node-labels parameter in kubelet --- docs/vars.md | 2 ++ .../node/templates/kubelet.standard.env.j2 | 16 ++++++++++++---- tests/ansible.cfg | 1 + 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/vars.md b/docs/vars.md index 5ea76b0e5..f4956c882 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -118,6 +118,8 @@ Stack](https://github.com/kubernetes-incubator/kubespray/blob/master/docs/dns-st * *kubelet_cgroup_driver* - Allows manual override of the cgroup-driver option for Kubelet. By default autodetection is used to match Docker configuration. +* *node_labels* - Labels applied to nodes via kubelet --node-labels parameter. + For example, labels can be set in the inventory as variables or more widely in group_vars ##### Custom flags for Kube Components For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments. This can be done by providing a list of flags. Example: diff --git a/roles/kubernetes/node/templates/kubelet.standard.env.j2 b/roles/kubernetes/node/templates/kubelet.standard.env.j2 index d33adfba7..05874a5de 100644 --- a/roles/kubernetes/node/templates/kubelet.standard.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.standard.env.j2 @@ -81,16 +81,24 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% endif %} {# Kubelet node labels #} +{% set role_node_labels = [] %} {% if inventory_hostname in groups['kube-master'] %} -{% set node_labels %}--node-labels=node-role.kubernetes.io/master=true{% endset %} +{% do role_node_labels.append('node-role.kubernetes.io/master=true') %} {% if not standalone_kubelet|bool %} -{% set node_labels %}{{ node_labels }},node-role.kubernetes.io/node=true{% endset %} +{% do role_node_labels.append('node-role.kubernetes.io/node=true') %} {% endif %} {% else %} -{% set node_labels %}--node-labels=node-role.kubernetes.io/node=true{% endset %} +{% do role_node_labels.append('node-role.kubernetes.io/node=true') %} {% endif %} +{% set inventory_node_labels = [] %} +{% if node_labels is defined %} +{% for labelname, labelvalue in node_labels.iteritems() %} +{% do inventory_node_labels.append(labelname + '=' + labelvalue) %} +{% endfor %} +{% endif %} +{% set all_node_labels = role_node_labels + inventory_node_labels %} -KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ kube_reserved }} {{ node_labels }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}" +KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ kube_reserved }} --node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}" {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium"] %} KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" {% elif kube_network_plugin is defined and kube_network_plugin == "weave" %} diff --git a/tests/ansible.cfg b/tests/ansible.cfg index 9e734403e..9c4057529 100644 --- a/tests/ansible.cfg +++ b/tests/ansible.cfg @@ -10,3 +10,4 @@ fact_caching_connection = /tmp stdout_callback = skippy library = ./library:../library callback_whitelist = profile_tasks +jinja2_extensions = jinja2.ext.do From 9fa995ac9d595cc75695fb8b977ac2dd75328c46 Mon Sep 17 00:00:00 2001 From: Dann Bohn Date: Fri, 23 Mar 2018 08:33:25 -0400 Subject: [PATCH 5/9] only sets nodeName in kubeadm-config when kube_override_hostname is set --- roles/kubernetes/master/templates/kubeadm-config.yaml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 index dc842a5e6..b2d2cd2e7 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 @@ -83,5 +83,7 @@ apiServerCertSANs: {% endfor %} certificatesDir: {{ kube_config_dir }}/ssl unifiedControlPlaneImage: "{{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}" +{% if kube_override_hostname|default('') %} nodeName: {{ inventory_hostname }} +{% endif %} From 1d0415a6cf5015373a35f2f50adc7749d1a014e0 Mon Sep 17 00:00:00 2001 From: Dann Bohn Date: Sat, 24 Mar 2018 13:29:07 -0400 Subject: [PATCH 6/9] fixes typo in kube_override_hostname for kubeadm --- roles/kubernetes/master/templates/kubeadm-config.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 index b2d2cd2e7..5ea5d712c 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 @@ -84,6 +84,6 @@ apiServerCertSANs: certificatesDir: {{ kube_config_dir }}/ssl unifiedControlPlaneImage: "{{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}" {% if kube_override_hostname|default('') %} -nodeName: {{ inventory_hostname }} +nodeName: {{ kube_override_hostname }} {% endif %} From 72a42238849d00f1e40aee50ea11c4d628ddb272 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Wed, 28 Mar 2018 16:26:36 +0300 Subject: [PATCH 7/9] Write cloud-config during kubelet configuration This file should only be updated during kubelet upgrade so that master components are not accidentally restarted first during preinstall stage. --- roles/kubernetes/node/tasks/main.yml | 13 +++++++++++++ roles/kubernetes/preinstall/tasks/main.yml | 13 ------------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index 4d5fa5df5..78e6d92d6 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -134,6 +134,19 @@ tags: - kube-proxy +- name: Write cloud-config + template: + src: "{{ cloud_provider }}-cloud-config.j2" + dest: "{{ kube_config_dir }}/cloud_config" + group: "{{ kube_cert_group }}" + mode: 0640 + when: + - cloud_provider is defined + - cloud_provider in [ 'openstack', 'azure', 'vsphere' ] + notify: restart kubelet + tags: + - cloud-provider + # reload-systemd - meta: flush_handlers diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml index f23040751..aca0c9606 100644 --- a/roles/kubernetes/preinstall/tasks/main.yml +++ b/roles/kubernetes/preinstall/tasks/main.yml @@ -256,19 +256,6 @@ tags: - bootstrap-os -- name: Write cloud-config - template: - src: "{{ cloud_provider }}-cloud-config.j2" - dest: "{{ kube_config_dir }}/cloud_config" - group: "{{ kube_cert_group }}" - mode: 0640 - when: - - inventory_hostname in groups['k8s-cluster'] - - cloud_provider is defined - - cloud_provider in [ 'openstack', 'azure', 'vsphere' ] - tags: - - cloud-provider - - import_tasks: etchosts.yml tags: - bootstrap-os From daeeae1a91aad8d633304f5961ee33df5ed813f1 Mon Sep 17 00:00:00 2001 From: Kuldip Madnani Date: Thu, 29 Mar 2018 11:37:32 -0500 Subject: [PATCH 8/9] Added retries in pre-upgrade.yml and retries while applying kube-dns.yml (#2553) * Added retries in pre-upgrade.yml and retries while applying kube-dns.yml * Removed trailing spaces --- roles/kubernetes-apps/ansible/tasks/main.yml | 4 ++++ roles/kubernetes/master/tasks/pre-upgrade.yml | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/roles/kubernetes-apps/ansible/tasks/main.yml b/roles/kubernetes-apps/ansible/tasks/main.yml index 55d417982..c03a78722 100644 --- a/roles/kubernetes-apps/ansible/tasks/main.yml +++ b/roles/kubernetes-apps/ansible/tasks/main.yml @@ -50,6 +50,10 @@ - dns_mode != 'none' - inventory_hostname == groups['kube-master'][0] - not item|skipped + register: resource_result + until: resource_result|succeeded + retries: 4 + delay: 5 tags: - dnsmasq diff --git a/roles/kubernetes/master/tasks/pre-upgrade.yml b/roles/kubernetes/master/tasks/pre-upgrade.yml index 3a9fe6417..56e57b015 100644 --- a/roles/kubernetes/master/tasks/pre-upgrade.yml +++ b/roles/kubernetes/master/tasks/pre-upgrade.yml @@ -30,4 +30,7 @@ with_items: - ["kube-apiserver", "kube-controller-manager", "kube-scheduler"] when: kube_apiserver_manifest_replaced.changed - run_once: true + register: remove_master_container + retries: 4 + until: remove_master_container.rc == 0 + delay: 5 \ No newline at end of file From 4d85e3765e1c3aefdca224edf3b60e0b0e8e5ebb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=AE=8F?= Date: Fri, 30 Mar 2018 09:19:00 +0800 Subject: [PATCH 9/9] remove redundancy code --- roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 index 7c8e0062d..57c2269a9 100644 --- a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 +++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 @@ -48,7 +48,6 @@ spec: {% elif kube_proxy_mode == 'ipvs' %} - --masquerade-all - --feature-gates=SupportIPVSProxyMode=true - - --proxy-mode=ipvs - --ipvs-min-sync-period=5s - --ipvs-sync-period=5s - --ipvs-scheduler=rr