diff --git a/docs/operations/large-deployments.md b/docs/operations/large-deployments.md index 3182c27b8..834ea98dc 100644 --- a/docs/operations/large-deployments.md +++ b/docs/operations/large-deployments.md @@ -50,3 +50,20 @@ For a large scaled deployments, consider the following configuration changes: For example, when deploying 200 nodes, you may want to run ansible with ``--forks=50``, ``--timeout=600`` and define the ``retry_stagger: 60``. + +Sysctl tuning with floor semantics +---------------------------------- + +Built-in sysctl floors are only applied when the node's current value is +lower, so pre-tuned images and newer kernels are left alone. Use +`additional_sysctl_min` to add or override floors (set a key to `0` to +disable a default); use `additional_sysctl` for exact-value semantics. + +```yaml +additional_sysctl_min: + net.core.somaxconn: 65535 + kernel.pid_max: 0 +``` + +See `roles/kubespray_defaults/vars/main/main.yml` for the built-in floor +set. diff --git a/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml b/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml index 0830ef4c4..c69161c4a 100644 --- a/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml +++ b/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml @@ -125,17 +125,18 @@ - { name: vm.panic_on_oom, value: 0 } when: kubelet_protect_kernel_defaults | bool -- name: Read current sysctl values - command: sysctl -n {{ item.key }} +- name: Compute effective sysctl minimum values (defaults + user overrides) + set_fact: + _sysctl_minimum_values_effective: "{{ sysctl_minimum_values | combine(additional_sysctl_min) }}" + +- name: Read current sysctl values for floor-managed keys + command: sysctl --ignore -n {{ item.key }} register: sysctl_settings changed_when: false - vars: - # For integer sysctls only - sysctl_minimum_values: - fs.inotify.max_user_instances: 8192 - loop: "{{ sysctl_minimum_values | dict2items }}" + check_mode: false # read-only; must run under --check so later loop has rc/stdout + loop: "{{ _sysctl_minimum_values_effective | dict2items }}" -- name: Increase sysctl value if lower than minimum +- name: Increase sysctl value if lower than configured minimum ansible.posix.sysctl: sysctl_file: "{{ sysctl_file_path }}" name: "{{ item.item.key }}" @@ -143,8 +144,8 @@ state: present reload: true ignoreerrors: "{{ sysctl_ignore_unknown_keys }}" - when: item.stdout | int < item.item.value - loop: "{{ sysctl_settings.results }}" + when: item.stdout | int < item.item.value | int + loop: "{{ sysctl_settings.results | selectattr('stdout') }}" - name: Check dummy module community.general.modprobe: diff --git a/roles/kubespray_defaults/defaults/main/main.yml b/roles/kubespray_defaults/defaults/main/main.yml index 0f4b8467c..2655ee9e9 100644 --- a/roles/kubespray_defaults/defaults/main/main.yml +++ b/roles/kubespray_defaults/defaults/main/main.yml @@ -620,10 +620,21 @@ kubelet_protect_kernel_defaults: true # Set additional sysctl variables to modify Linux kernel variables, for example: # additional_sysctl: -# - { name: kernel.pid_max, value: 131072 } +# - { name: vm.swappiness, value: 10 } # +# `additional_sysctl` uses exact-value semantics. For floor semantics +# (only apply when the node's current value is lower), use +# `additional_sysctl_min` below. additional_sysctl: [] +# User-defined sysctl floors, merged on top of the built-in floor values +# (user values take precedence). Set a key to 0 to disable a default floor. +# Example: +# additional_sysctl_min: +# net.core.somaxconn: 65535 +# kernel.pid_max: 0 +additional_sysctl_min: {} + ## List of key=value pairs that describe feature gates for ## the k8s cluster. kube_feature_gates: [] diff --git a/roles/kubespray_defaults/vars/main/main.yml b/roles/kubespray_defaults/vars/main/main.yml index 1e17488b0..14231b368 100644 --- a/roles/kubespray_defaults/vars/main/main.yml +++ b/roles/kubespray_defaults/vars/main/main.yml @@ -20,6 +20,27 @@ etcd_supported_versions: kube_proxy_deployed: "{{ 'addon/kube-proxy' not in kubeadm_init_phases_skip }}" +# Sysctl floors: only applied when the node's current value is lower than +# the desired one, so pre-tuned images and newer kernels are not regressed. +# Values are aligned with OpenShift Node Tuning Operator and kops, and +# are outside the kubelet protect-kernel-defaults validation list. +sysctl_minimum_values: + fs.inotify.max_user_instances: 8192 + fs.inotify.max_user_watches: 65536 + kernel.pid_max: 4194304 + fs.aio-max-nr: 1048576 + vm.max_map_count: 262144 + net.netfilter.nf_conntrack_max: 1048576 + net.core.somaxconn: 32768 + net.core.netdev_max_backlog: 16384 + fs.file-max: 2097152 + net.ipv4.neigh.default.gc_thresh1: 8192 + net.ipv4.neigh.default.gc_thresh2: 32768 + net.ipv4.neigh.default.gc_thresh3: 65536 + net.ipv6.neigh.default.gc_thresh1: 8192 + net.ipv6.neigh.default.gc_thresh2: 32768 + net.ipv6.neigh.default.gc_thresh3: 65536 + # The lowest version allowed to upgrade from (same as calico_version in the previous branch) calico_min_version_required: "3.27.0"