From b7d99749b8bade2a13f390353ec203ef59b0ab0a Mon Sep 17 00:00:00 2001 From: Kay Yan Date: Thu, 10 Sep 2026 15:46:33 +0800 Subject: [PATCH] preinstall: generalize minimum-value semantics for managed sysctls (#13224) * preinstall: generalize minimum-value semantics for managed sysctls Move the hardcoded sysctl floor map out of 0080-system-configurations.yml into a default variable, and introduce additional_sysctl_min so users can declare their own sysctl floors. Floor semantics only apply a managed value when the node's current value is lower; pre-tuned images (Bottlerocket, EKS AMIs, OpenShift RHCOS, hardened distros) and newer kernels that already set higher values are left alone. Default floors are aligned with prior art from the OpenShift Node Tuning Operator (TuneD '=>' floor syntax in the openshift / openshift-node profiles) and kops. None of the floor keys are part of the kubelet protect-kernel-defaults validation list, so the floor mechanism cannot interfere with kubelet startup. Keys not present on the running kernel/distro are skipped without failing. additional_sysctl (exact-value semantics) is unchanged for backward compatibility. Signed-off-by: Kay Yan * preinstall: refine sysctl floor handling Keep built-in floor values internal while preserving user overrides through additional_sysctl_min. Use sysctl --ignore for unknown keys and filter empty results without masking other read errors. Signed-off-by: Kay Yan --------- Signed-off-by: Kay Yan --- docs/operations/large-deployments.md | 17 +++++++++++++++ .../tasks/0080-system-configurations.yml | 21 ++++++++++--------- .../kubespray_defaults/defaults/main/main.yml | 13 +++++++++++- roles/kubespray_defaults/vars/main/main.yml | 21 +++++++++++++++++++ 4 files changed, 61 insertions(+), 11 deletions(-) diff --git a/docs/operations/large-deployments.md b/docs/operations/large-deployments.md index 3182c27b8..834ea98dc 100644 --- a/docs/operations/large-deployments.md +++ b/docs/operations/large-deployments.md @@ -50,3 +50,20 @@ For a large scaled deployments, consider the following configuration changes: For example, when deploying 200 nodes, you may want to run ansible with ``--forks=50``, ``--timeout=600`` and define the ``retry_stagger: 60``. + +Sysctl tuning with floor semantics +---------------------------------- + +Built-in sysctl floors are only applied when the node's current value is +lower, so pre-tuned images and newer kernels are left alone. Use +`additional_sysctl_min` to add or override floors (set a key to `0` to +disable a default); use `additional_sysctl` for exact-value semantics. + +```yaml +additional_sysctl_min: + net.core.somaxconn: 65535 + kernel.pid_max: 0 +``` + +See `roles/kubespray_defaults/vars/main/main.yml` for the built-in floor +set. diff --git a/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml b/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml index 0830ef4c4..c69161c4a 100644 --- a/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml +++ b/roles/kubernetes/preinstall/tasks/0080-system-configurations.yml @@ -125,17 +125,18 @@ - { name: vm.panic_on_oom, value: 0 } when: kubelet_protect_kernel_defaults | bool -- name: Read current sysctl values - command: sysctl -n {{ item.key }} +- name: Compute effective sysctl minimum values (defaults + user overrides) + set_fact: + _sysctl_minimum_values_effective: "{{ sysctl_minimum_values | combine(additional_sysctl_min) }}" + +- name: Read current sysctl values for floor-managed keys + command: sysctl --ignore -n {{ item.key }} register: sysctl_settings changed_when: false - vars: - # For integer sysctls only - sysctl_minimum_values: - fs.inotify.max_user_instances: 8192 - loop: "{{ sysctl_minimum_values | dict2items }}" + check_mode: false # read-only; must run under --check so later loop has rc/stdout + loop: "{{ _sysctl_minimum_values_effective | dict2items }}" -- name: Increase sysctl value if lower than minimum +- name: Increase sysctl value if lower than configured minimum ansible.posix.sysctl: sysctl_file: "{{ sysctl_file_path }}" name: "{{ item.item.key }}" @@ -143,8 +144,8 @@ state: present reload: true ignoreerrors: "{{ sysctl_ignore_unknown_keys }}" - when: item.stdout | int < item.item.value - loop: "{{ sysctl_settings.results }}" + when: item.stdout | int < item.item.value | int + loop: "{{ sysctl_settings.results | selectattr('stdout') }}" - name: Check dummy module community.general.modprobe: diff --git a/roles/kubespray_defaults/defaults/main/main.yml b/roles/kubespray_defaults/defaults/main/main.yml index 0f4b8467c..2655ee9e9 100644 --- a/roles/kubespray_defaults/defaults/main/main.yml +++ b/roles/kubespray_defaults/defaults/main/main.yml @@ -620,10 +620,21 @@ kubelet_protect_kernel_defaults: true # Set additional sysctl variables to modify Linux kernel variables, for example: # additional_sysctl: -# - { name: kernel.pid_max, value: 131072 } +# - { name: vm.swappiness, value: 10 } # +# `additional_sysctl` uses exact-value semantics. For floor semantics +# (only apply when the node's current value is lower), use +# `additional_sysctl_min` below. additional_sysctl: [] +# User-defined sysctl floors, merged on top of the built-in floor values +# (user values take precedence). Set a key to 0 to disable a default floor. +# Example: +# additional_sysctl_min: +# net.core.somaxconn: 65535 +# kernel.pid_max: 0 +additional_sysctl_min: {} + ## List of key=value pairs that describe feature gates for ## the k8s cluster. kube_feature_gates: [] diff --git a/roles/kubespray_defaults/vars/main/main.yml b/roles/kubespray_defaults/vars/main/main.yml index 1e17488b0..14231b368 100644 --- a/roles/kubespray_defaults/vars/main/main.yml +++ b/roles/kubespray_defaults/vars/main/main.yml @@ -20,6 +20,27 @@ etcd_supported_versions: kube_proxy_deployed: "{{ 'addon/kube-proxy' not in kubeadm_init_phases_skip }}" +# Sysctl floors: only applied when the node's current value is lower than +# the desired one, so pre-tuned images and newer kernels are not regressed. +# Values are aligned with OpenShift Node Tuning Operator and kops, and +# are outside the kubelet protect-kernel-defaults validation list. +sysctl_minimum_values: + fs.inotify.max_user_instances: 8192 + fs.inotify.max_user_watches: 65536 + kernel.pid_max: 4194304 + fs.aio-max-nr: 1048576 + vm.max_map_count: 262144 + net.netfilter.nf_conntrack_max: 1048576 + net.core.somaxconn: 32768 + net.core.netdev_max_backlog: 16384 + fs.file-max: 2097152 + net.ipv4.neigh.default.gc_thresh1: 8192 + net.ipv4.neigh.default.gc_thresh2: 32768 + net.ipv4.neigh.default.gc_thresh3: 65536 + net.ipv6.neigh.default.gc_thresh1: 8192 + net.ipv6.neigh.default.gc_thresh2: 32768 + net.ipv6.neigh.default.gc_thresh3: 65536 + # The lowest version allowed to upgrade from (same as calico_version in the previous branch) calico_min_version_required: "3.27.0"