preinstall: generalize minimum-value semantics for managed sysctls (#13224)

* preinstall: generalize minimum-value semantics for managed sysctls

Move the hardcoded sysctl floor map out of 0080-system-configurations.yml
into a default variable, and introduce additional_sysctl_min so users can
declare their own sysctl floors. Floor semantics only apply a managed
value when the node's current value is lower; pre-tuned images (Bottlerocket,
EKS AMIs, OpenShift RHCOS, hardened distros) and newer kernels that
already set higher values are left alone.

Default floors are aligned with prior art from the OpenShift Node Tuning
Operator (TuneD '=>' floor syntax in the openshift / openshift-node
profiles) and kops. None of the floor keys are part of the kubelet
protect-kernel-defaults validation list, so the floor mechanism cannot
interfere with kubelet startup. Keys not present on the running
kernel/distro are skipped without failing.

additional_sysctl (exact-value semantics) is unchanged for backward
compatibility.

Signed-off-by: Kay Yan <kay.yan@daocloud.io>

* preinstall: refine sysctl floor handling

Keep built-in floor values internal while preserving user overrides
through additional_sysctl_min.

Use sysctl --ignore for unknown keys and filter empty results without
masking other read errors.

Signed-off-by: Kay Yan <kay.yan@daocloud.io>

---------

Signed-off-by: Kay Yan <kay.yan@daocloud.io>

docs/operations/large-deployments.md

@@ -50,3 +50,20 @@ For a large scaled deployments, consider the following configuration changes:
 
 For example, when deploying 200 nodes, you may want to run ansible with
 ``--forks=50``, ``--timeout=600`` and define the ``retry_stagger: 60``.
+
+Sysctl tuning with floor semantics
+----------------------------------
+
+Built-in sysctl floors are only applied when the node's current value is
+lower, so pre-tuned images and newer kernels are left alone. Use
+`additional_sysctl_min` to add or override floors (set a key to `0` to
+disable a default); use `additional_sysctl` for exact-value semantics.
+
+```yaml
+additional_sysctl_min:
+  net.core.somaxconn: 65535
+  kernel.pid_max: 0
+```
+
+See `roles/kubespray_defaults/vars/main/main.yml` for the built-in floor
+set.

roles/kubernetes/preinstall/tasks/0080-system-configurations.yml

@@ -125,17 +125,18 @@
     - { name: vm.panic_on_oom, value: 0 }
   when: kubelet_protect_kernel_defaults | bool
 
-- name: Read current sysctl values
-  command: sysctl -n {{ item.key }}
+- name: Compute effective sysctl minimum values (defaults + user overrides)
+  set_fact:
+    _sysctl_minimum_values_effective: "{{ sysctl_minimum_values | combine(additional_sysctl_min) }}"
+
+- name: Read current sysctl values for floor-managed keys
+  command: sysctl --ignore -n {{ item.key }}
   register: sysctl_settings
   changed_when: false
-  vars:
-    # For integer sysctls only
-    sysctl_minimum_values:
-      fs.inotify.max_user_instances: 8192
-  loop: "{{ sysctl_minimum_values | dict2items }}"
+  check_mode: false  # read-only; must run under --check so later loop has rc/stdout
+  loop: "{{ _sysctl_minimum_values_effective | dict2items }}"
 
-- name: Increase sysctl value if lower than minimum
+- name: Increase sysctl value if lower than configured minimum
   ansible.posix.sysctl:
     sysctl_file: "{{ sysctl_file_path }}"
     name: "{{ item.item.key }}"
@@ -143,8 +144,8 @@
     state: present
     reload: true
     ignoreerrors: "{{ sysctl_ignore_unknown_keys }}"
-  when: item.stdout | int < item.item.value
-  loop: "{{ sysctl_settings.results }}"
+  when: item.stdout | int < item.item.value | int
+  loop: "{{ sysctl_settings.results | selectattr('stdout') }}"
 
 - name: Check dummy module
   community.general.modprobe:

roles/kubespray_defaults/defaults/main/main.yml

@@ -620,10 +620,21 @@ kubelet_protect_kernel_defaults: true
 
 # Set additional sysctl variables to modify Linux kernel variables, for example:
 # additional_sysctl:
-#  - { name: kernel.pid_max, value: 131072 }
+#  - { name: vm.swappiness, value: 10 }
 #
+# `additional_sysctl` uses exact-value semantics. For floor semantics
+# (only apply when the node's current value is lower), use
+# `additional_sysctl_min` below.
 additional_sysctl: []
 
+# User-defined sysctl floors, merged on top of the built-in floor values
+# (user values take precedence). Set a key to 0 to disable a default floor.
+# Example:
+# additional_sysctl_min:
+#   net.core.somaxconn: 65535
+#   kernel.pid_max: 0
+additional_sysctl_min: {}
+
 ## List of key=value pairs that describe feature gates for
 ## the k8s cluster.
 kube_feature_gates: []

roles/kubespray_defaults/vars/main/main.yml

@@ -20,6 +20,27 @@ etcd_supported_versions:
 
 kube_proxy_deployed: "{{ 'addon/kube-proxy' not in kubeadm_init_phases_skip }}"
 
+# Sysctl floors: only applied when the node's current value is lower than
+# the desired one, so pre-tuned images and newer kernels are not regressed.
+# Values are aligned with OpenShift Node Tuning Operator and kops, and
+# are outside the kubelet protect-kernel-defaults validation list.
+sysctl_minimum_values:
+  fs.inotify.max_user_instances: 8192
+  fs.inotify.max_user_watches: 65536
+  kernel.pid_max: 4194304
+  fs.aio-max-nr: 1048576
+  vm.max_map_count: 262144
+  net.netfilter.nf_conntrack_max: 1048576
+  net.core.somaxconn: 32768
+  net.core.netdev_max_backlog: 16384
+  fs.file-max: 2097152
+  net.ipv4.neigh.default.gc_thresh1: 8192
+  net.ipv4.neigh.default.gc_thresh2: 32768
+  net.ipv4.neigh.default.gc_thresh3: 65536
+  net.ipv6.neigh.default.gc_thresh1: 8192
+  net.ipv6.neigh.default.gc_thresh2: 32768
+  net.ipv6.neigh.default.gc_thresh3: 65536
+
 # The lowest version allowed to upgrade from (same as calico_version in the previous branch)
 calico_min_version_required: "3.27.0"