mirror of
https://github.com/ansible/awx.git
synced 2026-03-19 09:57:33 -02:30
Fixup conversion of memory and cpu settings to support k8s resource request format (#11725)
fix memory and cpu settings to suport k8s resource request format * fix conversion of memory setting to bytes This setting has not been getting set by default, and needed some fixing up to be compatible with setting the memory in the same way as we set it in the operator, as well as with other changes from last year which assume that ansible runner is returning memory in bytes. This way we can start setting this setting in the operator, and get a more accurate reflection of how much memory is available to the control pod in k8s. On platforms where services are all sharing memory, we deduct a penalty from the memory available. On k8s we don't need to do this because the web, redis, and task containers each have memory allocated to them. * Support CPU setting expressed in units used by k8s This setting has not been getting set by default, and needed some fixing up to be compatible with setting the CPU resource request/limits in the same way as we set it in the resource requests/limits. This way we can start setting this setting in the operator, and get a more accurate reflection of how much cpu is available to the control pod in k8s. Because cpu on k8s can be partial cores, migrate cpu field to decimal. k8s does not allow granularity of less than 100m (equivalent to 0.1 cores), so only store up to 1 decimal place. fix analytics to deal with decimal cpu need to use DjangoJSONEncoder when Decimal fields in data passed to json.dumps
This commit is contained in:
Elijah DeLee
GitHub
@@ -692,28 +692,33 @@ def parse_yaml_or_json(vars_str, silent_failure=True):
|
||||
return vars_dict
|
||||
|
||||
|
||||
def get_cpu_effective_capacity(cpu_count):
|
||||
from django.conf import settings
|
||||
def convert_cpu_str_to_decimal_cpu(cpu_str):
|
||||
"""Convert a string indicating cpu units to decimal.
|
||||
|
||||
settings_abscpu = getattr(settings, 'SYSTEM_TASK_ABS_CPU', None)
|
||||
env_abscpu = os.getenv('SYSTEM_TASK_ABS_CPU', None)
|
||||
Useful for dealing with cpu setting that may be expressed in units compatible with
|
||||
kubernetes.
|
||||
|
||||
if env_abscpu is not None:
|
||||
return int(env_abscpu)
|
||||
elif settings_abscpu is not None:
|
||||
return int(settings_abscpu)
|
||||
See https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource/#cpu-units
|
||||
"""
|
||||
cpu = cpu_str
|
||||
millicores = False
|
||||
|
||||
settings_forkcpu = getattr(settings, 'SYSTEM_TASK_FORKS_CPU', None)
|
||||
env_forkcpu = os.getenv('SYSTEM_TASK_FORKS_CPU', None)
|
||||
if cpu_str[-1] == 'm':
|
||||
cpu = cpu_str[:-1]
|
||||
millicores = True
|
||||
|
||||
if env_forkcpu:
|
||||
forkcpu = int(env_forkcpu)
|
||||
elif settings_forkcpu:
|
||||
forkcpu = int(settings_forkcpu)
|
||||
else:
|
||||
forkcpu = 4
|
||||
try:
|
||||
cpu = float(cpu)
|
||||
except ValueError:
|
||||
cpu = 1.0
|
||||
millicores = False
|
||||
logger.warning(f"Could not convert SYSTEM_TASK_ABS_CPU {cpu_str} to a decimal number, falling back to default of 1 cpu")
|
||||
|
||||
return cpu_count * forkcpu
|
||||
if millicores:
|
||||
cpu = cpu / 1000
|
||||
|
||||
# Per kubernetes docs, fractional CPU less than .1 are not allowed
|
||||
return max(0.1, round(cpu, 1))
|
||||
|
||||
|
||||
def get_corrected_cpu(cpu_count): # formerlly get_cpu_capacity
|
||||
@@ -725,34 +730,70 @@ def get_corrected_cpu(cpu_count): # formerlly get_cpu_capacity
|
||||
settings_abscpu = getattr(settings, 'SYSTEM_TASK_ABS_CPU', None)
|
||||
env_abscpu = os.getenv('SYSTEM_TASK_ABS_CPU', None)
|
||||
|
||||
if env_abscpu is not None or settings_abscpu is not None:
|
||||
return 0
|
||||
if env_abscpu is not None:
|
||||
return convert_cpu_str_to_decimal_cpu(env_abscpu)
|
||||
elif settings_abscpu is not None:
|
||||
return convert_cpu_str_to_decimal_cpu(settings_abscpu)
|
||||
|
||||
return cpu_count # no correction
|
||||
|
||||
|
||||
def get_mem_effective_capacity(mem_mb):
|
||||
def get_cpu_effective_capacity(cpu_count):
|
||||
from django.conf import settings
|
||||
|
||||
settings_absmem = getattr(settings, 'SYSTEM_TASK_ABS_MEM', None)
|
||||
env_absmem = os.getenv('SYSTEM_TASK_ABS_MEM', None)
|
||||
cpu_count = get_corrected_cpu(cpu_count)
|
||||
|
||||
if env_absmem is not None:
|
||||
return int(env_absmem)
|
||||
elif settings_absmem is not None:
|
||||
return int(settings_absmem)
|
||||
settings_forkcpu = getattr(settings, 'SYSTEM_TASK_FORKS_CPU', None)
|
||||
env_forkcpu = os.getenv('SYSTEM_TASK_FORKS_CPU', None)
|
||||
|
||||
settings_forkmem = getattr(settings, 'SYSTEM_TASK_FORKS_MEM', None)
|
||||
env_forkmem = os.getenv('SYSTEM_TASK_FORKS_MEM', None)
|
||||
|
||||
if env_forkmem:
|
||||
forkmem = int(env_forkmem)
|
||||
elif settings_forkmem:
|
||||
forkmem = int(settings_forkmem)
|
||||
if env_forkcpu:
|
||||
forkcpu = int(env_forkcpu)
|
||||
elif settings_forkcpu:
|
||||
forkcpu = int(settings_forkcpu)
|
||||
else:
|
||||
forkmem = 100
|
||||
forkcpu = 4
|
||||
|
||||
return max(1, ((mem_mb // 1024 // 1024) - 2048) // forkmem)
|
||||
return max(1, int(cpu_count * forkcpu))
|
||||
|
||||
|
||||
def convert_mem_str_to_bytes(mem_str):
|
||||
"""Convert string with suffix indicating units to memory in bytes (base 2)
|
||||
|
||||
Useful for dealing with memory setting that may be expressed in units compatible with
|
||||
kubernetes.
|
||||
|
||||
See https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory
|
||||
"""
|
||||
# If there is no suffix, the memory sourced from the request is in bytes
|
||||
if mem_str.isdigit():
|
||||
return int(mem_str)
|
||||
|
||||
conversions = {
|
||||
'Ei': lambda x: x * 2**60,
|
||||
'E': lambda x: x * 10**18,
|
||||
'Pi': lambda x: x * 2**50,
|
||||
'P': lambda x: x * 10**15,
|
||||
'Ti': lambda x: x * 2**40,
|
||||
'T': lambda x: x * 10**12,
|
||||
'Gi': lambda x: x * 2**30,
|
||||
'G': lambda x: x * 10**9,
|
||||
'Mi': lambda x: x * 2**20,
|
||||
'M': lambda x: x * 10**6,
|
||||
'Ki': lambda x: x * 2**10,
|
||||
'K': lambda x: x * 10**3,
|
||||
}
|
||||
mem = 0
|
||||
mem_unit = None
|
||||
for i, char in enumerate(mem_str):
|
||||
if not char.isdigit():
|
||||
mem_unit = mem_str[i:]
|
||||
mem = int(mem_str[:i])
|
||||
break
|
||||
if not mem_unit or mem_unit not in conversions.keys():
|
||||
error = f"Unsupported value for SYSTEM_TASK_ABS_MEM: {mem_str}, memory must be expressed in bytes or with known suffix: {conversions.keys()}. Falling back to 1 byte"
|
||||
logger.warning(error)
|
||||
return 1
|
||||
return max(1, conversions[mem_unit](mem))
|
||||
|
||||
|
||||
def get_corrected_memory(memory):
|
||||
@@ -761,12 +802,48 @@ def get_corrected_memory(memory):
|
||||
settings_absmem = getattr(settings, 'SYSTEM_TASK_ABS_MEM', None)
|
||||
env_absmem = os.getenv('SYSTEM_TASK_ABS_MEM', None)
|
||||
|
||||
if env_absmem is not None or settings_absmem is not None:
|
||||
return 0
|
||||
# Runner returns memory in bytes
|
||||
# so we convert memory from settings to bytes as well.
|
||||
if env_absmem is not None:
|
||||
return convert_mem_str_to_bytes(env_absmem)
|
||||
elif settings_absmem is not None:
|
||||
return convert_mem_str_to_bytes(settings_absmem)
|
||||
|
||||
return memory
|
||||
|
||||
|
||||
def get_mem_effective_capacity(mem_bytes):
|
||||
from django.conf import settings
|
||||
|
||||
mem_bytes = get_corrected_memory(mem_bytes)
|
||||
|
||||
settings_mem_mb_per_fork = getattr(settings, 'SYSTEM_TASK_FORKS_MEM', None)
|
||||
env_mem_mb_per_fork = os.getenv('SYSTEM_TASK_FORKS_MEM', None)
|
||||
|
||||
if env_mem_mb_per_fork:
|
||||
mem_mb_per_fork = int(env_mem_mb_per_fork)
|
||||
elif settings_mem_mb_per_fork:
|
||||
mem_mb_per_fork = int(settings_mem_mb_per_fork)
|
||||
else:
|
||||
mem_mb_per_fork = 100
|
||||
|
||||
# Per docs, deduct 2GB of memory from the available memory
|
||||
# to cover memory consumption of background tasks when redis/web etc are colocated with
|
||||
# the other control processes
|
||||
memory_penalty_bytes = 2147483648
|
||||
if settings.IS_K8S:
|
||||
# In k8s, this is dealt with differently because
|
||||
# redis and the web containers have their own memory allocation
|
||||
memory_penalty_bytes = 0
|
||||
|
||||
# convert memory to megabytes because our setting of how much memory we
|
||||
# should allocate per fork is in megabytes
|
||||
mem_mb = (mem_bytes - memory_penalty_bytes) // 2**20
|
||||
max_forks_based_on_memory = mem_mb // mem_mb_per_fork
|
||||
|
||||
return max(1, max_forks_based_on_memory)
|
||||
|
||||
|
||||
_inventory_updates = threading.local()
|
||||
_task_manager = threading.local()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user