diff --git a/awx/main/utils/autoscale.py b/awx/main/utils/autoscale.py new file mode 100644 index 0000000000..49f07a251c --- /dev/null +++ b/awx/main/utils/autoscale.py @@ -0,0 +1,27 @@ +from celery.utils.log import get_logger +from celery.worker.autoscale import Autoscaler, AUTOSCALE_KEEPALIVE +from django.conf import settings +import psutil + +logger = get_logger('awx.main.tasks') + + +class DynamicAutoScaler(Autoscaler): + + def __init__(self, pool, max_concurrency, min_concurrency=0, worker=None, + keepalive=AUTOSCALE_KEEPALIVE, mutex=None): + super(DynamicAutoScaler, self).__init__(pool, max_concurrency, + min_concurrency, worker, + keepalive, mutex) + settings_absmem = getattr(settings, 'SYSTEM_TASK_ABS_MEM', None) + if settings_absmem is not None: + total_memory_gb = int(settings_absmem) + else: + total_memory_gb = (psutil.virtual_memory().total >> 30) + 1 # noqa: round up + + # 5 workers per GB of total memory + self.max_concurrency = min(max_concurrency, (total_memory_gb * 5)) + logger.warn('celery worker dynamic --autoscale={},{}'.format( + self.max_concurrency, + self.min_concurrency + )) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 4fdc097c33..7adde14d26 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -458,6 +458,7 @@ CELERY_TRACK_STARTED = True CELERYD_TASK_TIME_LIMIT = None CELERYD_TASK_SOFT_TIME_LIMIT = None CELERYD_POOL_RESTARTS = True +CELERYD_AUTOSCALER = 'awx.main.utils.autoscale:DynamicAutoScaler' CELERY_RESULT_BACKEND = 'djcelery.backends.database:DatabaseBackend' CELERY_IMPORTS = ('awx.main.scheduler.tasks',) CELERY_QUEUES = (