From 604fac2295e22f04a3538021cb75de4e2365a99c Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Fri, 29 Jul 2022 16:11:34 -0400 Subject: [PATCH] Update task management to only do things with ready instances --- awx/main/scheduler/task_manager_models.py | 6 +++++- awx/main/tasks/system.py | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/awx/main/scheduler/task_manager_models.py b/awx/main/scheduler/task_manager_models.py index b84cdfcf82..b9187c0e9c 100644 --- a/awx/main/scheduler/task_manager_models.py +++ b/awx/main/scheduler/task_manager_models.py @@ -37,7 +37,11 @@ class TaskManagerInstances: def __init__(self, active_tasks, instances=None, instance_fields=('node_type', 'capacity', 'hostname', 'enabled')): self.instances_by_hostname = dict() if instances is None: - instances = Instance.objects.filter(hostname__isnull=False, enabled=True).exclude(node_type='hop').only(*instance_fields) + instances = ( + Instance.objects.filter(hostname__isnull=False, node_state=Instance.States.READY, enabled=True) + .exclude(node_type='hop') + .only('node_type', 'node_state', 'capacity', 'hostname', 'enabled') + ) for instance in instances: self.instances_by_hostname[instance.hostname] = TaskManagerInstance(instance) diff --git a/awx/main/tasks/system.py b/awx/main/tasks/system.py index f416bdd825..c2443b1a51 100644 --- a/awx/main/tasks/system.py +++ b/awx/main/tasks/system.py @@ -349,9 +349,13 @@ def _cleanup_images_and_files(**kwargs): logger.info(f'Performed local cleanup with kwargs {kwargs}, output:\n{stdout}') # if we are the first instance alphabetically, then run cleanup on execution nodes - checker_instance = Instance.objects.filter(node_type__in=['hybrid', 'control'], enabled=True, capacity__gt=0).order_by('-hostname').first() + checker_instance = ( + Instance.objects.filter(node_type__in=['hybrid', 'control'], node_state=Instance.States.READY, enabled=True, capacity__gt=0) + .order_by('-hostname') + .first() + ) if checker_instance and this_inst.hostname == checker_instance.hostname: - for inst in Instance.objects.filter(node_type='execution', enabled=True, capacity__gt=0): + for inst in Instance.objects.filter(node_type='execution', node_state=Instance.States.READY, enabled=True, capacity__gt=0): runner_cleanup_kwargs = inst.get_cleanup_task_kwargs(**kwargs) if not runner_cleanup_kwargs: continue