From 8bc1490368d0a24a0ca6ad969f0befeac6d70b8f Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Thu, 11 May 2017 14:34:40 -0400 Subject: [PATCH] Increase test coverage for task scheduler inventory updates Also fixes some bugs found in that process --- awx/main/models/inventory.py | 8 +- awx/main/scheduler/__init__.py | 22 ++---- awx/main/scheduler/dependency_graph.py | 2 +- .../task_management/test_scheduler.py | 76 ++++++++++++++++++- 4 files changed, 84 insertions(+), 24 deletions(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 02f1d2acd2..5096010aef 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -1384,12 +1384,12 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin): @property def preferred_instance_groups(self): - if self.inventory is not None and self.inventory.organization is not None: - organization_groups = [x for x in self.inventory.organization.instance_groups.all()] + if self.inventory_source.inventory is not None and self.inventory_source.inventory.organization is not None: + organization_groups = [x for x in self.inventory_source.inventory.organization.instance_groups.all()] else: organization_groups = [] - if self.inventory is not None: - inventory_groups = [x for x in self.inventory.instance_groups.all()] + if self.inventory_source.inventory is not None: + inventory_groups = [x for x in self.inventory_source.inventory.instance_groups.all()] template_groups = [x for x in super(InventoryUpdate, self).preferred_instance_groups] return template_groups + inventory_groups + organization_groups diff --git a/awx/main/scheduler/__init__.py b/awx/main/scheduler/__init__.py index 28207d27bf..20da8bd72d 100644 --- a/awx/main/scheduler/__init__.py +++ b/awx/main/scheduler/__init__.py @@ -239,22 +239,14 @@ class TaskManager(): return project_task def create_inventory_update(self, task, inventory_source_task): - dep = InventorySource.objects.get(id=inventory_source_task.id).create_inventory_update(launch_type='dependency') + inventory_task = InventorySource.objects.get(id=inventory_source_task.id).create_inventory_update(launch_type='inventory_taskendency') - dep.created = task.created - timedelta(seconds=2) - dep.status = 'pending' - dep.save() - - ''' - Update internal datastructures with the newly created inventory update - ''' - # Should be only 1 inventory update. The one for the job (task) - latest_inventory_updates = self.get_latest_inventory_update_tasks([task]) - self.process_latest_inventory_updates(latest_inventory_updates) - - inventory_sources = self.get_inventory_source_tasks([task]) - self.process_inventory_sources(inventory_sources) - return dep + inventory_task.created = task.created - timedelta(seconds=2) + inventory_task.status = 'pending' + inventory_task.save() + # inventory_sources = self.get_inventory_source_tasks([task]) + # self.process_inventory_sources(inventory_sources) + return inventory_task def capture_chain_failure_dependencies(self, task, dependencies): for dep in dependencies: diff --git a/awx/main/scheduler/dependency_graph.py b/awx/main/scheduler/dependency_graph.py index db21c322fa..44ab2225e3 100644 --- a/awx/main/scheduler/dependency_graph.py +++ b/awx/main/scheduler/dependency_graph.py @@ -133,7 +133,7 @@ class DependencyGraph(object): if type(job) is ProjectUpdate: self.mark_project_update(job) elif type(job) is InventoryUpdate: - self.mark_inventory_update(job.inventory_source__inventory_id) + self.mark_inventory_update(job.inventory_source.inventory_id) self.mark_inventory_source_update(job.inventory_source_id) elif type(job) is Job: self.mark_job_template_job(job) diff --git a/awx/main/tests/functional/task_management/test_scheduler.py b/awx/main/tests/functional/task_management/test_scheduler.py index 425b9a2763..c345eb85bb 100644 --- a/awx/main/tests/functional/task_management/test_scheduler.py +++ b/awx/main/tests/functional/task_management/test_scheduler.py @@ -39,6 +39,28 @@ def test_single_jt_multi_job_launch_blocks_last(default_instance_group, job_temp TaskManager.start_task.assert_called_once_with(j2, default_instance_group) +@pytest.mark.django_db +def test_single_jt_multi_job_launch_allow_simul_allowed(default_instance_group, job_template_factory, mocker): + objects = job_template_factory('jt', organization='org1', project='proj', + inventory='inv', credential='cred', + jobs=["job_should_start", "job_should_not_start"]) + jt = objects.job_template + jt.save() + + j1 = objects.jobs["job_should_start"] + j1.allow_simultaneous = True + j1.status = 'pending' + j1.save() + j2 = objects.jobs["job_should_not_start"] + j2.allow_simultaneous = True + j2.status = 'pending' + j2.save() + with mock.patch("awx.main.scheduler.TaskManager.start_task"): + TaskManager().schedule() + TaskManager.start_task.assert_has_calls([mock.call(j1, default_instance_group), + mock.call(j2, default_instance_group)]) + + @pytest.mark.django_db def test_multi_jt_capacity_blocking(default_instance_group, job_template_factory, mocker): objects1 = job_template_factory('jt1', organization='org1', project='proj1', @@ -68,7 +90,7 @@ def test_multi_jt_capacity_blocking(default_instance_group, job_template_factory @pytest.mark.django_db -def test_single_job_dependencies_launch(default_instance_group, job_template_factory, mocker): +def test_single_job_dependencies_project_launch(default_instance_group, job_template_factory, mocker): objects = job_template_factory('jt', organization='org1', project='proj', inventory='inv', credential='cred', jobs=["job_should_start"]) @@ -94,10 +116,40 @@ def test_single_job_dependencies_launch(default_instance_group, job_template_fac with mock.patch("awx.main.scheduler.TaskManager.start_task"): TaskManager().schedule() TaskManager.start_task.assert_called_once_with(j, default_instance_group) + + +@pytest.mark.django_db +def test_single_job_dependencies_inventory_update_launch(default_instance_group, job_template_factory, mocker, inventory_source_factory): + objects = job_template_factory('jt', organization='org1', project='proj', + inventory='inv', credential='cred', + jobs=["job_should_start"]) + j = objects.jobs["job_should_start"] + j.status = 'pending' + j.save() + i = objects.inventory + ii = inventory_source_factory("ec2") + ii.source = "ec2" + ii.update_on_launch = True + ii.update_cache_timeout = 0 + ii.save() + i.inventory_sources.add(ii) + with mock.patch("awx.main.scheduler.TaskManager.start_task"): + tm = TaskManager() + with mock.patch.object(TaskManager, "create_inventory_update", wraps=tm.create_inventory_update) as mock_iu: + tm.schedule() + mock_iu.assert_called_once_with(j, ii) + iu = [x for x in ii.inventory_updates.all()] + assert len(iu) == 1 + TaskManager.start_task.assert_called_once_with(iu[0], default_instance_group, [iu[0]]) + iu[0].status = "successful" + iu[0].save() + with mock.patch("awx.main.scheduler.TaskManager.start_task"): + TaskManager().schedule() + TaskManager.start_task.assert_called_once_with(j, default_instance_group) @pytest.mark.django_db -def test_shared_dependencies_launch(default_instance_group, job_template_factory, mocker): +def test_shared_dependencies_launch(default_instance_group, job_template_factory, mocker, inventory_source_factory): objects = job_template_factory('jt', organization='org1', project='proj', inventory='inv', credential='cred', jobs=["first_job", "second_job"]) @@ -109,17 +161,31 @@ def test_shared_dependencies_launch(default_instance_group, job_template_factory j2.save() p = objects.project p.scm_update_on_launch = True - p.scm_update_cache_timeout = 10 + p.scm_update_cache_timeout = 300 p.scm_type = "git" p.scm_url = "http://github.com/ansible/ansible.git" p.save() + + i = objects.inventory + ii = inventory_source_factory("ec2") + ii.source = "ec2" + ii.update_on_launch = True + ii.update_cache_timeout = 300 + ii.save() + i.inventory_sources.add(ii) + with mock.patch("awx.main.scheduler.TaskManager.start_task"): TaskManager().schedule() pu = p.project_updates.first() - TaskManager.start_task.assert_called_once_with(pu, default_instance_group, [pu]) + iu = ii.inventory_updates.first() + TaskManager.start_task.assert_has_calls([mock.call(pu, default_instance_group, [pu, iu]), + mock.call(iu, default_instance_group, [pu, iu])]) pu.status = "successful" pu.finished = pu.created + timedelta(seconds=1) pu.save() + iu.status = "successful" + iu.finished = iu.created + timedelta(seconds=1) + iu.save() with mock.patch("awx.main.scheduler.TaskManager.start_task"): TaskManager().schedule() TaskManager.start_task.assert_called_once_with(j1, default_instance_group) @@ -129,4 +195,6 @@ def test_shared_dependencies_launch(default_instance_group, job_template_factory TaskManager().schedule() TaskManager.start_task.assert_called_once_with(j2, default_instance_group) pu = [x for x in p.project_updates.all()] + iu = [x for x in ii.inventory_updates.all()] assert len(pu) == 1 + assert len(iu) == 1