block deletion of resources w unprocessed events

awx/api/views.py

@@ -206,6 +206,13 @@ class RelatedJobsPreventDeleteMixin(object):
         active_jobs = obj.get_active_jobs()
         if len(active_jobs) > 0:
             raise ActiveJobConflict(active_jobs)
+        time_cutoff = now() - dateutil.relativedelta.relativedelta(minutes=1)
+        recent_jobs = obj._get_related_jobs().filter(finished__gte = time_cutoff)
+        for unified_job in recent_jobs.get_real_instances():
+            if not unified_job.event_processing_finished:
+                raise PermissionDenied(_(
+                    'Related job {} is still processing events.'
+                ).format(unified_job.log_format))
         return super(RelatedJobsPreventDeleteMixin, self).perform_destroy(obj)
 
 

awx/main/models/ha.py

@@ -19,10 +19,7 @@ from awx.main.fields import JSONField
 from awx.main.models.inventory import InventoryUpdate
 from awx.main.models.jobs import Job
 from awx.main.models.projects import ProjectUpdate
-from awx.main.models.unified_jobs import (
+from awx.main.models.unified_jobs import UnifiedJob
-    UnifiedJob,
-    ACTIVE_STATES
-)
 from awx.main.utils import get_cpu_capacity, get_mem_capacity, get_system_task_capacity
 from awx.main.models.mixins import RelatedJobsMixin
 
@@ -159,9 +156,8 @@ class InstanceGroup(models.Model, RelatedJobsMixin):
     '''
     RelatedJobsMixin
     '''
-    def _get_active_jobs(self):
+    def _get_related_jobs(self):
-        return UnifiedJob.objects.filter(instance_group=self,
+        return UnifiedJob.objects.filter(instance_group=self)
-                                         status__in=ACTIVE_STATES)
 
 
     class Meta:

awx/main/models/inventory.py

@@ -33,7 +33,6 @@ from awx.main.managers import HostManager
 from awx.main.models.base import * # noqa
 from awx.main.models.events import InventoryUpdateEvent
 from awx.main.models.unified_jobs import * # noqa
-from awx.main.models.unified_jobs import ACTIVE_STATES
 from awx.main.models.mixins import (
     ResourceMixin,
     TaskManagerInventoryUpdateMixin,
@@ -497,14 +496,11 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin):
     '''
     RelatedJobsMixin
     '''
-    def _get_active_jobs(self):
+    def _get_related_jobs(self):
         return UnifiedJob.objects.non_polymorphic().filter(
-            Q(status__in=ACTIVE_STATES) &
+            Q(Job___inventory=self) |
-            (
+            Q(InventoryUpdate___inventory_source__inventory=self) |
-                Q(Job___inventory=self) |
+            Q(AdHocCommand___inventory=self)
-                Q(InventoryUpdate___inventory_source__inventory=self) |
-                Q(AdHocCommand___inventory=self)
-            )
         )
 
 
@@ -963,9 +959,8 @@ class Group(CommonModelNameNotUnique, RelatedJobsMixin):
     '''
     RelatedJobsMixin
     '''
-    def _get_active_jobs(self):
+    def _get_related_jobs(self):
-        return InventoryUpdate.objects.filter(status__in=ACTIVE_STATES,
+        return InventoryUpdate.objects.filter(inventory_source__in=self.inventory_sources.all())
-                                              inventory_source__in=self.inventory_sources.all())
 
 
 class InventorySourceOptions(BaseModel):
@@ -1583,9 +1578,8 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions, RelatedJobsMix
     '''
     RelatedJobsMixin
     '''
-    def _get_active_jobs(self):
+    def _get_related_jobs(self):
-        return InventoryUpdate.objects.filter(status__in=ACTIVE_STATES,
+        return InventoryUpdate.objects.filter(inventory_source=self)
-                                              inventory_source=self)
 
 
 class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin, TaskManagerInventoryUpdateMixin):

awx/main/models/jobs.py

@@ -29,7 +29,6 @@ from awx.api.versioning import reverse
 from awx.main.models.base import * # noqa
 from awx.main.models.events import JobEvent, SystemJobEvent
 from awx.main.models.unified_jobs import * # noqa
-from awx.main.models.unified_jobs import ACTIVE_STATES
 from awx.main.models.notifications import (
     NotificationTemplate,
     JobNotificationMixin,
@@ -455,8 +454,8 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
     '''
     RelatedJobsMixin
     '''
-    def _get_active_jobs(self):
+    def _get_related_jobs(self):
-        return Job.objects.filter(status__in=ACTIVE_STATES, job_template=self)
+        return Job.objects.filter(job_template=self)
 
 
 class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskManagerJobMixin):

awx/main/models/mixins.py

@@ -457,8 +457,11 @@ class RelatedJobsMixin(object):
     Returns a list of active jobs (i.e. running) associated with the calling
     resource (self). Expected to return a QuerySet
     '''
+    def _get_related_jobs(self):
+        return self.objects.none()
+
     def _get_active_jobs(self):
-        return []
+        return self._get_related_jobs().filter(status__in=('new', 'pending', 'waiting', 'running'))
 
     '''
     Returns [{'id': '1', 'type': 'job'}, {'id': 2, 'type': 'project_update'}, ...]

awx/main/models/organization.py

@@ -20,10 +20,7 @@ from awx.main.models.rbac import (
     ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
     ROLE_SINGLETON_SYSTEM_AUDITOR,
 )
-from awx.main.models.unified_jobs import (
+from awx.main.models.unified_jobs import UnifiedJob
-    UnifiedJob,
-    ACTIVE_STATES,
-)
 from awx.main.models.mixins import ResourceMixin, CustomVirtualEnvMixin, RelatedJobsMixin
 
 __all__ = ['Organization', 'Team', 'Profile', 'UserSessionMembership']
@@ -82,15 +79,12 @@ class Organization(CommonModel, NotificationFieldsModel, ResourceMixin, CustomVi
     '''
     RelatedJobsMixin
     '''
-    def _get_active_jobs(self):
+    def _get_related_jobs(self):
         project_ids = self.projects.all().values_list('id')
         return UnifiedJob.objects.non_polymorphic().filter(
-            Q(status__in=ACTIVE_STATES) &
+            Q(Job___project__in=project_ids) |
-            (
+            Q(ProjectUpdate___project__in=project_ids) |
-                Q(Job___project__in=project_ids) |
+            Q(InventoryUpdate___inventory_source__inventory__organization=self)
-                Q(ProjectUpdate___project__in=project_ids) |
-                Q(InventoryUpdate___inventory_source__inventory__organization=self)
-            )
         )
 
 

awx/main/models/projects.py

@@ -454,17 +454,13 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
     '''
     RelatedJobsMixin
     '''
-    def _get_active_jobs(self):
+    def _get_related_jobs(self):
         return UnifiedJob.objects.non_polymorphic().filter(
-            models.Q(status__in=ACTIVE_STATES) &
+            models.Q(Job___project=self) |
-            (
+            models.Q(ProjectUpdate___project=self)
-                models.Q(Job___project=self) |
-                models.Q(ProjectUpdate___project=self)
-            )
         )
 
 
-
 class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin, TaskManagerProjectUpdateMixin):
     '''
     Internal job for tracking project updates from SCM.

awx/main/models/workflow.py

@@ -30,7 +30,6 @@ from awx.main.models.mixins import (
     SurveyJobMixin,
     RelatedJobsMixin,
 )
-from awx.main.models.unified_jobs import ACTIVE_STATES
 from awx.main.models.jobs import LaunchTimeConfig
 from awx.main.models.credential import Credential
 from awx.main.redact import REPLACE_STR
@@ -414,8 +413,8 @@ class WorkflowJobTemplate(UnifiedJobTemplate, WorkflowJobOptions, SurveyJobTempl
     '''
     RelatedJobsMixin
     '''
-    def _get_active_jobs(self):
+    def _get_related_jobs(self):
-        return WorkflowJob.objects.filter(status__in=ACTIVE_STATES, workflow_job_template=self)
+        return WorkflowJob.objects.filter(workflow_job_template=self)
 
 
 class WorkflowJob(UnifiedJob, WorkflowJobOptions, SurveyJobMixin, JobNotificationMixin):

awx/main/tests/functional/api/test_job.py

@@ -1,8 +1,15 @@
 import pytest
+import mock
+
+from dateutil.parser import parse
+from dateutil.relativedelta import relativedelta
+
+from rest_framework.exceptions import PermissionDenied
 
 from awx.api.versioning import reverse
+from awx.api.views import RelatedJobsPreventDeleteMixin, UnifiedJobDeletionMixin
 
-from awx.main.models import JobTemplate, User
+from awx.main.models import JobTemplate, User, Job
 
 
 @pytest.mark.django_db
@@ -81,3 +88,50 @@ def test_job_relaunch_on_failed_hosts(post, inventory, project, machine_credenti
         expect=201
     )
     assert r.data.get('limit') == hosts
+
+
+@pytest.mark.django_db
+def test_block_unprocessed_events(delete, admin_user, mocker):
+    time_of_finish = parse("Thu Feb 28 09:10:20 2013 -0500")
+    job = Job.objects.create(
+        emitted_events=1,
+        status='finished',
+        finished=time_of_finish
+    )
+    request = mock.MagicMock()
+
+    class MockView(UnifiedJobDeletionMixin):
+        model = Job
+
+        def get_object():
+            return job
+
+    view = MockView()
+
+    time_of_request = time_of_finish + relativedelta(seconds=2)
+    with mock.patch('awx.api.views.now', lambda: time_of_request):
+        with mock.patch.object(view, 'get_object') as get_obj:
+            get_obj.return_value = job
+            r = view.destroy(request)
+            assert r.status_code == 400
+
+
+@pytest.mark.django_db
+def test_block_related_unprocessed_events(mocker, organization, project, delete, admin_user):
+    job_template = JobTemplate.objects.create(
+        project=project,
+        playbook='helloworld.yml'
+    )
+    time_of_finish = parse("Thu Feb 23 14:17:24 2012 -0500")
+    Job.objects.create(
+        emitted_events=1,
+        status='finished',
+        finished=time_of_finish,
+        job_template=job_template,
+        project=project
+    )
+    view = RelatedJobsPreventDeleteMixin()
+    time_of_request = time_of_finish + relativedelta(seconds=2)
+    with mock.patch('awx.api.views.now', lambda: time_of_request):
+        with pytest.raises(PermissionDenied):
+            view.perform_destroy(organization)