Vastly improve overall group delete performance

Conflicts:
	awx/main/models/inventory.py
	awx/main/tasks.py

awx/main/models/base.py

@@ -271,10 +271,10 @@ class PrimordialModel(CreatedModifiedModel):
 
     tags = TaggableManager(blank=True)
 
-    def mark_inactive(self, save=True, update_fields=None):
+    def mark_inactive(self, save=True, update_fields=None, skip_active_check=False):
         '''Use instead of delete to rename and mark inactive.'''
         update_fields = update_fields or []
-        if self.active:
+        if skip_active_check or self.active:
             dtnow = now()
             if 'name' in self._meta.get_all_field_names():
                 self.name   = "_deleted_%s_%s" % (dtnow.isoformat(), self.name)

awx/main/models/inventory.py

@@ -20,7 +20,7 @@ import zmq
 
 # Django
 from django.conf import settings
-from django.db import models
+from django.db import models, connection
 from django.db.models import Q
 from django.utils.translation import ugettext_lazy as _
 from django.core.exceptions import ValidationError, NON_FIELD_ERRORS
@@ -389,12 +389,12 @@ class Host(CommonModelNameNotUnique):
     def get_absolute_url(self):
         return reverse('api:host_detail', args=(self.pk,))
 
-    def mark_inactive(self, save=True, from_inventory_import=False):
+    def mark_inactive(self, save=True, from_inventory_import=False, skip_active_check=False):
         '''
         When marking hosts inactive, remove all associations to related
         inventory sources.
         '''
-        super(Host, self).mark_inactive(save=save)
+        super(Host, self).mark_inactive(save=save, skip_active_check=skip_active_check)
         if not from_inventory_import:
             self.inventory_sources.clear()
 
@@ -532,19 +532,58 @@ class Group(CommonModelNameNotUnique):
 
     def mark_inactive_recursive(self):
         from awx.main.tasks import update_inventory_computed_fields, bulk_inventory_element_delete
-        group_data = {'parent': self.id, 'inventory': self.inventory.id,
+        from awx.main.utils import ignore_inventory_computed_fields
-                                             'children': [{'id': c.id} for c in self.children.all()],
+        from awx.main.signals import disable_activity_stream
-                                             'hosts': [{'id': h.id} for h in self.hosts.all()]}
+        # group_data = {'parent': self.id, 'inventory': self.inventory.id,
-        self.mark_inactive()
+        #                                      'children': [{'id': c.id} for c in self.children.all()],
-        bulk_inventory_element_delete.delay(group_data)
+        #                                      'hosts': [{'id': h.id} for h in self.hosts.all()]}
+        #self.mark_inactive(clear_children=False)
+        def remove_host_from_group(host, group):
+            #host.groups.remove(group)
+            host.inventory_sources.through.objects.filter(inventorysource__group=group).delete()
+            return host.groups.count() < 2
+        def mark_actual():
+            initial_hosts = self.hosts.all().prefetch_related('groups', 'inventory_sources')
+            linked_children = [(self, c) for c in self.children.all().prefetch_related('parents', 'hosts', 'inventory_sources', 'children')]
+            marked_hosts = []
+            marked_groups = [self]
+            for host in initial_hosts:
+                is_last_group = remove_host_from_group(host, self)
+                if is_last_group:
+                    marked_hosts.append(host)
+            self.hosts.through.objects.filter(group=self).delete()
+            self.children.through.objects.filter(to_group=self).delete()
+            for subgroup in linked_children:
+                parent, group = subgroup
+                #group.parents.remove(parent)
+                if group.parents.count() > 1:
+                    continue
+                all_group_hosts = group.hosts.all()
+                for host in group.hosts.all():
+                    is_last_group = remove_host_from_group(host, group)
+                    if is_last_group:
+                        marked_hosts.append(host)
+                group.hosts.through.objects.filter(group=group).delete()
+                for childgroup in group.children.all().prefetch_related('parents', 'hosts', 'inventory_sources', 'children'):
+                    linked_children.append((group, childgroup))
+                marked_groups.append(group)
+                group.children.through.objects.filter(to_group=group).delete()
+            all_groups = [g.id for g in marked_groups]
+            all_hosts = [h.id for h in marked_hosts]
+            Group.objects.filter(id__in=all_groups).update(active=False)
+            Host.objects.filter(id__in=all_hosts).update(active=False)
+            bulk_inventory_element_delete.delay(self.inventory.id, groups=all_groups, hosts=all_hosts)
+        with ignore_inventory_computed_fields():
+            with disable_activity_stream():
+                mark_actual()
 
-    def mark_inactive(self, save=True, recompute=True, from_inventory_import=False):
+    def mark_inactive(self, save=True, recompute=True, from_inventory_import=False, skip_active_check=False):
         '''
         When marking groups inactive, remove all associations to related
         groups/hosts/inventory_sources.
         '''
         def mark_actual():
-            super(Group, self).mark_inactive(save=save)
+            super(Group, self).mark_inactive(save=save, skip_active_check=skip_active_check)
             self.inventory_source.mark_inactive(save=save)
             self.inventory_sources.clear()
             self.parents.clear()
@@ -553,7 +592,7 @@ class Group(CommonModelNameNotUnique):
         i = self.inventory
 
         if from_inventory_import:
-            super(Group, self).mark_inactive(save=save)
+            super(Group, self).mark_inactive(save=save, skip_active_check=skip_active_check)
         elif recompute:
             with ignore_inventory_computed_fields():
                 mark_actual()

awx/main/tasks.py

@@ -48,44 +48,15 @@ logger = logging.getLogger('awx.main.tasks')
 # FIXME: Cleanly cancel task when celery worker is stopped.
 
 @task()
-def bulk_inventory_element_delete(group_details):
+def bulk_inventory_element_delete(inventory, hosts=[], groups=[]):
-    def remove_host_from_group(host, group):
+    from awx.main.signals import disable_activity_stream
-        host.groups.remove(group)
-        host_inv_sources = host.inventory_sources.all()
-        for inv_source in group.inventory_sources.all():
-            if inv_source in host_inv_sources:
-                host.inventory_sources.remove(inv_source)
-        return host.groups.count() < 1
-    def mark_actual(group_details):
-        overall_parent = Group.objects.get(id=group_details['parent'])
-        linked_children = [(overall_parent , Group.objects.get(id=g['id'])) for g in group_details['children']]
-        initial_hosts = [Host.objects.get(id=h['id']) for h in group_details['hosts']]
-        marked_hosts = []
-        marked_groups = []
-        for host in initial_hosts:
-            last_group = remove_host_from_group(host, overall_parent)
-            if last_group:
-                marked_hosts.append(host)
-        for subgroup in linked_children:
-            parent, group = subgroup
-            if parent is not None:
-                group.parents.remove(parent)
-            if group.parents.count() > 0:
-                continue
-            for host in group.hosts.all():
-                last_group = remove_host_from_group(host, group)
-                if last_group:
-                    marked_hosts.append(host)
-            for childgroup in group.children.all():
-                linked_children.append((group, childgroup))
-            marked_groups.append(group)
-        for group in marked_groups:
-            group.mark_inactive()
-        for host in marked_hosts:
-            host.mark_inactive()
     with ignore_inventory_computed_fields():
-        mark_actual(group_details)
+        with disable_activity_stream():
-    update_inventory_computed_fields.delay(group_details['inventory'], True)
+            for group in groups:
+                Group.objects.get(id=group).mark_inactive(skip_active_check=True)
+            for host in hosts:
+                Host.objects.get(id=host).mark_inactive(skip_active_check=True)
+    update_inventory_computed_fields(inventory)
 
 @task(bind=True)
 def tower_periodic_scheduler(self):