From e2fb4277894c41b1a24fb574f53a1bf4ac4c9482 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Fri, 23 May 2014 13:59:42 -0400 Subject: [PATCH] Vastly improve overall group delete performance Conflicts: awx/main/models/inventory.py awx/main/tasks.py --- awx/main/models/base.py | 4 +-- awx/main/models/inventory.py | 61 +++++++++++++++++++++++++++++------- awx/main/tasks.py | 45 +++++--------------------- 3 files changed, 60 insertions(+), 50 deletions(-) diff --git a/awx/main/models/base.py b/awx/main/models/base.py index dda2574369..cfeb393fb9 100644 --- a/awx/main/models/base.py +++ b/awx/main/models/base.py @@ -271,10 +271,10 @@ class PrimordialModel(CreatedModifiedModel): tags = TaggableManager(blank=True) - def mark_inactive(self, save=True, update_fields=None): + def mark_inactive(self, save=True, update_fields=None, skip_active_check=False): '''Use instead of delete to rename and mark inactive.''' update_fields = update_fields or [] - if self.active: + if skip_active_check or self.active: dtnow = now() if 'name' in self._meta.get_all_field_names(): self.name = "_deleted_%s_%s" % (dtnow.isoformat(), self.name) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 9aff59895c..f37c946593 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -20,7 +20,7 @@ import zmq # Django from django.conf import settings -from django.db import models +from django.db import models, connection from django.db.models import Q from django.utils.translation import ugettext_lazy as _ from django.core.exceptions import ValidationError, NON_FIELD_ERRORS @@ -389,12 +389,12 @@ class Host(CommonModelNameNotUnique): def get_absolute_url(self): return reverse('api:host_detail', args=(self.pk,)) - def mark_inactive(self, save=True, from_inventory_import=False): + def mark_inactive(self, save=True, from_inventory_import=False, skip_active_check=False): ''' When marking hosts inactive, remove all associations to related inventory sources. ''' - super(Host, self).mark_inactive(save=save) + super(Host, self).mark_inactive(save=save, skip_active_check=skip_active_check) if not from_inventory_import: self.inventory_sources.clear() @@ -532,19 +532,58 @@ class Group(CommonModelNameNotUnique): def mark_inactive_recursive(self): from awx.main.tasks import update_inventory_computed_fields, bulk_inventory_element_delete - group_data = {'parent': self.id, 'inventory': self.inventory.id, - 'children': [{'id': c.id} for c in self.children.all()], - 'hosts': [{'id': h.id} for h in self.hosts.all()]} - self.mark_inactive() - bulk_inventory_element_delete.delay(group_data) + from awx.main.utils import ignore_inventory_computed_fields + from awx.main.signals import disable_activity_stream + # group_data = {'parent': self.id, 'inventory': self.inventory.id, + # 'children': [{'id': c.id} for c in self.children.all()], + # 'hosts': [{'id': h.id} for h in self.hosts.all()]} + #self.mark_inactive(clear_children=False) + def remove_host_from_group(host, group): + #host.groups.remove(group) + host.inventory_sources.through.objects.filter(inventorysource__group=group).delete() + return host.groups.count() < 2 + def mark_actual(): + initial_hosts = self.hosts.all().prefetch_related('groups', 'inventory_sources') + linked_children = [(self, c) for c in self.children.all().prefetch_related('parents', 'hosts', 'inventory_sources', 'children')] + marked_hosts = [] + marked_groups = [self] + for host in initial_hosts: + is_last_group = remove_host_from_group(host, self) + if is_last_group: + marked_hosts.append(host) + self.hosts.through.objects.filter(group=self).delete() + self.children.through.objects.filter(to_group=self).delete() + for subgroup in linked_children: + parent, group = subgroup + #group.parents.remove(parent) + if group.parents.count() > 1: + continue + all_group_hosts = group.hosts.all() + for host in group.hosts.all(): + is_last_group = remove_host_from_group(host, group) + if is_last_group: + marked_hosts.append(host) + group.hosts.through.objects.filter(group=group).delete() + for childgroup in group.children.all().prefetch_related('parents', 'hosts', 'inventory_sources', 'children'): + linked_children.append((group, childgroup)) + marked_groups.append(group) + group.children.through.objects.filter(to_group=group).delete() + all_groups = [g.id for g in marked_groups] + all_hosts = [h.id for h in marked_hosts] + Group.objects.filter(id__in=all_groups).update(active=False) + Host.objects.filter(id__in=all_hosts).update(active=False) + bulk_inventory_element_delete.delay(self.inventory.id, groups=all_groups, hosts=all_hosts) + with ignore_inventory_computed_fields(): + with disable_activity_stream(): + mark_actual() - def mark_inactive(self, save=True, recompute=True, from_inventory_import=False): + def mark_inactive(self, save=True, recompute=True, from_inventory_import=False, skip_active_check=False): ''' When marking groups inactive, remove all associations to related groups/hosts/inventory_sources. ''' def mark_actual(): - super(Group, self).mark_inactive(save=save) + super(Group, self).mark_inactive(save=save, skip_active_check=skip_active_check) self.inventory_source.mark_inactive(save=save) self.inventory_sources.clear() self.parents.clear() @@ -553,7 +592,7 @@ class Group(CommonModelNameNotUnique): i = self.inventory if from_inventory_import: - super(Group, self).mark_inactive(save=save) + super(Group, self).mark_inactive(save=save, skip_active_check=skip_active_check) elif recompute: with ignore_inventory_computed_fields(): mark_actual() diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 12f1fae5e1..8ef24acf7f 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -48,44 +48,15 @@ logger = logging.getLogger('awx.main.tasks') # FIXME: Cleanly cancel task when celery worker is stopped. @task() -def bulk_inventory_element_delete(group_details): - def remove_host_from_group(host, group): - host.groups.remove(group) - host_inv_sources = host.inventory_sources.all() - for inv_source in group.inventory_sources.all(): - if inv_source in host_inv_sources: - host.inventory_sources.remove(inv_source) - return host.groups.count() < 1 - def mark_actual(group_details): - overall_parent = Group.objects.get(id=group_details['parent']) - linked_children = [(overall_parent , Group.objects.get(id=g['id'])) for g in group_details['children']] - initial_hosts = [Host.objects.get(id=h['id']) for h in group_details['hosts']] - marked_hosts = [] - marked_groups = [] - for host in initial_hosts: - last_group = remove_host_from_group(host, overall_parent) - if last_group: - marked_hosts.append(host) - for subgroup in linked_children: - parent, group = subgroup - if parent is not None: - group.parents.remove(parent) - if group.parents.count() > 0: - continue - for host in group.hosts.all(): - last_group = remove_host_from_group(host, group) - if last_group: - marked_hosts.append(host) - for childgroup in group.children.all(): - linked_children.append((group, childgroup)) - marked_groups.append(group) - for group in marked_groups: - group.mark_inactive() - for host in marked_hosts: - host.mark_inactive() +def bulk_inventory_element_delete(inventory, hosts=[], groups=[]): + from awx.main.signals import disable_activity_stream with ignore_inventory_computed_fields(): - mark_actual(group_details) - update_inventory_computed_fields.delay(group_details['inventory'], True) + with disable_activity_stream(): + for group in groups: + Group.objects.get(id=group).mark_inactive(skip_active_check=True) + for host in hosts: + Host.objects.get(id=host).mark_inactive(skip_active_check=True) + update_inventory_computed_fields(inventory) @task(bind=True) def tower_periodic_scheduler(self):