External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-01-13 19:10:07 -03:30
Code Issues Packages Projects Releases Wiki Activity

Vastly improve overall group delete performance

Browse Source 
Conflicts:
	awx/main/models/inventory.py
	awx/main/tasks.py
This commit is contained in:
Matthew Jones 2014-05-23 13:59:42 -04:00
parent 12ecc46398
commit e2fb427789
3 changed files with 60 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
awx/main/models/base.py
View File

@ -271,10 +271,10 @@ class PrimordialModel(CreatedModifiedModel):
tags = TaggableManager(blank=True)
def mark_inactive(self, save=True, update_fields=None):
def mark_inactive(self, save=True, update_fields=None, skip_active_check=False):
'''Use instead of delete to rename and mark inactive.'''
update_fields = update_fields or []
if self.active:
if skip_active_check or self.active:
dtnow = now()
if 'name' in self._meta.get_all_field_names():
self.name = "_deleted_%s_%s" % (dtnow.isoformat(), self.name)

61
awx/main/models/inventory.py
View File

@ -20,7 +20,7 @@ import zmq
# Django
from django.conf import settings
from django.db import models
from django.db import models, connection
from django.db.models import Q
from django.utils.translation import ugettext_lazy as _
from django.core.exceptions import ValidationError, NON_FIELD_ERRORS
@ -389,12 +389,12 @@ class Host(CommonModelNameNotUnique):
def get_absolute_url(self):
return reverse('api:host_detail', args=(self.pk,))
def mark_inactive(self, save=True, from_inventory_import=False):
def mark_inactive(self, save=True, from_inventory_import=False, skip_active_check=False):
'''
When marking hosts inactive, remove all associations to related
inventory sources.
'''
super(Host, self).mark_inactive(save=save)
super(Host, self).mark_inactive(save=save, skip_active_check=skip_active_check)
if not from_inventory_import:
self.inventory_sources.clear()
@ -532,19 +532,58 @@ class Group(CommonModelNameNotUnique):
def mark_inactive_recursive(self):
from awx.main.tasks import update_inventory_computed_fields, bulk_inventory_element_delete
group_data = {'parent': self.id, 'inventory': self.inventory.id,
'children': [{'id': c.id} for c in self.children.all()],
'hosts': [{'id': h.id} for h in self.hosts.all()]}
self.mark_inactive()
bulk_inventory_element_delete.delay(group_data)
from awx.main.utils import ignore_inventory_computed_fields
from awx.main.signals import disable_activity_stream
# group_data = {'parent': self.id, 'inventory': self.inventory.id,
# 'children': [{'id': c.id} for c in self.children.all()],
# 'hosts': [{'id': h.id} for h in self.hosts.all()]}
#self.mark_inactive(clear_children=False)
def remove_host_from_group(host, group):
#host.groups.remove(group)
host.inventory_sources.through.objects.filter(inventorysource__group=group).delete()
return host.groups.count() < 2
def mark_actual():
initial_hosts = self.hosts.all().prefetch_related('groups', 'inventory_sources')
linked_children = [(self, c) for c in self.children.all().prefetch_related('parents', 'hosts', 'inventory_sources', 'children')]
marked_hosts = []
marked_groups = [self]
for host in initial_hosts:
is_last_group = remove_host_from_group(host, self)
if is_last_group:
marked_hosts.append(host)
self.hosts.through.objects.filter(group=self).delete()
self.children.through.objects.filter(to_group=self).delete()
for subgroup in linked_children:
parent, group = subgroup
#group.parents.remove(parent)
if group.parents.count() > 1:
continue
all_group_hosts = group.hosts.all()
for host in group.hosts.all():
is_last_group = remove_host_from_group(host, group)
if is_last_group:
marked_hosts.append(host)
group.hosts.through.objects.filter(group=group).delete()
for childgroup in group.children.all().prefetch_related('parents', 'hosts', 'inventory_sources', 'children'):
linked_children.append((group, childgroup))
marked_groups.append(group)
group.children.through.objects.filter(to_group=group).delete()
all_groups = [g.id for g in marked_groups]
all_hosts = [h.id for h in marked_hosts]
Group.objects.filter(id__in=all_groups).update(active=False)
Host.objects.filter(id__in=all_hosts).update(active=False)
bulk_inventory_element_delete.delay(self.inventory.id, groups=all_groups, hosts=all_hosts)
with ignore_inventory_computed_fields():
with disable_activity_stream():
mark_actual()
def mark_inactive(self, save=True, recompute=True, from_inventory_import=False):
def mark_inactive(self, save=True, recompute=True, from_inventory_import=False, skip_active_check=False):
'''
When marking groups inactive, remove all associations to related
groups/hosts/inventory_sources.
'''
def mark_actual():
super(Group, self).mark_inactive(save=save)
super(Group, self).mark_inactive(save=save, skip_active_check=skip_active_check)
self.inventory_source.mark_inactive(save=save)
self.inventory_sources.clear()
self.parents.clear()
@ -553,7 +592,7 @@ class Group(CommonModelNameNotUnique):
i = self.inventory
if from_inventory_import:
super(Group, self).mark_inactive(save=save)
super(Group, self).mark_inactive(save=save, skip_active_check=skip_active_check)
elif recompute:
with ignore_inventory_computed_fields():
mark_actual()

45
awx/main/tasks.py
View File

@ -48,44 +48,15 @@ logger = logging.getLogger('awx.main.tasks')
# FIXME: Cleanly cancel task when celery worker is stopped.
@task()
def bulk_inventory_element_delete(group_details):
def remove_host_from_group(host, group):
host.groups.remove(group)
host_inv_sources = host.inventory_sources.all()
for inv_source in group.inventory_sources.all():
if inv_source in host_inv_sources:
host.inventory_sources.remove(inv_source)
return host.groups.count() < 1
def mark_actual(group_details):
overall_parent = Group.objects.get(id=group_details['parent'])
linked_children = [(overall_parent , Group.objects.get(id=g['id'])) for g in group_details['children']]
initial_hosts = [Host.objects.get(id=h['id']) for h in group_details['hosts']]
marked_hosts = []
marked_groups = []
for host in initial_hosts:
last_group = remove_host_from_group(host, overall_parent)
if last_group:
marked_hosts.append(host)
for subgroup in linked_children:
parent, group = subgroup
if parent is not None:
group.parents.remove(parent)
if group.parents.count() > 0:
continue
for host in group.hosts.all():
last_group = remove_host_from_group(host, group)
if last_group:
marked_hosts.append(host)
for childgroup in group.children.all():
linked_children.append((group, childgroup))
marked_groups.append(group)
for group in marked_groups:
group.mark_inactive()
for host in marked_hosts:
host.mark_inactive()
def bulk_inventory_element_delete(inventory, hosts=[], groups=[]):
from awx.main.signals import disable_activity_stream
with ignore_inventory_computed_fields():
mark_actual(group_details)
update_inventory_computed_fields.delay(group_details['inventory'], True)
with disable_activity_stream():
for group in groups:
Group.objects.get(id=group).mark_inactive(skip_active_check=True)
for host in hosts:
Host.objects.get(id=host).mark_inactive(skip_active_check=True)
update_inventory_computed_fields(inventory)
@task(bind=True)
def tower_periodic_scheduler(self):