External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-03-06 03:01:06 -03:30
Code Issues Packages Projects Releases Wiki Activity

AC-982 Work on eliminating unnecessary queries from inventory import and activity stream logging.

Browse Source
This commit is contained in:
Chris Church
2014-01-27 15:27:27 -05:00
parent ff40234a9b
commit d17f58743e
4 changed files with 70 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
awx/main/management/commands/inventory_import.py
View File

@@ -17,8 +17,9 @@ import traceback
import yaml import yaml
# Django # Django
from django.conf import settings
from django.core.management.base import NoArgsCommand, CommandError from django.core.management.base import NoArgsCommand, CommandError
from django.db import transaction from django.db import connection, transaction
from django.contrib.auth.models import User from django.contrib.auth.models import User
# AWX # AWX
@@ -445,7 +446,7 @@ class Command(NoArgsCommand):
else: else:
q = dict(name=self.inventory_name) q = dict(name=self.inventory_name)
try: try:
self.inventory = Inventory.objects.get(**q) self.inventory = Inventory.objects.filter(active=True).get(**q)
except Inventory.DoesNotExist: except Inventory.DoesNotExist:
raise CommandError('Inventory with %s = %s cannot be found' % q.items()[0]) raise CommandError('Inventory with %s = %s cannot be found' % q.items()[0])
except Inventory.MultipleObjectsReturned: except Inventory.MultipleObjectsReturned:
@@ -459,7 +460,8 @@ class Command(NoArgsCommand):
if inventory_source_id: if inventory_source_id:
try: try:
self.inventory_source = InventorySource.objects.get(pk=inventory_source_id, self.inventory_source = InventorySource.objects.get(pk=inventory_source_id,
inventory=self.inventory) inventory=self.inventory,
active=True)
except InventorySource.DoesNotExist: except InventorySource.DoesNotExist:
raise CommandError('Inventory source with id=%s not found' % \ raise CommandError('Inventory source with id=%s not found' % \
inventory_source_id) inventory_source_id)
@@ -467,19 +469,21 @@ class Command(NoArgsCommand):
# Otherwise, create a new inventory source to capture this invocation # Otherwise, create a new inventory source to capture this invocation
# via command line. # via command line.
else: else:
self.inventory_source, created = InventorySource.objects.get_or_create( with ignore_inventory_computed_fields():
inventory=self.inventory, self.inventory_source, created = InventorySource.objects.get_or_create(
group=None, inventory=self.inventory,
source='file', group=None,
source_path=os.path.abspath(self.source), source='file',
overwrite=self.overwrite, source_path=os.path.abspath(self.source),
overwrite_vars=self.overwrite_vars, overwrite=self.overwrite,
) overwrite_vars=self.overwrite_vars,
self.inventory_update = self.inventory_source.inventory_updates.create( active=True,
job_args=json.dumps(sys.argv), )
job_env=dict(os.environ.items()), self.inventory_update = self.inventory_source.inventory_updates.create(
job_cwd=os.getcwd(), job_args=json.dumps(sys.argv),
) job_env=dict(os.environ.items()),
job_cwd=os.getcwd(),
)
# FIXME: Wait or raise error if inventory is being updated by another # FIXME: Wait or raise error if inventory is being updated by another
# source. # source.
@@ -499,11 +503,11 @@ class Command(NoArgsCommand):
del_hosts = self.inventory_source.group.all_hosts del_hosts = self.inventory_source.group.all_hosts
# FIXME: Also include hosts from inventory_source.managed_hosts? # FIXME: Also include hosts from inventory_source.managed_hosts?
else: else:
del_hosts = self.inventory.hosts.all() del_hosts = self.inventory.hosts.filter(active=True)
del_hosts = del_hosts.exclude(name__in=self.all_group.all_hosts.keys()) del_hosts = del_hosts.exclude(name__in=self.all_group.all_hosts.keys())
for host in del_hosts: for host in del_hosts:
host_name = host.name host_name = host.name
host.delete() host.mark_inactive()
self.logger.info('Deleted host "%s"', host_name) self.logger.info('Deleted host "%s"', host_name)
# If overwrite is set, for each group in the database that is NOT in # If overwrite is set, for each group in the database that is NOT in
@@ -515,11 +519,11 @@ class Command(NoArgsCommand):
del_groups = self.inventory_source.group.all_children del_groups = self.inventory_source.group.all_children
# FIXME: Also include groups from inventory_source.managed_groups? # FIXME: Also include groups from inventory_source.managed_groups?
else: else:
del_groups = self.inventory.groups.all() del_groups = self.inventory.groups.filter(active=True)
del_groups = del_groups.exclude(name__in=self.all_group.all_groups.keys()) del_groups = del_groups.exclude(name__in=self.all_group.all_groups.keys())
for group in del_groups: for group in del_groups:
group_name = group.name group_name = group.name
group.delete() group.mark_inactive()
self.logger.info('Group "%s" deleted', group_name) self.logger.info('Group "%s" deleted', group_name)
# If overwrite is set, clear all invalid child relationships for groups # If overwrite is set, clear all invalid child relationships for groups
@@ -531,22 +535,22 @@ class Command(NoArgsCommand):
if self.inventory_source.group: if self.inventory_source.group:
db_groups = self.inventory_source.group.all_children db_groups = self.inventory_source.group.all_children
else: else:
db_groups = self.inventory.groups.all() db_groups = self.inventory.groups.filter(active=True)
for db_group in db_groups: for db_group in db_groups:
db_children = db_group.children.all() db_children = db_group.children.filter(active=True)
mem_children = self.all_group.all_groups[db_group.name].children mem_children = self.all_group.all_groups[db_group.name].children
mem_children_names = [g.name for g in mem_children] mem_children_names = [g.name for g in mem_children]
for db_child in db_children.exclude(name__in=mem_children_names): for db_child in db_children.exclude(name__in=mem_children_names):
if db_child not in db_group.children.all(): if db_child not in db_group.children.filter(active=True):
continue continue
db_group.children.remove(db_child) db_group.children.remove(db_child)
self.logger.info('Group "%s" removed from group "%s"', self.logger.info('Group "%s" removed from group "%s"',
db_child.name, db_group.name) db_child.name, db_group.name)
db_hosts = db_group.hosts.all() db_hosts = db_group.hosts.filter(active=True)
mem_hosts = self.all_group.all_groups[db_group.name].hosts mem_hosts = self.all_group.all_groups[db_group.name].hosts
mem_host_names = [h.name for h in mem_hosts] mem_host_names = [h.name for h in mem_hosts]
for db_host in db_hosts.exclude(name__in=mem_host_names): for db_host in db_hosts.exclude(name__in=mem_host_names):
if db_host not in db_group.hosts.all(): if db_host not in db_group.hosts.filter(active=True):
continue continue
db_group.hosts.remove(db_host) db_group.hosts.remove(db_host)
self.logger.info('Host "%s" removed from group "%s"', self.logger.info('Host "%s" removed from group "%s"',
@@ -739,10 +743,11 @@ class Command(NoArgsCommand):
status, tb, exc = 'error', '', None status, tb, exc = 'error', '', None
try: try:
# Update inventory update for this command line invocation. # Update inventory update for this command line invocation.
if self.inventory_update: with ignore_inventory_computed_fields():
self.inventory_update.status = 'running' if self.inventory_update:
self.inventory_update.save() self.inventory_update.status = 'running'
transaction.commit() self.inventory_update.save()
transaction.commit()
# Load inventory from source. # Load inventory from source.
self.all_group = load_inventory_source(self.source) self.all_group = load_inventory_source(self.source)
@@ -762,6 +767,11 @@ class Command(NoArgsCommand):
self.logger.info('Inventory import completed for %s in %0.1fs', self.logger.info('Inventory import completed for %s in %0.1fs',
inv_name, time.time() - begin) inv_name, time.time() - begin)
status = 'successful' status = 'successful'
if settings.DEBUG:
sqltime = sum(float(x['time']) for x in connection.queries)
self.logger.info('Inventory import required %d queries '
'taking %0.3fs', len(connection.queries),
sqltime)
except Exception, e: except Exception, e:
if isinstance(e, KeyboardInterrupt): if isinstance(e, KeyboardInterrupt):
status = 'canceled' status = 'canceled'
@@ -775,11 +785,12 @@ class Command(NoArgsCommand):
transaction.rollback() transaction.rollback()
if self.inventory_update: if self.inventory_update:
self.inventory_update = InventoryUpdate.objects.get(pk=self.inventory_update.pk) with ignore_inventory_computed_fields():
self.inventory_update.result_traceback = tb self.inventory_update = InventoryUpdate.objects.get(pk=self.inventory_update.pk)
self.inventory_update.status = status self.inventory_update.result_traceback = tb
self.inventory_update.save(update_fields=['status', 'result_traceback']) self.inventory_update.status = status
transaction.commit() self.inventory_update.save(update_fields=['status', 'result_traceback'])
transaction.commit()
if exc and isinstance(exc, CommandError): if exc and isinstance(exc, CommandError):
sys.exit(1) sys.exit(1)

24
awx/main/signals.py
View File

@@ -32,10 +32,11 @@ def ignore_inventory_computed_fields():
Context manager to ignore updating inventory computed fields. Context manager to ignore updating inventory computed fields.
''' '''
try: try:
previous_value = getattr(_inventory_updating, 'is_updating', False)
_inventory_updating.is_updating = True _inventory_updating.is_updating = True
yield yield
finally: finally:
_inventory_updating.is_updating = False _inventory_updating.is_updating = previous_value
def update_inventory_computed_fields(sender, **kwargs): def update_inventory_computed_fields(sender, **kwargs):
''' '''
@@ -67,9 +68,13 @@ def update_inventory_computed_fields(sender, **kwargs):
logger.debug('%s %s, updating inventory computed fields: %r %r', logger.debug('%s %s, updating inventory computed fields: %r %r',
sender_name, sender_action, sender, kwargs) sender_name, sender_action, sender, kwargs)
with ignore_inventory_computed_fields(): with ignore_inventory_computed_fields():
inventory = instance.inventory try:
update_hosts = issubclass(sender, Job) inventory = instance.inventory
inventory.update_computed_fields(update_hosts=update_hosts) except Inventory.DoesNotExist:
pass
else:
update_hosts = issubclass(sender, Job)
inventory.update_computed_fields(update_hosts=update_hosts)
post_save.connect(update_inventory_computed_fields, sender=Host) post_save.connect(update_inventory_computed_fields, sender=Host)
post_delete.connect(update_inventory_computed_fields, sender=Host) post_delete.connect(update_inventory_computed_fields, sender=Host)
@@ -201,6 +206,9 @@ model_serializer_mapping = {Organization: OrganizationSerializer,
def activity_stream_create(sender, instance, created, **kwargs): def activity_stream_create(sender, instance, created, **kwargs):
if created: if created:
# Skip recording any inventory source directly associated with a group.
if isinstance(instance, InventorySource) and instance.group:
return
# TODO: Rethink details of the new instance # TODO: Rethink details of the new instance
object1 = camelcase_to_underscore(instance.__class__.__name__) object1 = camelcase_to_underscore(instance.__class__.__name__)
activity_entry = ActivityStream( activity_entry = ActivityStream(
@@ -211,6 +219,8 @@ def activity_stream_create(sender, instance, created, **kwargs):
getattr(activity_entry, object1).add(instance) getattr(activity_entry, object1).add(instance)
def activity_stream_update(sender, instance, **kwargs): def activity_stream_update(sender, instance, **kwargs):
if instance.id is None:
return
try: try:
old = sender.objects.get(id=instance.id) old = sender.objects.get(id=instance.id)
except sender.DoesNotExist: except sender.DoesNotExist:
@@ -238,6 +248,9 @@ def activity_stream_delete(sender, instance, **kwargs):
old = sender.objects.get(id=instance.id) old = sender.objects.get(id=instance.id)
except sender.DoesNotExist: except sender.DoesNotExist:
return return
# Skip recording any inventory source directly associated with a group.
if isinstance(instance, InventorySource) and instance.group:
return
changes = model_instance_diff(old, instance) changes = model_instance_diff(old, instance)
object1 = camelcase_to_underscore(instance.__class__.__name__) object1 = camelcase_to_underscore(instance.__class__.__name__)
activity_entry = ActivityStream( activity_entry = ActivityStream(
@@ -262,6 +275,9 @@ def activity_stream_associate(sender, instance, **kwargs):
obj2_id = entity_acted obj2_id = entity_acted
obj2_actual = obj2.objects.get(id=obj2_id) obj2_actual = obj2.objects.get(id=obj2_id)
object2 = camelcase_to_underscore(obj2.__name__) object2 = camelcase_to_underscore(obj2.__name__)
# Skip recording any inventory source changes here.
if isinstance(obj1, InventorySource) or isinstance(obj2_actual, InventorySource):
continue
activity_entry = ActivityStream( activity_entry = ActivityStream(
operation=action, operation=action,
object1=object1, object1=object1,

2
awx/main/tests/commands.py
View File

@@ -753,5 +753,5 @@ class InventoryImportTest(BaseCommandMixin, BaseLiveServerTest):
self.assertNotEqual(new_inv.groups.count(), 0) self.assertNotEqual(new_inv.groups.count(), 0)
self.assertNotEqual(new_inv.total_hosts, 0) self.assertNotEqual(new_inv.total_hosts, 0)
self.assertNotEqual(new_inv.total_groups, 0) self.assertNotEqual(new_inv.total_groups, 0)
self.assertElapsedLessThan(120) self.assertElapsedLessThan(60)

5
awx/main/tests/data/large_ec2_inventory.py
View File

@@ -866,7 +866,10 @@ inv_list = {
], ],
"www-test.axialmarket.com": [ "www-test.axialmarket.com": [
"ec2-54-234-233-19.compute-1.amazonaws.com" "ec2-54-234-233-19.compute-1.amazonaws.com"
] ],
#'_meta': {
# 'hostvars': {}
#}
} }
host_vars = { host_vars = {