diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index 032eaea84a..4f2b75b5fa 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -17,8 +17,9 @@ import traceback import yaml # Django +from django.conf import settings from django.core.management.base import NoArgsCommand, CommandError -from django.db import transaction +from django.db import connection, transaction from django.contrib.auth.models import User # AWX @@ -445,7 +446,7 @@ class Command(NoArgsCommand): else: q = dict(name=self.inventory_name) try: - self.inventory = Inventory.objects.get(**q) + self.inventory = Inventory.objects.filter(active=True).get(**q) except Inventory.DoesNotExist: raise CommandError('Inventory with %s = %s cannot be found' % q.items()[0]) except Inventory.MultipleObjectsReturned: @@ -459,7 +460,8 @@ class Command(NoArgsCommand): if inventory_source_id: try: self.inventory_source = InventorySource.objects.get(pk=inventory_source_id, - inventory=self.inventory) + inventory=self.inventory, + active=True) except InventorySource.DoesNotExist: raise CommandError('Inventory source with id=%s not found' % \ inventory_source_id) @@ -467,19 +469,21 @@ class Command(NoArgsCommand): # Otherwise, create a new inventory source to capture this invocation # via command line. else: - self.inventory_source, created = InventorySource.objects.get_or_create( - inventory=self.inventory, - group=None, - source='file', - source_path=os.path.abspath(self.source), - overwrite=self.overwrite, - overwrite_vars=self.overwrite_vars, - ) - self.inventory_update = self.inventory_source.inventory_updates.create( - job_args=json.dumps(sys.argv), - job_env=dict(os.environ.items()), - job_cwd=os.getcwd(), - ) + with ignore_inventory_computed_fields(): + self.inventory_source, created = InventorySource.objects.get_or_create( + inventory=self.inventory, + group=None, + source='file', + source_path=os.path.abspath(self.source), + overwrite=self.overwrite, + overwrite_vars=self.overwrite_vars, + active=True, + ) + self.inventory_update = self.inventory_source.inventory_updates.create( + job_args=json.dumps(sys.argv), + job_env=dict(os.environ.items()), + job_cwd=os.getcwd(), + ) # FIXME: Wait or raise error if inventory is being updated by another # source. @@ -499,11 +503,11 @@ class Command(NoArgsCommand): del_hosts = self.inventory_source.group.all_hosts # FIXME: Also include hosts from inventory_source.managed_hosts? else: - del_hosts = self.inventory.hosts.all() + del_hosts = self.inventory.hosts.filter(active=True) del_hosts = del_hosts.exclude(name__in=self.all_group.all_hosts.keys()) for host in del_hosts: host_name = host.name - host.delete() + host.mark_inactive() self.logger.info('Deleted host "%s"', host_name) # If overwrite is set, for each group in the database that is NOT in @@ -515,11 +519,11 @@ class Command(NoArgsCommand): del_groups = self.inventory_source.group.all_children # FIXME: Also include groups from inventory_source.managed_groups? else: - del_groups = self.inventory.groups.all() + del_groups = self.inventory.groups.filter(active=True) del_groups = del_groups.exclude(name__in=self.all_group.all_groups.keys()) for group in del_groups: group_name = group.name - group.delete() + group.mark_inactive() self.logger.info('Group "%s" deleted', group_name) # If overwrite is set, clear all invalid child relationships for groups @@ -531,22 +535,22 @@ class Command(NoArgsCommand): if self.inventory_source.group: db_groups = self.inventory_source.group.all_children else: - db_groups = self.inventory.groups.all() + db_groups = self.inventory.groups.filter(active=True) for db_group in db_groups: - db_children = db_group.children.all() + db_children = db_group.children.filter(active=True) mem_children = self.all_group.all_groups[db_group.name].children mem_children_names = [g.name for g in mem_children] for db_child in db_children.exclude(name__in=mem_children_names): - if db_child not in db_group.children.all(): + if db_child not in db_group.children.filter(active=True): continue db_group.children.remove(db_child) self.logger.info('Group "%s" removed from group "%s"', db_child.name, db_group.name) - db_hosts = db_group.hosts.all() + db_hosts = db_group.hosts.filter(active=True) mem_hosts = self.all_group.all_groups[db_group.name].hosts mem_host_names = [h.name for h in mem_hosts] for db_host in db_hosts.exclude(name__in=mem_host_names): - if db_host not in db_group.hosts.all(): + if db_host not in db_group.hosts.filter(active=True): continue db_group.hosts.remove(db_host) self.logger.info('Host "%s" removed from group "%s"', @@ -739,10 +743,11 @@ class Command(NoArgsCommand): status, tb, exc = 'error', '', None try: # Update inventory update for this command line invocation. - if self.inventory_update: - self.inventory_update.status = 'running' - self.inventory_update.save() - transaction.commit() + with ignore_inventory_computed_fields(): + if self.inventory_update: + self.inventory_update.status = 'running' + self.inventory_update.save() + transaction.commit() # Load inventory from source. self.all_group = load_inventory_source(self.source) @@ -762,6 +767,11 @@ class Command(NoArgsCommand): self.logger.info('Inventory import completed for %s in %0.1fs', inv_name, time.time() - begin) status = 'successful' + if settings.DEBUG: + sqltime = sum(float(x['time']) for x in connection.queries) + self.logger.info('Inventory import required %d queries ' + 'taking %0.3fs', len(connection.queries), + sqltime) except Exception, e: if isinstance(e, KeyboardInterrupt): status = 'canceled' @@ -775,11 +785,12 @@ class Command(NoArgsCommand): transaction.rollback() if self.inventory_update: - self.inventory_update = InventoryUpdate.objects.get(pk=self.inventory_update.pk) - self.inventory_update.result_traceback = tb - self.inventory_update.status = status - self.inventory_update.save(update_fields=['status', 'result_traceback']) - transaction.commit() + with ignore_inventory_computed_fields(): + self.inventory_update = InventoryUpdate.objects.get(pk=self.inventory_update.pk) + self.inventory_update.result_traceback = tb + self.inventory_update.status = status + self.inventory_update.save(update_fields=['status', 'result_traceback']) + transaction.commit() if exc and isinstance(exc, CommandError): sys.exit(1) diff --git a/awx/main/signals.py b/awx/main/signals.py index 91a0cb2da4..efb00f69a2 100644 --- a/awx/main/signals.py +++ b/awx/main/signals.py @@ -32,10 +32,11 @@ def ignore_inventory_computed_fields(): Context manager to ignore updating inventory computed fields. ''' try: + previous_value = getattr(_inventory_updating, 'is_updating', False) _inventory_updating.is_updating = True yield finally: - _inventory_updating.is_updating = False + _inventory_updating.is_updating = previous_value def update_inventory_computed_fields(sender, **kwargs): ''' @@ -67,9 +68,13 @@ def update_inventory_computed_fields(sender, **kwargs): logger.debug('%s %s, updating inventory computed fields: %r %r', sender_name, sender_action, sender, kwargs) with ignore_inventory_computed_fields(): - inventory = instance.inventory - update_hosts = issubclass(sender, Job) - inventory.update_computed_fields(update_hosts=update_hosts) + try: + inventory = instance.inventory + except Inventory.DoesNotExist: + pass + else: + update_hosts = issubclass(sender, Job) + inventory.update_computed_fields(update_hosts=update_hosts) post_save.connect(update_inventory_computed_fields, sender=Host) post_delete.connect(update_inventory_computed_fields, sender=Host) @@ -201,6 +206,9 @@ model_serializer_mapping = {Organization: OrganizationSerializer, def activity_stream_create(sender, instance, created, **kwargs): if created: + # Skip recording any inventory source directly associated with a group. + if isinstance(instance, InventorySource) and instance.group: + return # TODO: Rethink details of the new instance object1 = camelcase_to_underscore(instance.__class__.__name__) activity_entry = ActivityStream( @@ -211,6 +219,8 @@ def activity_stream_create(sender, instance, created, **kwargs): getattr(activity_entry, object1).add(instance) def activity_stream_update(sender, instance, **kwargs): + if instance.id is None: + return try: old = sender.objects.get(id=instance.id) except sender.DoesNotExist: @@ -238,6 +248,9 @@ def activity_stream_delete(sender, instance, **kwargs): old = sender.objects.get(id=instance.id) except sender.DoesNotExist: return + # Skip recording any inventory source directly associated with a group. + if isinstance(instance, InventorySource) and instance.group: + return changes = model_instance_diff(old, instance) object1 = camelcase_to_underscore(instance.__class__.__name__) activity_entry = ActivityStream( @@ -262,6 +275,9 @@ def activity_stream_associate(sender, instance, **kwargs): obj2_id = entity_acted obj2_actual = obj2.objects.get(id=obj2_id) object2 = camelcase_to_underscore(obj2.__name__) + # Skip recording any inventory source changes here. + if isinstance(obj1, InventorySource) or isinstance(obj2_actual, InventorySource): + continue activity_entry = ActivityStream( operation=action, object1=object1, diff --git a/awx/main/tests/commands.py b/awx/main/tests/commands.py index 81f203c700..e773ed3cbf 100644 --- a/awx/main/tests/commands.py +++ b/awx/main/tests/commands.py @@ -753,5 +753,5 @@ class InventoryImportTest(BaseCommandMixin, BaseLiveServerTest): self.assertNotEqual(new_inv.groups.count(), 0) self.assertNotEqual(new_inv.total_hosts, 0) self.assertNotEqual(new_inv.total_groups, 0) - self.assertElapsedLessThan(120) + self.assertElapsedLessThan(60) diff --git a/awx/main/tests/data/large_ec2_inventory.py b/awx/main/tests/data/large_ec2_inventory.py index dfab151fa1..55fc66cc02 100755 --- a/awx/main/tests/data/large_ec2_inventory.py +++ b/awx/main/tests/data/large_ec2_inventory.py @@ -866,7 +866,10 @@ inv_list = { ], "www-test.axialmarket.com": [ "ec2-54-234-233-19.compute-1.amazonaws.com" - ] + ], + #'_meta': { + # 'hostvars': {} + #} } host_vars = {