AC-1302 Update EC2 inventory script to create hierarchical groups.

Conflicts:
	awx/main/tests/inventory.py

awx/main/management/commands/inventory_import.py

@@ -733,6 +733,7 @@ class Command(NoArgsCommand):
                     db_group.children.remove(db_child)
                     self.logger.info('Group "%s" removed from group "%s"',
                                      db_child.name, db_group.name)
+            # FIXME: Inventory source group relationships
             # Delete group/host relationships not present in imported data.
             db_hosts = db_group.hosts.filter(active=True)
             del_host_pks = set(db_hosts.values_list('pk', flat=True))
@@ -804,6 +805,12 @@ class Command(NoArgsCommand):
             queries_before = len(connection.queries)
         inv_src_group = self.inventory_source.group
         all_group_names = sorted(self.all_group.all_groups.keys())
+        root_group_names = set()
+        for k,v in self.all_group.all_groups.items():
+            if not v.parents:
+                root_group_names.add(k)
+            if len(v.parents) == 1 and v.parents[0].name == 'all':
+                root_group_names.add(k)
         existing_group_names = set()
         for offset in xrange(0, len(all_group_names), self._batch_size):
             group_names = all_group_names[offset:(offset + self._batch_size)]
@@ -824,7 +831,7 @@ class Command(NoArgsCommand):
                 else:
                     self.logger.info('Group "%s" variables unmodified', group.name)
                 existing_group_names.add(group.name)
-                if inv_src_group and inv_src_group != group:
+                if inv_src_group and inv_src_group != group and group.name in root_group_names:
                     self._batch_add_m2m(inv_src_group.children, group)
                 self._batch_add_m2m(self.inventory_source.groups, group)
         for group_name in all_group_names:
@@ -836,7 +843,7 @@ class Command(NoArgsCommand):
             #group.inventory_source
             InventorySource.objects.create(group=group, inventory=self.inventory, name=('%s (%s)' % (group_name, self.inventory.name)))
             self.logger.info('Group "%s" added', group.name)
-            if inv_src_group:
+            if inv_src_group and group_name in root_group_names:
                 self._batch_add_m2m(inv_src_group.children, group)
             self._batch_add_m2m(self.inventory_source.groups, group)
         if inv_src_group:
@@ -908,7 +915,6 @@ class Command(NoArgsCommand):
         if self.inventory_source.group:
             self._batch_add_m2m(self.inventory_source.group.hosts, db_host)
         self._batch_add_m2m(self.inventory_source.hosts, db_host)
-        #host.update_computed_fields(False, False)
 
     def _create_update_hosts(self):
         '''
@@ -995,7 +1001,6 @@ class Command(NoArgsCommand):
             if self.inventory_source.group:
                 self._batch_add_m2m(self.inventory_source.group.hosts, db_host)
             self._batch_add_m2m(self.inventory_source.hosts, db_host)
-            #host.update_computed_fields(False, False)
 
         if self.inventory_source.group:
             self._batch_add_m2m(self.inventory_source.group.hosts, flush=True)

awx/main/tasks.py

@@ -806,6 +806,7 @@ class RunInventoryUpdate(BaseTask):
             ec2_opts.setdefault('destination_variable', 'public_dns_name')
             ec2_opts.setdefault('vpc_destination_variable', 'ip_address')
             ec2_opts.setdefault('route53', 'False')
+            ec2_opts.setdefault('nested_groups', 'True')
             ec2_opts['cache_path'] = tempfile.mkdtemp(prefix='awx_ec2_')
             ec2_opts['cache_max_age'] = '300'
             for k,v in ec2_opts.items():

awx/main/tests/inventory.py

@@ -1398,7 +1398,7 @@ class InventoryUpdatesTest(BaseTransactionTest):
         self.group = group
         inventory_source = self.update_inventory_source(self.group,
             source='ec2', credential=credential, source_regions=source_regions,
-            source_vars='---')
+            source_vars='---\n\nnested_groups: false\n')
         # Check first without instance_id set (to import by name only).
         with self.settings(EC2_INSTANCE_ID_VAR=''):
             self.check_inventory_source(inventory_source)
@@ -1428,6 +1428,58 @@ class InventoryUpdatesTest(BaseTransactionTest):
         # its own child).
         self.assertTrue(self.group in self.inventory.root_groups)
 
+    def test_update_from_ec2_with_nested_groups(self):
+        source_username = getattr(settings, 'TEST_AWS_ACCESS_KEY_ID', '')
+        source_password = getattr(settings, 'TEST_AWS_SECRET_ACCESS_KEY', '')
+        source_regions = getattr(settings, 'TEST_AWS_REGIONS', 'all')
+        if not all([source_username, source_password]):
+            self.skipTest('no test ec2 credentials defined!')
+        credential = Credential.objects.create(kind='aws',
+                                               user=self.super_django_user,
+                                               username=source_username,
+                                               password=source_password)
+        group = self.group
+        group.name = 'AWS Inventory'
+        group.save()
+        self.group = group
+        inventory_source = self.update_inventory_source(self.group,
+            source='ec2', credential=credential, source_regions=source_regions,
+            source_vars='---') # nested_groups is true by default.
+        self.check_inventory_source(inventory_source)
+        # Manually disable all hosts, verify a new update re-enables them.
+        for host in self.inventory.hosts.all():
+            host.enabled = False
+            host.save()
+        self.check_inventory_source(inventory_source, initial=False)
+        # Verify that main group is in top level groups (hasn't been added as
+        # its own child).
+        self.assertTrue(self.group in self.inventory.root_groups)
+        # Verify that returned groups are nested:
+        child_names = self.group.children.values_list('name', flat=True)
+        for name in child_names:
+            self.assertFalse(name.startswith('us-'))
+            self.assertFalse(name.startswith('type_'))
+            self.assertFalse(name.startswith('key_'))
+            self.assertFalse(name.startswith('security_group_'))
+            self.assertFalse(name.startswith('tag_'))
+        self.assertTrue('ec2' in child_names)
+        self.assertTrue('regions' in child_names)
+        self.assertTrue('types' in child_names)
+        self.assertTrue('keys' in child_names)
+        self.assertTrue('security_groups' in child_names)
+        self.assertTrue('tags' in child_names)
+        # Print out group/host tree for debugging.
+        return
+        print
+        def draw_tree(g, d=0):
+            print ('  ' * d) + '+ ' + g.name
+            for h in g.hosts.order_by('name'):
+                print ('  ' * d) + '  - ' + h.name
+            for c in g.children.order_by('name'):
+                draw_tree(c, d+1)
+        for g in self.inventory.root_groups.order_by('name'):
+            draw_tree(g)
+        
     def test_update_from_rax(self):
         source_username = getattr(settings, 'TEST_RACKSPACE_USERNAME', '')
         source_password = getattr(settings, 'TEST_RACKSPACE_API_KEY', '')

awx/plugins/inventory/ec2.ini.example

@@ -52,3 +52,7 @@ cache_path = ~/.ansible/tmp
 # The number of seconds a cache file is considered valid. After this many
 # seconds, a new API call will be made, and the cache file will be updated.
 cache_max_age = 300
+
+# For Ansible Tower, organize groups into a nested/hierarchy instead of a flat
+# namespace.
+nested_groups = True

awx/plugins/inventory/ec2.py

@@ -231,7 +231,11 @@ class Ec2Inventory(object):
         self.cache_path_index = cache_dir + "/ansible-ec2.index"
         self.cache_max_age = config.getint('ec2', 'cache_max_age')
 
-
+        # Ansible Tower - configure nested groups instead of flat namespace.
+        if config.has_option('ec2', 'nested_groups'):
+            self.nested_groups = config.getboolean('ec2', 'nested_groups')
+        else:
+            self.nested_groups = False
 
     def parse_cli_args(self):
         ''' Command line argument processing '''
@@ -321,14 +325,14 @@ class Ec2Inventory(object):
             for instance in reservation.instances:
                 return instance
 
-
     def add_instance(self, instance, region):
         ''' Adds an instance to the inventory and index, as long as it is
         addressable '''
 
+        # For Ansible Tower, return all instances regardless of state.
         # Only want running instances
-        if instance.state != 'running':
+        #if instance.state != 'running':
-            return
+        #    return
 
         # Select the best destination address
         if instance.subnet_id:
@@ -347,23 +351,36 @@ class Ec2Inventory(object):
         self.inventory[instance.id] = [dest]
 
         # Inventory: Group by region
-        self.push(self.inventory, region, dest)
+        if self.nested_groups:
+            self.push_group(self.inventory, 'regions', region)
+        else:
+            self.push(self.inventory, region, dest)
 
         # Inventory: Group by availability zone
         self.push(self.inventory, instance.placement, dest)
+        if self.nested_groups:
+            self.push_group(self.inventory, region, instance.placement)
 
         # Inventory: Group by instance type
-        self.push(self.inventory, self.to_safe('type_' + instance.instance_type), dest)
+        type_name = self.to_safe('type_' + instance.instance_type)
+        self.push(self.inventory, type_name, dest)
+        if self.nested_groups:
+            self.push_group(self.inventory, 'types', type_name)
 
         # Inventory: Group by key pair
         if instance.key_name:
-            self.push(self.inventory, self.to_safe('key_' + instance.key_name), dest)
+            key_name = self.to_safe('key_' + instance.key_name)
+            self.push(self.inventory, key_name, dest)
+            if self.nested_groups:
+                self.push_group(self.inventory, 'keys', key_name)
         
         # Inventory: Group by security group
         try:
             for group in instance.groups:
                 key = self.to_safe("security_group_" + group.name)
                 self.push(self.inventory, key, dest)
+                if self.nested_groups:
+                    self.push_group(self.inventory, 'security_groups', key)
         except AttributeError:
             print 'Package boto seems a bit older.'
             print 'Please upgrade boto >= 2.3.0.'
@@ -373,12 +390,17 @@ class Ec2Inventory(object):
         for k, v in instance.tags.iteritems():
             key = self.to_safe("tag_" + k + "=" + v)
             self.push(self.inventory, key, dest)
+            if self.nested_groups:
+                self.push_group(self.inventory, 'tags', self.to_safe("tag_" + k))
+                self.push_group(self.inventory, self.to_safe("tag_" + k), key)
 
         # Inventory: Group by Route53 domain names if enabled
         if self.route53_enabled:
             route53_names = self.get_instance_route53_names(instance)
             for name in route53_names:
                 self.push(self.inventory, name, dest)
+                if self.nested_groups:
+                    self.push_group(self.inventory, 'route53', name)
 
         # Global Tag: tag all EC2 instances
         self.push(self.inventory, 'ec2', dest)
@@ -561,6 +583,12 @@ class Ec2Inventory(object):
         else:
             my_dict[key] = [element]
 
+    def push_group(self, my_dict, key, element):
+        '''Push a group as a child of another group.'''
+        parent_group = my_dict.setdefault(key, {})
+        child_groups = parent_group.setdefault('children', [])
+        if element not in child_groups:
+            child_groups.append(element)
 
     def get_inventory_from_cache(self):
         ''' Reads the inventory from the cache file and returns it as a JSON