Merge pull request #278 from cchurch/openstack_inventory_updates

OpenStack inventory updates

awx/main/management/commands/inventory_import.py

@@ -530,7 +530,8 @@ class Command(NoArgsCommand):
                     'to load'),
         make_option('--enabled-var', dest='enabled_var', type='str',
                     default=None, metavar='v', help='host variable used to '
-                    'set/clear enabled flag when host is online/offline'),
+                    'set/clear enabled flag when host is online/offline, may '
+                    'be specified as "foo.bar" to traverse nested dicts.'),
         make_option('--enabled-value', dest='enabled_value', type='str',
                     default=None, metavar='v', help='value of host variable '
                     'specified by --enabled-var that indicates host is '
@@ -547,7 +548,8 @@ class Command(NoArgsCommand):
                     'variables.'),
         make_option('--instance-id-var', dest='instance_id_var', type='str',
                     default=None, metavar='v', help='host variable that '
-                    'specifies the unique, immutable instance ID'),
+                    'specifies the unique, immutable instance ID, may be '
+                    'specified as "foo.bar" to traverse nested dicts.'),
     )
 
     def init_logging(self):
@@ -567,6 +569,51 @@ class Command(NoArgsCommand):
         self.logger.addHandler(handler)
         self.logger.propagate = False
 
+    def _get_instance_id(self, from_dict, default=''):
+        '''
+        Retrieve the instance ID from the given dict of host variables.
+        
+        The instance ID variable may be specified as 'foo.bar', in which case
+        the lookup will traverse into nested dicts, equivalent to:
+
+        from_dict.get('foo', {}).get('bar', default)
+        '''
+        instance_id = default
+        if getattr(self, 'instance_id_var', None):
+            for key in self.instance_id_var.split('.'):
+                if not hasattr(from_dict, 'get'):
+                    instance_id = default
+                    break
+                instance_id = from_dict.get(key, default)
+                from_dict = instance_id
+        return instance_id
+
+    def _get_enabled(self, from_dict, default=None):
+        '''
+        Retrieve the enabled state from the given dict of host variables.
+
+        The enabled variable may be specified as 'foo.bar', in which case
+        the lookup will traverse into nested dicts, equivalent to:
+
+        from_dict.get('foo', {}).get('bar', default)
+        '''
+        enabled = default
+        if getattr(self, 'enabled_var', None):
+            default = object()
+            for key in self.enabled_var.split('.'):
+                if not hasattr(from_dict, 'get'):
+                    enabled = default
+                    break
+                enabled = from_dict.get(key, default)
+                from_dict = enabled
+            if enabled is not default:
+                enabled_value = getattr(self, 'enabled_value', None)
+                if enabled_value is not None:
+                    enabled = bool(unicode(enabled_value) == unicode(enabled))
+                else:
+                    enabled = bool(enabled)
+        return enabled
+
     def load_inventory_from_database(self):
         '''
         Load inventory and related objects from the database.
@@ -643,9 +690,9 @@ class Command(NoArgsCommand):
             else:
                 host_qs = self.inventory.hosts.all()
             host_qs = host_qs.filter(active=True, instance_id='',
-                                     variables__contains=self.instance_id_var)
+                                     variables__contains=self.instance_id_var.split('.')[0])
             for host in host_qs:
-                instance_id = host.variables_dict.get(self.instance_id_var, '')
+                instance_id = self._get_instance_id(host.variables_dict)
                 if not instance_id:
                     continue
                 self.db_instance_id_map[instance_id] = host.pk
@@ -658,7 +705,7 @@ class Command(NoArgsCommand):
         self.mem_instance_id_map = {}
         if self.instance_id_var:
             for mem_host in self.all_group.all_hosts.values():
-                instance_id = mem_host.variables.get(self.instance_id_var, '')
+                instance_id = self._get_instance_id(mem_host.variables)
                 if not instance_id:
                     self.logger.warning('Host "%s" has no "%s" variable',
                                         mem_host.name, self.instance_id_var)
@@ -908,13 +955,7 @@ class Command(NoArgsCommand):
             db_host.variables = json.dumps(db_variables)
             update_fields.append('variables')
         # Update host enabled flag.
-        enabled = None
+        enabled = self._get_enabled(mem_host.variables)
-        if self.enabled_var and self.enabled_var in mem_host.variables:
-            value = mem_host.variables[self.enabled_var]
-            if self.enabled_value is not None:
-                enabled = bool(unicode(self.enabled_value) == unicode(value))
-            else:
-                enabled = bool(value)
         if enabled is not None and db_host.enabled != enabled:
             db_host.enabled = enabled
             update_fields.append('enabled')
@@ -924,10 +965,7 @@ class Command(NoArgsCommand):
             db_host.name = mem_host.name
             update_fields.append('name')
         # Update host instance_id.
-        if self.instance_id_var:
+        instance_id = self._get_instance_id(mem_host.variables)
-            instance_id = mem_host.variables.get(self.instance_id_var, '')
-        else:
-            instance_id = ''
         if instance_id != db_host.instance_id:
             old_instance_id = db_host.instance_id
             db_host.instance_id = instance_id
@@ -973,10 +1011,8 @@ class Command(NoArgsCommand):
         mem_host_name_map = {}
         mem_host_names_to_update = set(self.all_group.all_hosts.keys())
         for k,v in self.all_group.all_hosts.iteritems():
-            instance_id = ''
             mem_host_name_map[k] = v
-            if self.instance_id_var:
+            instance_id = self._get_instance_id(v.variables)
-                instance_id = v.variables.get(self.instance_id_var, '')
             if instance_id in self.db_instance_id_map:
                 mem_host_pk_map[self.db_instance_id_map[instance_id]] = v
             elif instance_id:
@@ -1023,16 +1059,11 @@ class Command(NoArgsCommand):
             mem_host = self.all_group.all_hosts[mem_host_name]
             host_attrs = dict(variables=json.dumps(mem_host.variables),
                               name=mem_host_name, description='imported')
-            enabled = None
+            enabled = self._get_enabled(mem_host.variables)
-            if self.enabled_var and self.enabled_var in mem_host.variables:
+            if enabled is not None:
-                value = mem_host.variables[self.enabled_var]
-                if self.enabled_value is not None:
-                    enabled = bool(unicode(self.enabled_value) == unicode(value))
-                else:
-                    enabled = bool(value)
                 host_attrs['enabled'] = enabled
             if self.instance_id_var:
-                instance_id = mem_host.variables.get(self.instance_id_var, '')
+                instance_id = self._get_instance_id(mem_host.variables)
                 host_attrs['instance_id'] = instance_id
             db_host = self.inventory.hosts.create(**host_attrs)
             if enabled is False:

awx/main/tasks.py

@@ -986,8 +986,24 @@ class RunInventoryUpdate(BaseTask):
                                   username=credential.username,
                                   password=decrypt_field(credential, "password"),
                                   project_name=credential.project)
-            private_state = str(inventory_update.source_vars_dict.get("private", "true"))
+            private_state = str(inventory_update.source_vars_dict.get('private', 'true'))
-            openstack_data = {"clouds": {"devstack": {"private": private_state, "auth": openstack_auth}}}
+            # Retrieve cache path from inventory update vars if available,
+            # otherwise create a temporary cache path only for this update.
+            cache = inventory_update.source_vars_dict.get('cache', {})
+            if not isinstance(cache, dict):
+                cache = {}
+            if not cache.get('path', ''):
+                cache_path = tempfile.mkdtemp(prefix='openstack_cache', dir=kwargs.get('private_data_dir', None))
+                cache['path'] = cache_path
+            openstack_data = {
+                'clouds': {
+                    'devstack': {
+                        'private': private_state,
+                        'auth': openstack_auth,
+                    },
+                },
+                'cache': cache,
+            }
             return dict(cloud_credential=yaml.safe_dump(openstack_data, default_flow_style=False, allow_unicode=True))
 
         cp = ConfigParser.ConfigParser()

awx/main/tests/inventory.py

@@ -2000,6 +2000,7 @@ class InventoryUpdatesTest(BaseTransactionTest):
                                                project=api_project)
         inventory_source = self.update_inventory_source(self.group, source='openstack', credential=credential)
         self.check_inventory_source(inventory_source)
+        self.assertFalse(self.group.all_hosts.filter(instance_id='').exists())
 
     def test_update_from_azure(self):
         source_username = getattr(settings, 'TEST_AZURE_USERNAME', '')

awx/plugins/inventory/openstack.py

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this software.  If not, see <http://www.gnu.org/licenses/>.
 
-# The OpenStack Inventory module uses os-client-config for configuation.
+# The OpenStack Inventory module uses os-client-config for configuration.
 # https://github.com/stackforge/os-client-config
 # This means it will either:
 #  - Respect normal OS_* environment variables like other OpenStack tools
@@ -50,13 +50,16 @@ import shade
 
 class OpenStackInventory(object):
 
-    def __init__(self, refresh=False):
+    def __init__(self, private=False, refresh=False):
-        config_files = [ os.environ.get('OPENSTACK_CONFIG_FILE', None) 
+        config_files = os_client_config.config.CONFIG_FILES
-            or '/etc/ansible/openstack.yml' ]
+        if os.environ.get('OPENSTACK_CONFIG_FILE', None):
+            config_files.insert(0, os.environ['OPENSTACK_CONFIG_FILE'])
+        config_files.append('/etc/ansible/openstack.yml')
         self.openstack_config = os_client_config.config.OpenStackConfig(
             config_files)
         self.clouds = shade.openstack_clouds(self.openstack_config)
-        self.refresh = True
+        self.private = private
+        self.refresh = refresh
 
         self.cache_max_age = self.openstack_config.get_cache_max_age()
         cache_path = self.openstack_config.get_cache_path()
@@ -92,8 +95,11 @@ class OpenStackInventory(object):
         hostvars = collections.defaultdict(dict)
 
         for cloud in self.clouds:
+            cloud.private = cloud.private or self.private
+
             # Cycle on servers
             for server in cloud.list_servers():
+
                 meta = cloud.get_server_meta(server)
 
                 if 'interface_ip' not in meta['server_vars']:
@@ -101,9 +107,9 @@ class OpenStackInventory(object):
                     continue
 
                 server_vars = meta['server_vars']
-                hostvars[server.name]['ansible_ssh_host'] = server_vars['interface_ip']
+                hostvars[server.name][
+                    'ansible_ssh_host'] = server_vars['interface_ip']
                 hostvars[server.name]['openstack'] = server_vars
-                hostvars[server.name]['id'] = server_vars['id']
 
                 for group in meta['groups']:
                     groups[group].append(server.name)
@@ -129,6 +135,9 @@ class OpenStackInventory(object):
 
 def parse_args():
     parser = argparse.ArgumentParser(description='OpenStack Inventory Module')
+    parser.add_argument('--private',
+                        action='store_true',
+                        help='Use private address for ansible host')
     parser.add_argument('--refresh', action='store_true',
                         help='Refresh cached information')
     group = parser.add_mutually_exclusive_group(required=True)
@@ -141,14 +150,13 @@ def parse_args():
 def main():
     args = parse_args()
     try:
-        inventory = OpenStackInventory(args.refresh)
+        inventory = OpenStackInventory(args.private, args.refresh)
         if args.list:
             inventory.list_instances()
         elif args.host:
             inventory.get_host(args.host)
     except shade.OpenStackCloudException as e:
-        print(e.message)
+        sys.exit(e.message)
-        sys.exit(1)
     sys.exit(0)
 
 

awx/settings/defaults.py

@@ -534,7 +534,7 @@ OPENSTACK_ENABLED_VALUE = 'ACTIVE'
 OPENSTACK_GROUP_FILTER = r'^.+$'
 OPENSTACK_HOST_FILTER = r'^.+$'
 OPENSTACK_EXCLUDE_EMPTY_GROUPS = True
-OPENSTACK_INSTANCE_ID_VAR = "id"
+OPENSTACK_INSTANCE_ID_VAR = 'openstack.id'
 
 # ---------------------
 # -- Activity Stream --

awx/settings/local_settings.py.example

@@ -492,6 +492,12 @@ TEST_VMWARE_HOST = ''
 TEST_VMWARE_USER = ''
 TEST_VMWARE_PASSWORD = ''
 
+# OpenStack credentials
+TEST_OPENSTACK_HOST = ''
+TEST_OPENSTACK_USER = ''
+TEST_OPENSTACK_PASSWORD = ''
+TEST_OPENSTACK_PROJECT = ''
+
 # Azure credentials.
 TEST_AZURE_USERNAME = ''
 TEST_AZURE_KEY_DATA = ''