diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index 101949045f..cb0c93e552 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -530,7 +530,8 @@ class Command(NoArgsCommand): 'to load'), make_option('--enabled-var', dest='enabled_var', type='str', default=None, metavar='v', help='host variable used to ' - 'set/clear enabled flag when host is online/offline'), + 'set/clear enabled flag when host is online/offline, may ' + 'be specified as "foo.bar" to traverse nested dicts.'), make_option('--enabled-value', dest='enabled_value', type='str', default=None, metavar='v', help='value of host variable ' 'specified by --enabled-var that indicates host is ' @@ -547,7 +548,8 @@ class Command(NoArgsCommand): 'variables.'), make_option('--instance-id-var', dest='instance_id_var', type='str', default=None, metavar='v', help='host variable that ' - 'specifies the unique, immutable instance ID'), + 'specifies the unique, immutable instance ID, may be ' + 'specified as "foo.bar" to traverse nested dicts.'), ) def init_logging(self): @@ -567,6 +569,51 @@ class Command(NoArgsCommand): self.logger.addHandler(handler) self.logger.propagate = False + def _get_instance_id(self, from_dict, default=''): + ''' + Retrieve the instance ID from the given dict of host variables. + + The instance ID variable may be specified as 'foo.bar', in which case + the lookup will traverse into nested dicts, equivalent to: + + from_dict.get('foo', {}).get('bar', default) + ''' + instance_id = default + if getattr(self, 'instance_id_var', None): + for key in self.instance_id_var.split('.'): + if not hasattr(from_dict, 'get'): + instance_id = default + break + instance_id = from_dict.get(key, default) + from_dict = instance_id + return instance_id + + def _get_enabled(self, from_dict, default=None): + ''' + Retrieve the enabled state from the given dict of host variables. + + The enabled variable may be specified as 'foo.bar', in which case + the lookup will traverse into nested dicts, equivalent to: + + from_dict.get('foo', {}).get('bar', default) + ''' + enabled = default + if getattr(self, 'enabled_var', None): + default = object() + for key in self.enabled_var.split('.'): + if not hasattr(from_dict, 'get'): + enabled = default + break + enabled = from_dict.get(key, default) + from_dict = enabled + if enabled is not default: + enabled_value = getattr(self, 'enabled_value', None) + if enabled_value is not None: + enabled = bool(unicode(enabled_value) == unicode(enabled)) + else: + enabled = bool(enabled) + return enabled + def load_inventory_from_database(self): ''' Load inventory and related objects from the database. @@ -643,9 +690,9 @@ class Command(NoArgsCommand): else: host_qs = self.inventory.hosts.all() host_qs = host_qs.filter(active=True, instance_id='', - variables__contains=self.instance_id_var) + variables__contains=self.instance_id_var.split('.')[0]) for host in host_qs: - instance_id = host.variables_dict.get(self.instance_id_var, '') + instance_id = self._get_instance_id(host.variables_dict) if not instance_id: continue self.db_instance_id_map[instance_id] = host.pk @@ -658,7 +705,7 @@ class Command(NoArgsCommand): self.mem_instance_id_map = {} if self.instance_id_var: for mem_host in self.all_group.all_hosts.values(): - instance_id = mem_host.variables.get(self.instance_id_var, '') + instance_id = self._get_instance_id(mem_host.variables) if not instance_id: self.logger.warning('Host "%s" has no "%s" variable', mem_host.name, self.instance_id_var) @@ -908,13 +955,7 @@ class Command(NoArgsCommand): db_host.variables = json.dumps(db_variables) update_fields.append('variables') # Update host enabled flag. - enabled = None - if self.enabled_var and self.enabled_var in mem_host.variables: - value = mem_host.variables[self.enabled_var] - if self.enabled_value is not None: - enabled = bool(unicode(self.enabled_value) == unicode(value)) - else: - enabled = bool(value) + enabled = self._get_enabled(mem_host.variables) if enabled is not None and db_host.enabled != enabled: db_host.enabled = enabled update_fields.append('enabled') @@ -924,10 +965,7 @@ class Command(NoArgsCommand): db_host.name = mem_host.name update_fields.append('name') # Update host instance_id. - if self.instance_id_var: - instance_id = mem_host.variables.get(self.instance_id_var, '') - else: - instance_id = '' + instance_id = self._get_instance_id(mem_host.variables) if instance_id != db_host.instance_id: old_instance_id = db_host.instance_id db_host.instance_id = instance_id @@ -973,10 +1011,8 @@ class Command(NoArgsCommand): mem_host_name_map = {} mem_host_names_to_update = set(self.all_group.all_hosts.keys()) for k,v in self.all_group.all_hosts.iteritems(): - instance_id = '' mem_host_name_map[k] = v - if self.instance_id_var: - instance_id = v.variables.get(self.instance_id_var, '') + instance_id = self._get_instance_id(v.variables) if instance_id in self.db_instance_id_map: mem_host_pk_map[self.db_instance_id_map[instance_id]] = v elif instance_id: @@ -1023,16 +1059,11 @@ class Command(NoArgsCommand): mem_host = self.all_group.all_hosts[mem_host_name] host_attrs = dict(variables=json.dumps(mem_host.variables), name=mem_host_name, description='imported') - enabled = None - if self.enabled_var and self.enabled_var in mem_host.variables: - value = mem_host.variables[self.enabled_var] - if self.enabled_value is not None: - enabled = bool(unicode(self.enabled_value) == unicode(value)) - else: - enabled = bool(value) + enabled = self._get_enabled(mem_host.variables) + if enabled is not None: host_attrs['enabled'] = enabled if self.instance_id_var: - instance_id = mem_host.variables.get(self.instance_id_var, '') + instance_id = self._get_instance_id(mem_host.variables) host_attrs['instance_id'] = instance_id db_host = self.inventory.hosts.create(**host_attrs) if enabled is False: diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 63e7056750..b06021ae96 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -986,8 +986,24 @@ class RunInventoryUpdate(BaseTask): username=credential.username, password=decrypt_field(credential, "password"), project_name=credential.project) - private_state = str(inventory_update.source_vars_dict.get("private", "true")) - openstack_data = {"clouds": {"devstack": {"private": private_state, "auth": openstack_auth}}} + private_state = str(inventory_update.source_vars_dict.get('private', 'true')) + # Retrieve cache path from inventory update vars if available, + # otherwise create a temporary cache path only for this update. + cache = inventory_update.source_vars_dict.get('cache', {}) + if not isinstance(cache, dict): + cache = {} + if not cache.get('path', ''): + cache_path = tempfile.mkdtemp(prefix='openstack_cache', dir=kwargs.get('private_data_dir', None)) + cache['path'] = cache_path + openstack_data = { + 'clouds': { + 'devstack': { + 'private': private_state, + 'auth': openstack_auth, + }, + }, + 'cache': cache, + } return dict(cloud_credential=yaml.safe_dump(openstack_data, default_flow_style=False, allow_unicode=True)) cp = ConfigParser.ConfigParser() diff --git a/awx/main/tests/inventory.py b/awx/main/tests/inventory.py index d2118af139..27590ec988 100644 --- a/awx/main/tests/inventory.py +++ b/awx/main/tests/inventory.py @@ -2000,6 +2000,7 @@ class InventoryUpdatesTest(BaseTransactionTest): project=api_project) inventory_source = self.update_inventory_source(self.group, source='openstack', credential=credential) self.check_inventory_source(inventory_source) + self.assertFalse(self.group.all_hosts.filter(instance_id='').exists()) def test_update_from_azure(self): source_username = getattr(settings, 'TEST_AZURE_USERNAME', '') diff --git a/awx/plugins/inventory/openstack.py b/awx/plugins/inventory/openstack.py index ecf4b9ce00..ae5fa78dce 100755 --- a/awx/plugins/inventory/openstack.py +++ b/awx/plugins/inventory/openstack.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with this software. If not, see . -# The OpenStack Inventory module uses os-client-config for configuation. +# The OpenStack Inventory module uses os-client-config for configuration. # https://github.com/stackforge/os-client-config # This means it will either: # - Respect normal OS_* environment variables like other OpenStack tools @@ -50,13 +50,16 @@ import shade class OpenStackInventory(object): - def __init__(self, refresh=False): - config_files = [ os.environ.get('OPENSTACK_CONFIG_FILE', None) - or '/etc/ansible/openstack.yml' ] + def __init__(self, private=False, refresh=False): + config_files = os_client_config.config.CONFIG_FILES + if os.environ.get('OPENSTACK_CONFIG_FILE', None): + config_files.insert(0, os.environ['OPENSTACK_CONFIG_FILE']) + config_files.append('/etc/ansible/openstack.yml') self.openstack_config = os_client_config.config.OpenStackConfig( config_files) self.clouds = shade.openstack_clouds(self.openstack_config) - self.refresh = True + self.private = private + self.refresh = refresh self.cache_max_age = self.openstack_config.get_cache_max_age() cache_path = self.openstack_config.get_cache_path() @@ -92,8 +95,11 @@ class OpenStackInventory(object): hostvars = collections.defaultdict(dict) for cloud in self.clouds: + cloud.private = cloud.private or self.private + # Cycle on servers for server in cloud.list_servers(): + meta = cloud.get_server_meta(server) if 'interface_ip' not in meta['server_vars']: @@ -101,9 +107,9 @@ class OpenStackInventory(object): continue server_vars = meta['server_vars'] - hostvars[server.name]['ansible_ssh_host'] = server_vars['interface_ip'] + hostvars[server.name][ + 'ansible_ssh_host'] = server_vars['interface_ip'] hostvars[server.name]['openstack'] = server_vars - hostvars[server.name]['id'] = server_vars['id'] for group in meta['groups']: groups[group].append(server.name) @@ -129,6 +135,9 @@ class OpenStackInventory(object): def parse_args(): parser = argparse.ArgumentParser(description='OpenStack Inventory Module') + parser.add_argument('--private', + action='store_true', + help='Use private address for ansible host') parser.add_argument('--refresh', action='store_true', help='Refresh cached information') group = parser.add_mutually_exclusive_group(required=True) @@ -141,14 +150,13 @@ def parse_args(): def main(): args = parse_args() try: - inventory = OpenStackInventory(args.refresh) + inventory = OpenStackInventory(args.private, args.refresh) if args.list: inventory.list_instances() elif args.host: inventory.get_host(args.host) except shade.OpenStackCloudException as e: - print(e.message) - sys.exit(1) + sys.exit(e.message) sys.exit(0) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 05fe8b0b6c..97b4024efb 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -534,7 +534,7 @@ OPENSTACK_ENABLED_VALUE = 'ACTIVE' OPENSTACK_GROUP_FILTER = r'^.+$' OPENSTACK_HOST_FILTER = r'^.+$' OPENSTACK_EXCLUDE_EMPTY_GROUPS = True -OPENSTACK_INSTANCE_ID_VAR = "id" +OPENSTACK_INSTANCE_ID_VAR = 'openstack.id' # --------------------- # -- Activity Stream -- diff --git a/awx/settings/local_settings.py.example b/awx/settings/local_settings.py.example index 62624b475a..042614c32d 100644 --- a/awx/settings/local_settings.py.example +++ b/awx/settings/local_settings.py.example @@ -492,6 +492,12 @@ TEST_VMWARE_HOST = '' TEST_VMWARE_USER = '' TEST_VMWARE_PASSWORD = '' +# OpenStack credentials +TEST_OPENSTACK_HOST = '' +TEST_OPENSTACK_USER = '' +TEST_OPENSTACK_PASSWORD = '' +TEST_OPENSTACK_PROJECT = '' + # Azure credentials. TEST_AZURE_USERNAME = '' TEST_AZURE_KEY_DATA = ''