From 622fbc116b2b39fd8a9a1b7859918062b162f887 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Tue, 29 Jan 2019 14:00:54 -0500 Subject: [PATCH] move script injection logic to inventory file --- awx/main/models/inventory.py | 300 ++++++++++++++++-- awx/main/tasks.py | 226 +------------ .../test_inventory_source_injectors.py | 15 +- 3 files changed, 294 insertions(+), 247 deletions(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index a2f74caa5c..6b5a806fae 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -14,6 +14,7 @@ import yaml import configparser import stat import tempfile +from io import StringIO from distutils.version import LooseVersion as Version # Django @@ -57,7 +58,7 @@ from awx.main.models.notifications import ( NotificationTemplate, JobNotificationMixin, ) -from awx.main.utils import _inventory_updates, region_sorting +from awx.main.utils import _inventory_updates, region_sorting, get_licenser __all__ = ['Inventory', 'Host', 'Group', 'InventorySource', 'InventoryUpdate', @@ -1314,9 +1315,9 @@ class InventorySourceOptions(BaseModel): ) return None - def get_inventory_plugin_name(self): + def get_inventory_plugin_name(self, ansible_version): if self.source in InventorySourceOptions.injectors: - return InventorySourceOptions.injectors[self.source].plugin_name + return InventorySourceOptions.injectors[self.source](ansible_version).use_plugin_name() if self.source in CLOUD_PROVIDERS or self.source == 'custom': # TODO: today, all vendored sources are scripts # in future release inventory plugins will replace these @@ -1822,8 +1823,9 @@ class CustomInventoryScript(CommonModelNameNotUnique, ResourceMixin): # TODO: move these to their own file somewhere? class PluginFileInjector(object): - plugin_name = None - initial_version = None + plugin_name = None # Ansible core name used to reference plugin + initial_version = None # at what version do we switch to the plugin + ini_env_reference = None # env var name that points to old ini config file def __init__(self, ansible_version): # This is InventoryOptions instance, could be source or inventory update @@ -1842,6 +1844,13 @@ class PluginFileInjector(object): Version(self.ansible_version) >= Version(self.initial_version) ) + def use_plugin_name(self): + if self.should_use_plugin() and self.plugin_name is not None: + return self.plugin_name + else: + # By default, if the plugin cannot be used, then we use old vendored scripts + return 'script' + @staticmethod def get_builtin_injector(source): from awx.main.models.credential import injectors as builtin_injectors @@ -1850,30 +1859,35 @@ class PluginFileInjector(object): return None return getattr(builtin_injectors, cred_kind) - def build_env(self, inventory_update, env, private_data_dir): + def build_env(self, inventory_update, env, private_data_dir, private_data_files): if self.should_use_plugin(): - injector_env = self.get_plugin_env(inventory_update, private_data_dir) + injector_env = self.get_plugin_env(inventory_update, private_data_dir, private_data_files) else: - injector_env = self.get_script_env(inventory_update, private_data_dir) + injector_env = self.get_script_env(inventory_update, private_data_dir, private_data_files) env.update(injector_env) return env - def get_plugin_env(self, inventory_update, private_data_dir, safe=False): - return self.get_script_env(inventory_update, private_data_dir, safe) + def get_plugin_env(self, inventory_update, private_data_dir, private_data_files, safe=False): + return self.get_script_env(inventory_update, private_data_dir, private_data_files, safe) - def get_script_env(self, inventory_update, private_data_dir, safe=False): + def get_script_env(self, inventory_update, private_data_dir, private_data_files, safe=False): """By default, we will apply the standard managed_by_tower injectors for the script injection """ injected_env = {} credential = inventory_update.get_cloud_credential() builtin_injector = self.get_builtin_injector(inventory_update.source) - if builtin_injector is None: - return {} - builtin_injector(credential, injected_env, private_data_dir) - if safe: - from awx.main.models.credential import build_safe_env - injected_env = build_safe_env(injected_env) + if builtin_injector is not None: + builtin_injector(credential, injected_env, private_data_dir) + if safe: + from awx.main.models.credential import build_safe_env + injected_env = build_safe_env(injected_env) + + # Put in env var reference to private data files, if relevant + if self.ini_env_reference: + cred_data = private_data_files.get('credentials', '') + injected_env[self.ini_env_reference] = cred_data[credential] + return injected_env def build_private_data(self, inventory_update, private_data_dir): @@ -1888,13 +1902,90 @@ class PluginFileInjector(object): def build_plugin_private_data(self, inventory_update, private_data_dir): return None + @staticmethod + def dump_cp(cp, credential): + """Dump config parser data and return it as a string. + Helper method intended for use by build_script_private_data + """ + if cp.sections(): + f = StringIO() + cp.write(f) + private_data = private_data = {'credentials': {}} + private_data['credentials'][credential] = f.getvalue() + return private_data + else: + return None + + +class azure_rm(PluginFileInjector): + ini_env_reference = 'AZURE_INI_PATH' + + def build_script_private_data(self, inventory_update, private_data_dir): + cp = configparser.RawConfigParser() + section = 'azure' + cp.add_section(section) + cp.set(section, 'include_powerstate', 'yes') + cp.set(section, 'group_by_resource_group', 'yes') + cp.set(section, 'group_by_location', 'yes') + cp.set(section, 'group_by_tag', 'yes') + + if inventory_update.source_regions and 'all' not in inventory_update.source_regions: + cp.set( + section, 'locations', + ','.join([x.strip() for x in inventory_update.source_regions.split(',')]) + ) + + azure_rm_opts = dict(inventory_update.source_vars_dict.items()) + for k, v in azure_rm_opts.items(): + cp.set(section, k, str(v)) + return self.dump_cp(cp, inventory_update.get_cloud_credential()) + + +class ec2(PluginFileInjector): + ini_env_reference = 'EC2_INI_PATH' + + def build_script_private_data(self, inventory_update, private_data_dir): + cp = configparser.RawConfigParser() + # Build custom ec2.ini for ec2 inventory script to use. + section = 'ec2' + cp.add_section(section) + ec2_opts = dict(inventory_update.source_vars_dict.items()) + regions = inventory_update.source_regions or 'all' + regions = ','.join([x.strip() for x in regions.split(',')]) + regions_blacklist = ','.join(settings.EC2_REGIONS_BLACKLIST) + ec2_opts['regions'] = regions + ec2_opts.setdefault('regions_exclude', regions_blacklist) + ec2_opts.setdefault('destination_variable', 'public_dns_name') + ec2_opts.setdefault('vpc_destination_variable', 'ip_address') + ec2_opts.setdefault('route53', 'False') + ec2_opts.setdefault('all_instances', 'True') + ec2_opts.setdefault('all_rds_instances', 'False') + ec2_opts.setdefault('include_rds_clusters', 'False') + ec2_opts.setdefault('rds', 'False') + ec2_opts.setdefault('nested_groups', 'True') + ec2_opts.setdefault('elasticache', 'False') + ec2_opts.setdefault('stack_filters', 'False') + if inventory_update.instance_filters: + ec2_opts.setdefault('instance_filters', inventory_update.instance_filters) + group_by = [x.strip().lower() for x in inventory_update.group_by.split(',') if x.strip()] + for choice in inventory_update.get_ec2_group_by_choices(): + value = bool((group_by and choice[0] in group_by) or (not group_by and choice[0] != 'instance_id')) + ec2_opts.setdefault('group_by_%s' % choice[0], str(value)) + if 'cache_path' not in ec2_opts: + cache_path = tempfile.mkdtemp(prefix='ec2_cache', dir=private_data_dir) + ec2_opts['cache_path'] = cache_path + ec2_opts.setdefault('cache_max_age', '300') + for k, v in ec2_opts.items(): + cp.set(section, k, str(v)) + return self.dump_cp(cp, inventory_update.get_cloud_credential()) + class gce(PluginFileInjector): plugin_name = 'gcp_compute' initial_version = '2.6' - def get_script_env(self, inventory_update, private_data_dir): - env = super(gce, self).get_script_env(inventory_update, private_data_dir) + def get_script_env(self, inventory_update, private_data_dir, private_data_files): + env = super(gce, self).get_script_env(inventory_update, private_data_dir, private_data_files) env['GCE_ZONE'] = inventory_update.source_regions if inventory_update.source_regions != 'all' else '' # noqa # by default, the GCE inventory source caches results on disk for @@ -1926,14 +2017,94 @@ class gce(PluginFileInjector): ret['zones'] = inventory_update.source_regions.split(',') return ret - def get_plugin_env(self, inventory_update, private_data_dir, safe=False): + def get_plugin_env(self, inventory_update, private_data_dir, private_data_files, safe=False): # gce wants everything defined in inventory & cred files return {} +class vmware(PluginFileInjector): + ini_env_reference = 'VMWARE_INI_PATH' + + def build_script_private_data(self, inventory_update, private_data_dir): + cp = configparser.RawConfigParser() + credential = inventory_update.get_cloud_credential() + + # Allow custom options to vmware inventory script. + section = 'vmware' + cp.add_section(section) + cp.set('vmware', 'cache_max_age', '0') + cp.set('vmware', 'validate_certs', str(settings.VMWARE_VALIDATE_CERTS)) + cp.set('vmware', 'username', credential.get_input('username', default='')) + cp.set('vmware', 'password', credential.get_input('password', default='')) + cp.set('vmware', 'server', credential.get_input('host', default='')) + + vmware_opts = dict(inventory_update.source_vars_dict.items()) + if inventory_update.instance_filters: + vmware_opts.setdefault('host_filters', inventory_update.instance_filters) + if inventory_update.group_by: + vmware_opts.setdefault('groupby_patterns', inventory_update.group_by) + + for k, v in vmware_opts.items(): + cp.set(section, k, str(v)) + + return self.dump_cp(cp, credential) + + +class openstack(PluginFileInjector): + ini_env_reference = 'OS_CLIENT_CONFIG_FILE' + + def build_script_private_data(self, inventory_update, private_data_dir): + credential = inventory_update.get_cloud_credential() + private_data = {'credentials': {}} + + openstack_auth = dict(auth_url=credential.get_input('host', default=''), + username=credential.get_input('username', default=''), + password=credential.get_input('password', default=''), + project_name=credential.get_input('project', default='')) + if credential.has_input('domain'): + openstack_auth['domain_name'] = credential.get_input('domain', default='') + + private_state = inventory_update.source_vars_dict.get('private', True) + verify_state = credential.get_input('verify_ssl', default=True) + # Retrieve cache path from inventory update vars if available, + # otherwise create a temporary cache path only for this update. + cache = inventory_update.source_vars_dict.get('cache', {}) + if not isinstance(cache, dict): + cache = {} + if not cache.get('path', ''): + cache_path = tempfile.mkdtemp(prefix='openstack_cache', dir=private_data_dir) + cache['path'] = cache_path + openstack_data = { + 'clouds': { + 'devstack': { + 'private': private_state, + 'verify': verify_state, + 'auth': openstack_auth, + }, + }, + 'cache': cache, + } + ansible_variables = { + 'use_hostnames': True, + 'expand_hostvars': False, + 'fail_on_errors': True, + } + provided_count = 0 + for var_name in ansible_variables: + if var_name in inventory_update.source_vars_dict: + ansible_variables[var_name] = inventory_update.source_vars_dict[var_name] + provided_count += 1 + if provided_count: + openstack_data['ansible'] = ansible_variables + private_data['credentials'][credential] = yaml.safe_dump( + openstack_data, default_flow_style=False, allow_unicode=True + ) + return private_data + + class rhv(PluginFileInjector): - def get_script_env(self, inventory_update, private_data_dir): + def get_script_env(self, inventory_update, private_data_dir, private_data_files): """Unlike the others, ovirt uses the custom credential templating """ env = {'INVENTORY_UPDATE_ID': inventory_update.pk} @@ -1947,5 +2118,92 @@ class rhv(PluginFileInjector): return env +class satellite6(PluginFileInjector): + ini_env_reference = 'FOREMAN_INI_PATH' + + def build_script_private_data(self, inventory_update, private_data_dir): + cp = configparser.RawConfigParser() + credential = inventory_update.get_cloud_credential() + + section = 'foreman' + cp.add_section(section) + + group_patterns = '[]' + group_prefix = 'foreman_' + want_hostcollections = 'False' + foreman_opts = dict(inventory_update.source_vars_dict.items()) + foreman_opts.setdefault('ssl_verify', 'False') + for k, v in foreman_opts.items(): + if k == 'satellite6_group_patterns' and isinstance(v, str): + group_patterns = v + elif k == 'satellite6_group_prefix' and isinstance(v, str): + group_prefix = v + elif k == 'satellite6_want_hostcollections' and isinstance(v, bool): + want_hostcollections = v + else: + cp.set(section, k, str(v)) + + if credential: + cp.set(section, 'url', credential.get_input('host', default='')) + cp.set(section, 'user', credential.get_input('username', default='')) + cp.set(section, 'password', credential.get_input('password', default='')) + + section = 'ansible' + cp.add_section(section) + cp.set(section, 'group_patterns', group_patterns) + cp.set(section, 'want_facts', 'True') + cp.set(section, 'want_hostcollections', str(want_hostcollections)) + cp.set(section, 'group_prefix', group_prefix) + + section = 'cache' + cp.add_section(section) + cp.set(section, 'path', '/tmp') + cp.set(section, 'max_age', '0') + + return self.dump_cp(cp, credential) + + +class cloudforms(PluginFileInjector): + ini_env_reference = 'CLOUDFORMS_INI_PATH' + + def build_script_private_data(self, inventory_update, private_data_dir): + cp = configparser.RawConfigParser() + credential = inventory_update.get_cloud_credential() + + section = 'cloudforms' + cp.add_section(section) + + if credential: + cp.set(section, 'url', credential.get_input('host', default='')) + cp.set(section, 'username', credential.get_input('username', default='')) + cp.set(section, 'password', credential.get_input('password', default='')) + cp.set(section, 'ssl_verify', "false") + + cloudforms_opts = dict(inventory_update.source_vars_dict.items()) + for opt in ['version', 'purge_actions', 'clean_group_keys', 'nest_tags', 'suffix', 'prefer_ipv4']: + if opt in cloudforms_opts: + cp.set(section, opt, str(cloudforms_opts[opt])) + + section = 'cache' + cp.add_section(section) + cp.set(section, 'max_age', "0") + cache_path = tempfile.mkdtemp( + prefix='cloudforms_cache', + dir=private_data_dir + ) + cp.set(section, 'path', cache_path) + + return self.dump_cp(cp, credential) + + +class tower(PluginFileInjector): + + def get_script_env(self, inventory_update, private_data_dir, private_data_files): + env = super(tower, self).get_script_env(inventory_update, private_data_dir, private_data_files) + env['TOWER_INVENTORY'] = inventory_update.instance_filters + env['TOWER_LICENSE_TYPE'] = get_licenser().validate().get('license_type', 'unlicensed') + return env + + for cls in PluginFileInjector.__subclasses__(): InventorySourceOptions.injectors[cls.__name__] = cls diff --git a/awx/main/tasks.py b/awx/main/tasks.py index bf542050ee..a3f031a800 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -3,7 +3,6 @@ # Python from collections import OrderedDict, namedtuple -import configparser import errno import fnmatch import functools @@ -24,7 +23,6 @@ try: import psutil except Exception: psutil = None -from io import StringIO import urllib.parse as urlparse # Django @@ -1940,194 +1938,8 @@ class RunInventoryUpdate(BaseTask): If no private data is needed, return None. """ if inventory_update.source in InventorySource.injectors: - injector = InventorySource.injectors[inventory_update.source](kwargs['ansible_version']) - return injector.build_private_data(inventory_update, kwargs.get('private_data_dir', None)) - - private_data = {'credentials': {}} - credential = inventory_update.get_cloud_credential() - - if inventory_update.source == 'openstack': - openstack_auth = dict(auth_url=credential.get_input('host', default=''), - username=credential.get_input('username', default=''), - password=credential.get_input('password', default=''), - project_name=credential.get_input('project', default='')) - if credential.has_input('domain'): - openstack_auth['domain_name'] = credential.get_input('domain', default='') - - private_state = inventory_update.source_vars_dict.get('private', True) - verify_state = credential.get_input('verify_ssl', default=True) - # Retrieve cache path from inventory update vars if available, - # otherwise create a temporary cache path only for this update. - cache = inventory_update.source_vars_dict.get('cache', {}) - if not isinstance(cache, dict): - cache = {} - if not cache.get('path', ''): - cache_path = tempfile.mkdtemp(prefix='openstack_cache', dir=private_data_dir) - cache['path'] = cache_path - openstack_data = { - 'clouds': { - 'devstack': { - 'private': private_state, - 'verify': verify_state, - 'auth': openstack_auth, - }, - }, - 'cache': cache, - } - ansible_variables = { - 'use_hostnames': True, - 'expand_hostvars': False, - 'fail_on_errors': True, - } - provided_count = 0 - for var_name in ansible_variables: - if var_name in inventory_update.source_vars_dict: - ansible_variables[var_name] = inventory_update.source_vars_dict[var_name] - provided_count += 1 - if provided_count: - openstack_data['ansible'] = ansible_variables - private_data['credentials'][credential] = yaml.safe_dump( - openstack_data, default_flow_style=False, allow_unicode=True - ) - return private_data - - cp = configparser.RawConfigParser() - # Build custom ec2.ini for ec2 inventory script to use. - if inventory_update.source == 'ec2': - section = 'ec2' - cp.add_section(section) - ec2_opts = dict(inventory_update.source_vars_dict.items()) - regions = inventory_update.source_regions or 'all' - regions = ','.join([x.strip() for x in regions.split(',')]) - regions_blacklist = ','.join(settings.EC2_REGIONS_BLACKLIST) - ec2_opts['regions'] = regions - ec2_opts.setdefault('regions_exclude', regions_blacklist) - ec2_opts.setdefault('destination_variable', 'public_dns_name') - ec2_opts.setdefault('vpc_destination_variable', 'ip_address') - ec2_opts.setdefault('route53', 'False') - ec2_opts.setdefault('all_instances', 'True') - ec2_opts.setdefault('all_rds_instances', 'False') - ec2_opts.setdefault('include_rds_clusters', 'False') - ec2_opts.setdefault('rds', 'False') - ec2_opts.setdefault('nested_groups', 'True') - ec2_opts.setdefault('elasticache', 'False') - ec2_opts.setdefault('stack_filters', 'False') - if inventory_update.instance_filters: - ec2_opts.setdefault('instance_filters', inventory_update.instance_filters) - group_by = [x.strip().lower() for x in inventory_update.group_by.split(',') if x.strip()] - for choice in inventory_update.get_ec2_group_by_choices(): - value = bool((group_by and choice[0] in group_by) or (not group_by and choice[0] != 'instance_id')) - ec2_opts.setdefault('group_by_%s' % choice[0], str(value)) - if 'cache_path' not in ec2_opts: - cache_path = tempfile.mkdtemp(prefix='ec2_cache', dir=private_data_dir) - ec2_opts['cache_path'] = cache_path - ec2_opts.setdefault('cache_max_age', '300') - for k, v in ec2_opts.items(): - cp.set(section, k, str(v)) - # Allow custom options to vmware inventory script. - elif inventory_update.source == 'vmware': - - section = 'vmware' - cp.add_section(section) - cp.set('vmware', 'cache_max_age', '0') - cp.set('vmware', 'validate_certs', str(settings.VMWARE_VALIDATE_CERTS)) - cp.set('vmware', 'username', credential.get_input('username', default='')) - cp.set('vmware', 'password', credential.get_input('password', default='')) - cp.set('vmware', 'server', credential.get_input('host', default='')) - - vmware_opts = dict(inventory_update.source_vars_dict.items()) - if inventory_update.instance_filters: - vmware_opts.setdefault('host_filters', inventory_update.instance_filters) - if inventory_update.group_by: - vmware_opts.setdefault('groupby_patterns', inventory_update.group_by) - - for k, v in vmware_opts.items(): - cp.set(section, k, str(v)) - - elif inventory_update.source == 'satellite6': - section = 'foreman' - cp.add_section(section) - - group_patterns = '[]' - group_prefix = 'foreman_' - want_hostcollections = 'False' - foreman_opts = dict(inventory_update.source_vars_dict.items()) - foreman_opts.setdefault('ssl_verify', 'False') - for k, v in foreman_opts.items(): - if k == 'satellite6_group_patterns' and isinstance(v, str): - group_patterns = v - elif k == 'satellite6_group_prefix' and isinstance(v, str): - group_prefix = v - elif k == 'satellite6_want_hostcollections' and isinstance(v, bool): - want_hostcollections = v - else: - cp.set(section, k, str(v)) - - if credential: - cp.set(section, 'url', credential.get_input('host', default='')) - cp.set(section, 'user', credential.get_input('username', default='')) - cp.set(section, 'password', credential.get_input('password', default='')) - - section = 'ansible' - cp.add_section(section) - cp.set(section, 'group_patterns', group_patterns) - cp.set(section, 'want_facts', 'True') - cp.set(section, 'want_hostcollections', str(want_hostcollections)) - cp.set(section, 'group_prefix', group_prefix) - - section = 'cache' - cp.add_section(section) - cp.set(section, 'path', '/tmp') - cp.set(section, 'max_age', '0') - - elif inventory_update.source == 'cloudforms': - section = 'cloudforms' - cp.add_section(section) - - if credential: - cp.set(section, 'url', credential.get_input('host', default='')) - cp.set(section, 'username', credential.get_input('username', default='')) - cp.set(section, 'password', credential.get_input('password', default='')) - cp.set(section, 'ssl_verify', "false") - - cloudforms_opts = dict(inventory_update.source_vars_dict.items()) - for opt in ['version', 'purge_actions', 'clean_group_keys', 'nest_tags', 'suffix', 'prefer_ipv4']: - if opt in cloudforms_opts: - cp.set(section, opt, str(cloudforms_opts[opt])) - - section = 'cache' - cp.add_section(section) - cp.set(section, 'max_age', "0") - cache_path = tempfile.mkdtemp( - prefix='cloudforms_cache', - dir=private_data_dir - ) - cp.set(section, 'path', cache_path) - - elif inventory_update.source == 'azure_rm': - section = 'azure' - cp.add_section(section) - cp.set(section, 'include_powerstate', 'yes') - cp.set(section, 'group_by_resource_group', 'yes') - cp.set(section, 'group_by_location', 'yes') - cp.set(section, 'group_by_tag', 'yes') - - if inventory_update.source_regions and 'all' not in inventory_update.source_regions: - cp.set( - section, 'locations', - ','.join([x.strip() for x in inventory_update.source_regions.split(',')]) - ) - - azure_rm_opts = dict(inventory_update.source_vars_dict.items()) - for k, v in azure_rm_opts.items(): - cp.set(section, k, str(v)) - - # Return INI content. - if cp.sections(): - f = StringIO() - cp.write(f) - private_data['credentials'][credential] = f.getvalue() - return private_data + injector = InventorySource.injectors[inventory_update.source](self.get_ansible_version(inventory_update)) + return injector.build_private_data(inventory_update, private_data_dir) def build_passwords(self, inventory_update, runtime_passwords): """Build a dictionary of authentication/credential information for @@ -2171,41 +1983,13 @@ class RunInventoryUpdate(BaseTask): env['INVENTORY_SOURCE_ID'] = str(inventory_update.inventory_source_id) env['INVENTORY_UPDATE_ID'] = str(inventory_update.pk) env.update(STANDARD_INVENTORY_UPDATE_ENV) - plugin_name = inventory_update.get_inventory_plugin_name() + plugin_name = inventory_update.get_inventory_plugin_name(self.get_ansible_version(inventory_update)) if plugin_name is not None: env['ANSIBLE_INVENTORY_ENABLED'] = plugin_name - # Set environment variables specific to each source. - # - # These are set here and then read in by the various Ansible inventory - # modules, which will actually do the inventory sync. - # - # The inventory modules are vendored in AWX in the - # `awx/plugins/inventory` directory; those files should be kept in - # sync with those in Ansible core at all times. - - ini_mapping = { - 'ec2': 'EC2_INI_PATH', - 'vmware': 'VMWARE_INI_PATH', - 'azure_rm': 'AZURE_INI_PATH', - 'openstack': 'OS_CLIENT_CONFIG_FILE', - 'satellite6': 'FOREMAN_INI_PATH', - 'cloudforms': 'CLOUDFORMS_INI_PATH' - } - if inventory_update.source in ini_mapping: - cred_data = private_data_files.get('credentials', {}) - env[ini_mapping[inventory_update.source]] = cred_data.get( - inventory_update.get_cloud_credential(), '' - ) - if inventory_update.source in InventorySource.injectors: - # TODO: mapping from credential.kind to inventory_source.source injector = InventorySource.injectors[inventory_update.source](self.get_ansible_version(inventory_update)) - env = injector.build_env(inventory_update, env, private_data_dir) - - if inventory_update.source == 'tower': - env['TOWER_INVENTORY'] = inventory_update.instance_filters - env['TOWER_LICENSE_TYPE'] = get_licenser().validate()['license_type'] + env = injector.build_env(inventory_update, env, private_data_dir, private_data_files) if inventory_update.source in ['scm', 'custom']: for env_k in inventory_update.source_vars_dict: @@ -2289,7 +2073,7 @@ class RunInventoryUpdate(BaseTask): if src in InventorySource.injectors: injector = InventorySource.injectors[inventory_update.source](self.get_ansible_version(inventory_update)) if injector.should_use_plugin(): - content = injector.inventory_contents(inventory_update, kwargs['private_data_dir']) + content = injector.inventory_contents(inventory_update, private_data_dir) # must be a statically named file inventory_path = os.path.join(private_data_dir, injector.filename) with open(inventory_path, 'w') as f: diff --git a/awx/main/tests/functional/test_inventory_source_injectors.py b/awx/main/tests/functional/test_inventory_source_injectors.py index 68ab8ee354..9c17659594 100644 --- a/awx/main/tests/functional/test_inventory_source_injectors.py +++ b/awx/main/tests/functional/test_inventory_source_injectors.py @@ -117,9 +117,9 @@ def read_content(private_data_dir, env, inventory_update): return a dictionary `content` with file contents, keyed off environment variable that references the file """ - references = {} + inverse_env = {} for key, value in env.items(): - references[value] = key + inverse_env[value] = key cache_file_regex = re.compile(r'/tmp/awx_{0}_[a-zA-Z0-9_]+/{1}_cache[a-zA-Z0-9_]+'.format( inventory_update.id, inventory_update.source) @@ -127,8 +127,11 @@ def read_content(private_data_dir, env, inventory_update): private_key_regex = re.compile(r'-----BEGIN ENCRYPTED PRIVATE KEY-----.*-----END ENCRYPTED PRIVATE KEY-----') dir_contents = {} + references = {} for filename in os.listdir(private_data_dir): abs_file_path = os.path.join(private_data_dir, filename) + if abs_file_path in inverse_env: + references[abs_file_path] = inverse_env[abs_file_path] try: with open(abs_file_path, 'r') as f: dir_contents[abs_file_path] = f.read() @@ -172,8 +175,8 @@ def read_content(private_data_dir, env, inventory_update): for abs_file_path, file_content in dir_contents.items(): if abs_file_path not in references: raise AssertionError( - "File {} is not referenced by any other file or environment variable:\n{}\n{}".format( - abs_file_path, json.dumps(env, indent=4), json.dumps(dir_contents, indent=4))) + "File {} is not referenced. References and files:\n{}\n{}".format( + abs_file_path, json.dumps(references, indent=4), json.dumps(dir_contents, indent=4))) reference_key = references[abs_file_path] file_content = private_key_regex.sub('{{private_key}}', file_content) content[reference_key] = file_content @@ -228,7 +231,7 @@ def test_inventory_script_structure(this_kind, script_or_plugin, inventory): base_dir = os.path.join(DATA, script_or_plugin) if not os.path.exists(base_dir): os.mkdir(base_dir) - ref_dir = os.path.join(base_dir, this_kind) + ref_dir = os.path.join(base_dir, this_kind) # this_kind is a global if set_files: create_reference_data(ref_dir, content) pytest.skip('You set MAKE_INVENTORY_REFERENCE_FILES, so this created files, unset to run actual test.') @@ -250,5 +253,7 @@ def test_inventory_script_structure(this_kind, script_or_plugin, inventory): with mock.patch('awx.main.models.inventory.PluginFileInjector.should_use_plugin', return_value=use_plugin): # Also do not send websocket status updates with mock.patch.object(UnifiedJob, 'websocket_emit_status', mock.Mock()): + # The point of this test is that we replace run_pexpect with assertions with mock.patch('awx.main.expect.run.run_pexpect', substitute_run): + # so this sets up everything for a run and then yields control over to substitute_run task.run(inventory_update.pk)