diff --git a/awx/main/tasks.py b/awx/main/tasks.py index cd6a3ce580..b188414da6 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -2285,15 +2285,18 @@ class RunInventoryUpdate(BaseTask): if src in InventorySource.injectors: cloud_cred = inventory_update.get_cloud_credential() injector = InventorySource.injectors[cloud_cred.kind](self.get_ansible_version(inventory_update)) - content = injector.inventory_contents(inventory_update) - # must be a statically named file - inventory_path = os.path.join(private_data_dir, injector.filename) - with open(inventory_path, 'w') as f: - f.write(content) - os.chmod(inventory_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + if injector.should_use_plugin(): + content = injector.inventory_contents(inventory_update) + # must be a statically named file + inventory_path = os.path.join(private_data_dir, injector.filename) + with open(inventory_path, 'w') as f: + f.write(content) + os.chmod(inventory_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + else: + # Use the vendored script path + inventory_path = self.get_path_to('..', 'plugins', 'inventory', '%s.py' % src) else: - # Get the path to the inventory plugin, and append it to our - # arguments. + # TODO: get rid of this else after all CLOUD_PROVIDERS have injectors written inventory_path = self.get_path_to('..', 'plugins', 'inventory', '%s.py' % src) elif src == 'scm': inventory_path = inventory_update.get_actual_source_path() diff --git a/awx/main/tests/data/inventory/plugins/gce/GCE_CREDENTIALS_FILE_PATH b/awx/main/tests/data/inventory/plugins/gce/GCE_CREDENTIALS_FILE_PATH new file mode 100644 index 0000000000..814f5081a7 --- /dev/null +++ b/awx/main/tests/data/inventory/plugins/gce/GCE_CREDENTIALS_FILE_PATH @@ -0,0 +1,7 @@ +{ + "type": "service_account", + "private_key": "{{private_key}}", + "client_email": "fooo", + "project_id": "fooo", + "token_uri": "https://accounts.google.com/o/oauth2/token" +} \ No newline at end of file diff --git a/awx/main/tests/data/inventory/plugins/gce/gcp_compute.yml b/awx/main/tests/data/inventory/plugins/gce/gcp_compute.yml new file mode 100644 index 0000000000..04bc7dcbe8 --- /dev/null +++ b/awx/main/tests/data/inventory/plugins/gce/gcp_compute.yml @@ -0,0 +1,9 @@ +auth_kind: serviceaccount +filters: null +plugin: gcp_compute +projects: +- fooo +service_account_file: creds.json +zones: +- us-east4-a +- us-west1-b diff --git a/awx/main/tests/data/inventory/scripts/azure_rm/AZURE_INI_PATH b/awx/main/tests/data/inventory/scripts/azure_rm/AZURE_INI_PATH new file mode 100644 index 0000000000..9e39d5560b --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/azure_rm/AZURE_INI_PATH @@ -0,0 +1,8 @@ +[azure] +include_powerstate = yes +group_by_resource_group = yes +group_by_location = yes +group_by_tag = yes +locations = southcentralus,westus +base_source_var = value_of_var + diff --git a/awx/main/tests/data/inventory/scripts/cloudforms/CLOUDFORMS_INI_PATH b/awx/main/tests/data/inventory/scripts/cloudforms/CLOUDFORMS_INI_PATH new file mode 100644 index 0000000000..de6b1f15bf --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/cloudforms/CLOUDFORMS_INI_PATH @@ -0,0 +1,16 @@ +[cloudforms] +url = https://foo.invalid +username = fooo +password = fooo +ssl_verify = false +version = 2.4 +purge_actions = maybe +clean_group_keys = this_key +nest_tags = yes +suffix = .ppt +prefer_ipv4 = yes + +[cache] +max_age = 0 +path = {{ cache_dir }} + diff --git a/awx/main/tests/data/inventory/scripts/cloudforms/cache_dir b/awx/main/tests/data/inventory/scripts/cloudforms/cache_dir new file mode 100644 index 0000000000..973b0fcf00 --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/cloudforms/cache_dir @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/awx/main/tests/data/inventory/scripts/ec2/EC2_INI_PATH b/awx/main/tests/data/inventory/scripts/ec2/EC2_INI_PATH new file mode 100644 index 0000000000..d43f16ab06 --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/ec2/EC2_INI_PATH @@ -0,0 +1,31 @@ +[ec2] +base_source_var = value_of_var +regions = us-east-2,ap-south-1 +regions_exclude = us-gov-west-1,cn-north-1 +destination_variable = public_dns_name +vpc_destination_variable = ip_address +route53 = False +all_instances = True +all_rds_instances = False +include_rds_clusters = False +rds = False +nested_groups = True +elasticache = False +stack_filters = False +instance_filters = foobaa +group_by_ami_id = False +group_by_availability_zone = False +group_by_aws_account = False +group_by_instance_id = False +group_by_instance_state = False +group_by_platform = False +group_by_instance_type = False +group_by_key_pair = False +group_by_region = False +group_by_security_group = False +group_by_tag_keys = False +group_by_tag_none = False +group_by_vpc_id = False +cache_path = {{ cache_dir }} +cache_max_age = 300 + diff --git a/awx/main/tests/data/inventory/scripts/ec2/cache_dir b/awx/main/tests/data/inventory/scripts/ec2/cache_dir new file mode 100644 index 0000000000..973b0fcf00 --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/ec2/cache_dir @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/awx/main/tests/data/inventory/scripts/gce/GCE_CREDENTIALS_FILE_PATH b/awx/main/tests/data/inventory/scripts/gce/GCE_CREDENTIALS_FILE_PATH new file mode 100644 index 0000000000..814f5081a7 --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/gce/GCE_CREDENTIALS_FILE_PATH @@ -0,0 +1,7 @@ +{ + "type": "service_account", + "private_key": "{{private_key}}", + "client_email": "fooo", + "project_id": "fooo", + "token_uri": "https://accounts.google.com/o/oauth2/token" +} \ No newline at end of file diff --git a/awx/main/tests/data/inventory/scripts/gce/GCE_INI_PATH b/awx/main/tests/data/inventory/scripts/gce/GCE_INI_PATH new file mode 100644 index 0000000000..0362854f72 --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/gce/GCE_INI_PATH @@ -0,0 +1,3 @@ +[cache] +cache_max_age = 0 + diff --git a/awx/main/tests/data/inventory/scripts/openstack/OS_CLIENT_CONFIG_FILE b/awx/main/tests/data/inventory/scripts/openstack/OS_CLIENT_CONFIG_FILE new file mode 100644 index 0000000000..fd93f64817 --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/openstack/OS_CLIENT_CONFIG_FILE @@ -0,0 +1,15 @@ +ansible: + expand_hostvars: true + fail_on_errors: true + use_hostnames: false +cache: + path: {{ cache_dir }} +clouds: + devstack: + auth: + auth_url: https://foo.invalid + domain_name: fooo + password: fooo + project_name: fooo + username: fooo + private: false diff --git a/awx/main/tests/data/inventory/scripts/openstack/cache_dir b/awx/main/tests/data/inventory/scripts/openstack/cache_dir new file mode 100644 index 0000000000..973b0fcf00 --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/openstack/cache_dir @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/awx/main/tests/data/inventory/scripts/rhv/OVIRT_INI_PATH b/awx/main/tests/data/inventory/scripts/rhv/OVIRT_INI_PATH new file mode 100644 index 0000000000..06c2180789 --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/rhv/OVIRT_INI_PATH @@ -0,0 +1,5 @@ +[ovirt] +ovirt_url=https://foo.invalid +ovirt_username=fooo +ovirt_password=fooo +ovirt_ca_file=fooo \ No newline at end of file diff --git a/awx/main/tests/data/inventory/scripts/satellite6/FOREMAN_INI_PATH b/awx/main/tests/data/inventory/scripts/satellite6/FOREMAN_INI_PATH new file mode 100644 index 0000000000..9fb7639c44 --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/satellite6/FOREMAN_INI_PATH @@ -0,0 +1,17 @@ +[foreman] +base_source_var = value_of_var +ssl_verify = False +url = https://foo.invalid +user = fooo +password = fooo + +[ansible] +group_patterns = foo_group_patterns +want_facts = True +want_hostcollections = True +group_prefix = foo_group_prefix + +[cache] +path = /tmp +max_age = 0 + diff --git a/awx/main/tests/data/inventory/scripts/vmware/VMWARE_INI_PATH b/awx/main/tests/data/inventory/scripts/vmware/VMWARE_INI_PATH new file mode 100644 index 0000000000..8a4d8a7700 --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/vmware/VMWARE_INI_PATH @@ -0,0 +1,10 @@ +[vmware] +cache_max_age = 0 +validate_certs = False +username = fooo +password = fooo +server = https://foo.invalid +base_source_var = value_of_var +host_filters = foobaa +groupby_patterns = fouo + diff --git a/awx/main/tests/functional/test_inventory_source_injectors.py b/awx/main/tests/functional/test_inventory_source_injectors.py new file mode 100644 index 0000000000..2277f597ca --- /dev/null +++ b/awx/main/tests/functional/test_inventory_source_injectors.py @@ -0,0 +1,260 @@ +import pytest +from unittest import mock +import os +import json +import re + +from awx.main.tasks import RunInventoryUpdate +from awx.main.models import InventorySource, Credential, CredentialType, UnifiedJob +from awx.main.constants import CLOUD_PROVIDERS +from awx.main.tests import data + +DATA = os.path.join(os.path.dirname(data.__file__), 'inventory') + +TEST_SOURCE_FIELDS = { + 'vmware': { + 'instance_filters': 'foobaa', + 'group_by': 'fouo' + }, + 'ec2': { + 'instance_filters': 'foobaa', + 'group_by': 'fouo', + 'source_regions': 'us-east-2,ap-south-1' + }, + 'gce': { + 'source_regions': 'us-east4-a,us-west1-b' # surfaced as env var + }, + 'azure_rm': { + 'source_regions': 'southcentralus,westus' + }, +} + +INI_TEST_VARS = { + 'ec2': {}, + 'gce': {}, + 'openstack': { + 'private': False, + 'use_hostnames': False, + 'expand_hostvars': True, + 'fail_on_errors': True + }, + 'rhv': {}, # there are none + 'tower': {}, # there are none + 'vmware': { + # setting VMWARE_VALIDATE_CERTS is duplicated with env var + }, + 'azure_rm': {}, # there are none + 'satellite6': { + 'satellite6_group_patterns': 'foo_group_patterns', + 'satellite6_group_prefix': 'foo_group_prefix', + 'satellite6_want_hostcollections': True + }, + 'cloudforms': { + 'version': '2.4', + 'purge_actions': 'maybe', + 'clean_group_keys': 'this_key', + 'nest_tags': 'yes', + 'suffix': '.ppt', + 'prefer_ipv4': 'yes' + } +} + + +def generate_fake_var(element): + """Given a credential type field element, makes up something acceptable. + """ + if element['type'] == 'string': + if element.get('format', None) == 'ssh_private_key': + # this example came from the internet + return '\n'.join([ + '-----BEGIN ENCRYPTED PRIVATE KEY-----' + 'MIIBpjBABgkqhkiG9w0BBQ0wMzAbBgkqhkiG9w0BBQwwDgQI5yNCu9T5SnsCAggA' + 'MBQGCCqGSIb3DQMHBAhJISTgOAxtYwSCAWDXK/a1lxHIbRZHud1tfRMR4ROqkmr4' + 'kVGAnfqTyGptZUt3ZtBgrYlFAaZ1z0wxnhmhn3KIbqebI4w0cIL/3tmQ6eBD1Ad1' + 'nSEjUxZCuzTkimXQ88wZLzIS9KHc8GhINiUu5rKWbyvWA13Ykc0w65Ot5MSw3cQc' + 'w1LEDJjTculyDcRQgiRfKH5376qTzukileeTrNebNq+wbhY1kEPAHojercB7d10E' + '+QcbjJX1Tb1Zangom1qH9t/pepmV0Hn4EMzDs6DS2SWTffTddTY4dQzvksmLkP+J' + 'i8hkFIZwUkWpT9/k7MeklgtTiy0lR/Jj9CxAIQVxP8alLWbIqwCNRApleSmqtitt' + 'Z+NdsuNeTm3iUaPGYSw237tjLyVE6pr0EJqLv7VUClvJvBnH2qhQEtWYB9gvE1dS' + 'BioGu40pXVfjiLqhEKVVVEoHpI32oMkojhCGJs8Oow4bAxkzQFCtuWB1' + '-----END ENCRYPTED PRIVATE KEY-----' + ]) + if element['id'] == 'host': + return 'https://foo.invalid' + return 'fooo' + elif element['type'] == 'boolean': + return False + raise Exception('No generator written for {} type'.format(element.get('type', 'unknown'))) + + +def credential_kind(source): + """Given the inventory source kind, return expected credential kind + """ + return source.replace('ec2', 'aws') + + +@pytest.fixture +def fake_credential_factory(source): + ct = CredentialType.defaults[credential_kind(source)]() + ct.save() + + inputs = {} + var_specs = {} # pivoted version of inputs + for element in ct.inputs.get('fields'): + var_specs[element['id']] = element + for var in var_specs.keys(): + inputs[var] = generate_fake_var(var_specs[var]) + + return Credential.objects.create( + credential_type=ct, + inputs=inputs + ) + + +def read_content(private_data_dir, env, inventory_update): + """Read the environmental data laid down by the task system + template out private and secret data so they will be readable and predictable + return a dictionary `content` with file contents, keyed off environment variable + that references the file + """ + references = {} + for key, value in env.items(): + references[value] = key + + cache_file_regex = re.compile(r'/tmp/awx_{0}_[a-zA-Z0-9_]+/{1}_cache[a-zA-Z0-9_]+'.format( + inventory_update.id, inventory_update.source) + ) + private_key_regex = re.compile(r'-----BEGIN ENCRYPTED PRIVATE KEY-----.*-----END ENCRYPTED PRIVATE KEY-----') + + dir_contents = {} + for filename in os.listdir(private_data_dir): + abs_file_path = os.path.join(private_data_dir, filename) + try: + with open(abs_file_path, 'r') as f: + dir_contents[abs_file_path] = f.read() + # Declare a reference to inventory plugin file if it exists + if abs_file_path.endswith('.yml') and 'plugin: ' in dir_contents[abs_file_path]: + references[abs_file_path] = filename # plugin filenames are universal + except IsADirectoryError: + dir_contents[abs_file_path] = '' + print('dir contents') + print(dir_contents) + + # Declare cross-file references, also use special keywords if it is the cache + cache_referenced = False + cache_present = False + for abs_file_path, file_content in dir_contents.copy().items(): + if cache_file_regex.match(file_content): + cache_referenced = True + for target_path in dir_contents.keys(): + if target_path in file_content: + if target_path in references: + raise AssertionError( + 'File {} is referenced by env var or other file as well as file {}:\n{}\n{}'.format( + target_path, abs_file_path, json.dumps(env, indent=4), json.dumps(dir_contents, indent=4))) + else: + if cache_file_regex.match(target_path): + cache_present = True + if os.path.isdir(target_path): + keyword = 'cache_dir' + else: + keyword = 'cache_file' + references[target_path] = keyword + new_file_content = cache_file_regex.sub('{{ ' + keyword + ' }}', file_content) + else: + references[target_path] = 'file_reference' + new_file_content = file_content.replace(target_path, '{{ file_reference }}') + dir_contents[abs_file_path] = new_file_content + if cache_referenced and not cache_present: + raise AssertionError( + 'A cache file was referenced but never created, files:\n{}'.format( + json.dumps(dir_contents, indent=4))) + print('dir contents') + print(dir_contents) + + content = {} + for abs_file_path, file_content in dir_contents.items(): + if abs_file_path not in references: + raise AssertionError( + "File {} is not referenced by any other file or environment variable:\n{}\n{}".format( + abs_file_path, json.dumps(env, indent=4), json.dumps(dir_contents, indent=4))) + reference_key = references[abs_file_path] + file_content = private_key_regex.sub('{{private_key}}', file_content) + content[reference_key] = file_content + print(' contents') + print(content) + + return content + + +def create_reference_data(ref_dir, content): + if not os.path.exists(ref_dir): + os.mkdir(ref_dir) + for env_name, content in content.items(): + with open(os.path.join(ref_dir, env_name), 'w') as f: + f.write(content) + + +@pytest.mark.django_db +@pytest.mark.parametrize('this_kind', CLOUD_PROVIDERS) +@pytest.mark.parametrize('script_or_plugin', ['scripts', 'plugins']) +def test_inventory_script_structure(this_kind, script_or_plugin, inventory): + src_vars = dict(base_source_var='value_of_var') + if this_kind in INI_TEST_VARS: + src_vars.update(INI_TEST_VARS[this_kind]) + extra_kwargs = {} + if this_kind in TEST_SOURCE_FIELDS: + extra_kwargs.update(TEST_SOURCE_FIELDS[this_kind]) + inventory_source = InventorySource.objects.create( + inventory=inventory, + source=this_kind, + source_vars=src_vars, + **extra_kwargs + ) + inventory_source.credentials.add(fake_credential_factory(this_kind)) + inventory_update = inventory_source.create_unified_job() + task = RunInventoryUpdate() + + use_plugin = bool(script_or_plugin == 'plugins') + if use_plugin: + if this_kind not in InventorySource.injectors: + pytest.skip('Injector class for this source is not written yet') + elif InventorySource.injectors[this_kind].initial_version is None: + pytest.skip('Use of inventory plugin is not enabled for this source') + + def substitute_run(args, cwd, env, stdout_handle, **_kw): + """This method will replace run_pexpect + instead of running, it will read the private data directory contents + It will make assertions that the contents are correct + If MAKE_INVENTORY_REFERENCE_FILES is set, it will produce reference files + """ + private_data_dir = env['AWX_PRIVATE_DATA_DIR'] + set_files = bool(os.getenv("MAKE_INVENTORY_REFERENCE_FILES", 'false').lower()[0] not in ['f', '0']) + content = read_content(private_data_dir, env, inventory_update) + base_dir = os.path.join(DATA, script_or_plugin) + if not os.path.exists(base_dir): + os.mkdir(base_dir) + ref_dir = os.path.join(base_dir, this_kind) + if set_files: + create_reference_data(ref_dir, content) + pytest.skip('You set MAKE_INVENTORY_REFERENCE_FILES, so this created files, unset to run actual test.') + else: + expected_file_list = os.listdir(ref_dir) + assert set(expected_file_list) == set(content.keys()), ( + 'Inventory update runtime environment does not have expected files' + ) + for f_name in expected_file_list: + with open(os.path.join(ref_dir, f_name), 'r') as f: + ref_content = f.read() + assert content[f_name] == ref_content + return ('successful', 0) + + # Mock this so that it will not send events to the callback receiver + # because doing so in pytest land creates large explosions + with mock.patch('awx.main.queue.CallbackQueueDispatcher.dispatch', lambda self, obj: None): + # Force the update to use the script injector + with mock.patch('awx.main.models.inventory.PluginFileInjector.should_use_plugin', return_value=use_plugin): + # Also do not send websocket status updates + with mock.patch.object(UnifiedJob, 'websocket_emit_status', mock.Mock()): + with mock.patch('awx.main.expect.run.run_pexpect', substitute_run): + task.run(inventory_update.pk) diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index fb9ddb0c94..e442643b8c 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -2035,11 +2035,12 @@ class TestInventoryUpdateCredentials(TestJobExecution): assert 'cache' in config.sections() assert config.getint('cache', 'cache_max_age') == 0 + # Change the initial version of the inventory plugin to force use of script + with mock.patch('awx.main.models.inventory.gce.initial_version', None): + run('') - run('') - - inventory_update.source_regions = 'us-east-4' - run('us-east-4') + self.instance.source_regions = 'us-east-4' + run('us-east-4') def test_openstack_source(self, inventory_update, private_data_dir): task = tasks.RunInventoryUpdate()