From 5cbcfbe0c68b67f45b6da0233d98fa341f0d3840 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Tue, 15 Jan 2019 07:28:55 -0500 Subject: [PATCH] Port inventory source injector tests to functional tests This new batch of tests assures that the injector logic for inventory source in their old script version remains untouched with the refactoring underway. Plugins are also tested by the same means of comparing to reference files, these will be used to assure that all parameters that used to be respected are still respected in the plugin system. --- awx/main/tasks.py | 19 +- .../plugins/gce/GCE_CREDENTIALS_FILE_PATH | 7 + .../inventory/plugins/gce/gcp_compute.yml | 9 + .../inventory/scripts/azure_rm/AZURE_INI_PATH | 8 + .../scripts/cloudforms/CLOUDFORMS_INI_PATH | 16 ++ .../inventory/scripts/cloudforms/cache_dir | 1 + .../data/inventory/scripts/ec2/EC2_INI_PATH | 31 +++ .../data/inventory/scripts/ec2/cache_dir | 1 + .../scripts/gce/GCE_CREDENTIALS_FILE_PATH | 7 + .../data/inventory/scripts/gce/GCE_INI_PATH | 3 + .../scripts/openstack/OS_CLIENT_CONFIG_FILE | 15 + .../inventory/scripts/openstack/cache_dir | 1 + .../data/inventory/scripts/rhv/OVIRT_INI_PATH | 5 + .../scripts/satellite6/FOREMAN_INI_PATH | 17 ++ .../inventory/scripts/vmware/VMWARE_INI_PATH | 10 + .../test_inventory_source_injectors.py | 260 ++++++++++++++++++ awx/main/tests/unit/test_tasks.py | 9 +- 17 files changed, 407 insertions(+), 12 deletions(-) create mode 100644 awx/main/tests/data/inventory/plugins/gce/GCE_CREDENTIALS_FILE_PATH create mode 100644 awx/main/tests/data/inventory/plugins/gce/gcp_compute.yml create mode 100644 awx/main/tests/data/inventory/scripts/azure_rm/AZURE_INI_PATH create mode 100644 awx/main/tests/data/inventory/scripts/cloudforms/CLOUDFORMS_INI_PATH create mode 100644 awx/main/tests/data/inventory/scripts/cloudforms/cache_dir create mode 100644 awx/main/tests/data/inventory/scripts/ec2/EC2_INI_PATH create mode 100644 awx/main/tests/data/inventory/scripts/ec2/cache_dir create mode 100644 awx/main/tests/data/inventory/scripts/gce/GCE_CREDENTIALS_FILE_PATH create mode 100644 awx/main/tests/data/inventory/scripts/gce/GCE_INI_PATH create mode 100644 awx/main/tests/data/inventory/scripts/openstack/OS_CLIENT_CONFIG_FILE create mode 100644 awx/main/tests/data/inventory/scripts/openstack/cache_dir create mode 100644 awx/main/tests/data/inventory/scripts/rhv/OVIRT_INI_PATH create mode 100644 awx/main/tests/data/inventory/scripts/satellite6/FOREMAN_INI_PATH create mode 100644 awx/main/tests/data/inventory/scripts/vmware/VMWARE_INI_PATH create mode 100644 awx/main/tests/functional/test_inventory_source_injectors.py diff --git a/awx/main/tasks.py b/awx/main/tasks.py index cd6a3ce580..b188414da6 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -2285,15 +2285,18 @@ class RunInventoryUpdate(BaseTask): if src in InventorySource.injectors: cloud_cred = inventory_update.get_cloud_credential() injector = InventorySource.injectors[cloud_cred.kind](self.get_ansible_version(inventory_update)) - content = injector.inventory_contents(inventory_update) - # must be a statically named file - inventory_path = os.path.join(private_data_dir, injector.filename) - with open(inventory_path, 'w') as f: - f.write(content) - os.chmod(inventory_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + if injector.should_use_plugin(): + content = injector.inventory_contents(inventory_update) + # must be a statically named file + inventory_path = os.path.join(private_data_dir, injector.filename) + with open(inventory_path, 'w') as f: + f.write(content) + os.chmod(inventory_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) + else: + # Use the vendored script path + inventory_path = self.get_path_to('..', 'plugins', 'inventory', '%s.py' % src) else: - # Get the path to the inventory plugin, and append it to our - # arguments. + # TODO: get rid of this else after all CLOUD_PROVIDERS have injectors written inventory_path = self.get_path_to('..', 'plugins', 'inventory', '%s.py' % src) elif src == 'scm': inventory_path = inventory_update.get_actual_source_path() diff --git a/awx/main/tests/data/inventory/plugins/gce/GCE_CREDENTIALS_FILE_PATH b/awx/main/tests/data/inventory/plugins/gce/GCE_CREDENTIALS_FILE_PATH new file mode 100644 index 0000000000..814f5081a7 --- /dev/null +++ b/awx/main/tests/data/inventory/plugins/gce/GCE_CREDENTIALS_FILE_PATH @@ -0,0 +1,7 @@ +{ + "type": "service_account", + "private_key": "{{private_key}}", + "client_email": "fooo", + "project_id": "fooo", + "token_uri": "https://accounts.google.com/o/oauth2/token" +} \ No newline at end of file diff --git a/awx/main/tests/data/inventory/plugins/gce/gcp_compute.yml b/awx/main/tests/data/inventory/plugins/gce/gcp_compute.yml new file mode 100644 index 0000000000..04bc7dcbe8 --- /dev/null +++ b/awx/main/tests/data/inventory/plugins/gce/gcp_compute.yml @@ -0,0 +1,9 @@ +auth_kind: serviceaccount +filters: null +plugin: gcp_compute +projects: +- fooo +service_account_file: creds.json +zones: +- us-east4-a +- us-west1-b diff --git a/awx/main/tests/data/inventory/scripts/azure_rm/AZURE_INI_PATH b/awx/main/tests/data/inventory/scripts/azure_rm/AZURE_INI_PATH new file mode 100644 index 0000000000..9e39d5560b --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/azure_rm/AZURE_INI_PATH @@ -0,0 +1,8 @@ +[azure] +include_powerstate = yes +group_by_resource_group = yes +group_by_location = yes +group_by_tag = yes +locations = southcentralus,westus +base_source_var = value_of_var + diff --git a/awx/main/tests/data/inventory/scripts/cloudforms/CLOUDFORMS_INI_PATH b/awx/main/tests/data/inventory/scripts/cloudforms/CLOUDFORMS_INI_PATH new file mode 100644 index 0000000000..de6b1f15bf --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/cloudforms/CLOUDFORMS_INI_PATH @@ -0,0 +1,16 @@ +[cloudforms] +url = https://foo.invalid +username = fooo +password = fooo +ssl_verify = false +version = 2.4 +purge_actions = maybe +clean_group_keys = this_key +nest_tags = yes +suffix = .ppt +prefer_ipv4 = yes + +[cache] +max_age = 0 +path = {{ cache_dir }} + diff --git a/awx/main/tests/data/inventory/scripts/cloudforms/cache_dir b/awx/main/tests/data/inventory/scripts/cloudforms/cache_dir new file mode 100644 index 0000000000..973b0fcf00 --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/cloudforms/cache_dir @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/awx/main/tests/data/inventory/scripts/ec2/EC2_INI_PATH b/awx/main/tests/data/inventory/scripts/ec2/EC2_INI_PATH new file mode 100644 index 0000000000..d43f16ab06 --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/ec2/EC2_INI_PATH @@ -0,0 +1,31 @@ +[ec2] +base_source_var = value_of_var +regions = us-east-2,ap-south-1 +regions_exclude = us-gov-west-1,cn-north-1 +destination_variable = public_dns_name +vpc_destination_variable = ip_address +route53 = False +all_instances = True +all_rds_instances = False +include_rds_clusters = False +rds = False +nested_groups = True +elasticache = False +stack_filters = False +instance_filters = foobaa +group_by_ami_id = False +group_by_availability_zone = False +group_by_aws_account = False +group_by_instance_id = False +group_by_instance_state = False +group_by_platform = False +group_by_instance_type = False +group_by_key_pair = False +group_by_region = False +group_by_security_group = False +group_by_tag_keys = False +group_by_tag_none = False +group_by_vpc_id = False +cache_path = {{ cache_dir }} +cache_max_age = 300 + diff --git a/awx/main/tests/data/inventory/scripts/ec2/cache_dir b/awx/main/tests/data/inventory/scripts/ec2/cache_dir new file mode 100644 index 0000000000..973b0fcf00 --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/ec2/cache_dir @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/awx/main/tests/data/inventory/scripts/gce/GCE_CREDENTIALS_FILE_PATH b/awx/main/tests/data/inventory/scripts/gce/GCE_CREDENTIALS_FILE_PATH new file mode 100644 index 0000000000..814f5081a7 --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/gce/GCE_CREDENTIALS_FILE_PATH @@ -0,0 +1,7 @@ +{ + "type": "service_account", + "private_key": "{{private_key}}", + "client_email": "fooo", + "project_id": "fooo", + "token_uri": "https://accounts.google.com/o/oauth2/token" +} \ No newline at end of file diff --git a/awx/main/tests/data/inventory/scripts/gce/GCE_INI_PATH b/awx/main/tests/data/inventory/scripts/gce/GCE_INI_PATH new file mode 100644 index 0000000000..0362854f72 --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/gce/GCE_INI_PATH @@ -0,0 +1,3 @@ +[cache] +cache_max_age = 0 + diff --git a/awx/main/tests/data/inventory/scripts/openstack/OS_CLIENT_CONFIG_FILE b/awx/main/tests/data/inventory/scripts/openstack/OS_CLIENT_CONFIG_FILE new file mode 100644 index 0000000000..fd93f64817 --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/openstack/OS_CLIENT_CONFIG_FILE @@ -0,0 +1,15 @@ +ansible: + expand_hostvars: true + fail_on_errors: true + use_hostnames: false +cache: + path: {{ cache_dir }} +clouds: + devstack: + auth: + auth_url: https://foo.invalid + domain_name: fooo + password: fooo + project_name: fooo + username: fooo + private: false diff --git a/awx/main/tests/data/inventory/scripts/openstack/cache_dir b/awx/main/tests/data/inventory/scripts/openstack/cache_dir new file mode 100644 index 0000000000..973b0fcf00 --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/openstack/cache_dir @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/awx/main/tests/data/inventory/scripts/rhv/OVIRT_INI_PATH b/awx/main/tests/data/inventory/scripts/rhv/OVIRT_INI_PATH new file mode 100644 index 0000000000..06c2180789 --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/rhv/OVIRT_INI_PATH @@ -0,0 +1,5 @@ +[ovirt] +ovirt_url=https://foo.invalid +ovirt_username=fooo +ovirt_password=fooo +ovirt_ca_file=fooo \ No newline at end of file diff --git a/awx/main/tests/data/inventory/scripts/satellite6/FOREMAN_INI_PATH b/awx/main/tests/data/inventory/scripts/satellite6/FOREMAN_INI_PATH new file mode 100644 index 0000000000..9fb7639c44 --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/satellite6/FOREMAN_INI_PATH @@ -0,0 +1,17 @@ +[foreman] +base_source_var = value_of_var +ssl_verify = False +url = https://foo.invalid +user = fooo +password = fooo + +[ansible] +group_patterns = foo_group_patterns +want_facts = True +want_hostcollections = True +group_prefix = foo_group_prefix + +[cache] +path = /tmp +max_age = 0 + diff --git a/awx/main/tests/data/inventory/scripts/vmware/VMWARE_INI_PATH b/awx/main/tests/data/inventory/scripts/vmware/VMWARE_INI_PATH new file mode 100644 index 0000000000..8a4d8a7700 --- /dev/null +++ b/awx/main/tests/data/inventory/scripts/vmware/VMWARE_INI_PATH @@ -0,0 +1,10 @@ +[vmware] +cache_max_age = 0 +validate_certs = False +username = fooo +password = fooo +server = https://foo.invalid +base_source_var = value_of_var +host_filters = foobaa +groupby_patterns = fouo + diff --git a/awx/main/tests/functional/test_inventory_source_injectors.py b/awx/main/tests/functional/test_inventory_source_injectors.py new file mode 100644 index 0000000000..2277f597ca --- /dev/null +++ b/awx/main/tests/functional/test_inventory_source_injectors.py @@ -0,0 +1,260 @@ +import pytest +from unittest import mock +import os +import json +import re + +from awx.main.tasks import RunInventoryUpdate +from awx.main.models import InventorySource, Credential, CredentialType, UnifiedJob +from awx.main.constants import CLOUD_PROVIDERS +from awx.main.tests import data + +DATA = os.path.join(os.path.dirname(data.__file__), 'inventory') + +TEST_SOURCE_FIELDS = { + 'vmware': { + 'instance_filters': 'foobaa', + 'group_by': 'fouo' + }, + 'ec2': { + 'instance_filters': 'foobaa', + 'group_by': 'fouo', + 'source_regions': 'us-east-2,ap-south-1' + }, + 'gce': { + 'source_regions': 'us-east4-a,us-west1-b' # surfaced as env var + }, + 'azure_rm': { + 'source_regions': 'southcentralus,westus' + }, +} + +INI_TEST_VARS = { + 'ec2': {}, + 'gce': {}, + 'openstack': { + 'private': False, + 'use_hostnames': False, + 'expand_hostvars': True, + 'fail_on_errors': True + }, + 'rhv': {}, # there are none + 'tower': {}, # there are none + 'vmware': { + # setting VMWARE_VALIDATE_CERTS is duplicated with env var + }, + 'azure_rm': {}, # there are none + 'satellite6': { + 'satellite6_group_patterns': 'foo_group_patterns', + 'satellite6_group_prefix': 'foo_group_prefix', + 'satellite6_want_hostcollections': True + }, + 'cloudforms': { + 'version': '2.4', + 'purge_actions': 'maybe', + 'clean_group_keys': 'this_key', + 'nest_tags': 'yes', + 'suffix': '.ppt', + 'prefer_ipv4': 'yes' + } +} + + +def generate_fake_var(element): + """Given a credential type field element, makes up something acceptable. + """ + if element['type'] == 'string': + if element.get('format', None) == 'ssh_private_key': + # this example came from the internet + return '\n'.join([ + '-----BEGIN ENCRYPTED PRIVATE KEY-----' + 'MIIBpjBABgkqhkiG9w0BBQ0wMzAbBgkqhkiG9w0BBQwwDgQI5yNCu9T5SnsCAggA' + 'MBQGCCqGSIb3DQMHBAhJISTgOAxtYwSCAWDXK/a1lxHIbRZHud1tfRMR4ROqkmr4' + 'kVGAnfqTyGptZUt3ZtBgrYlFAaZ1z0wxnhmhn3KIbqebI4w0cIL/3tmQ6eBD1Ad1' + 'nSEjUxZCuzTkimXQ88wZLzIS9KHc8GhINiUu5rKWbyvWA13Ykc0w65Ot5MSw3cQc' + 'w1LEDJjTculyDcRQgiRfKH5376qTzukileeTrNebNq+wbhY1kEPAHojercB7d10E' + '+QcbjJX1Tb1Zangom1qH9t/pepmV0Hn4EMzDs6DS2SWTffTddTY4dQzvksmLkP+J' + 'i8hkFIZwUkWpT9/k7MeklgtTiy0lR/Jj9CxAIQVxP8alLWbIqwCNRApleSmqtitt' + 'Z+NdsuNeTm3iUaPGYSw237tjLyVE6pr0EJqLv7VUClvJvBnH2qhQEtWYB9gvE1dS' + 'BioGu40pXVfjiLqhEKVVVEoHpI32oMkojhCGJs8Oow4bAxkzQFCtuWB1' + '-----END ENCRYPTED PRIVATE KEY-----' + ]) + if element['id'] == 'host': + return 'https://foo.invalid' + return 'fooo' + elif element['type'] == 'boolean': + return False + raise Exception('No generator written for {} type'.format(element.get('type', 'unknown'))) + + +def credential_kind(source): + """Given the inventory source kind, return expected credential kind + """ + return source.replace('ec2', 'aws') + + +@pytest.fixture +def fake_credential_factory(source): + ct = CredentialType.defaults[credential_kind(source)]() + ct.save() + + inputs = {} + var_specs = {} # pivoted version of inputs + for element in ct.inputs.get('fields'): + var_specs[element['id']] = element + for var in var_specs.keys(): + inputs[var] = generate_fake_var(var_specs[var]) + + return Credential.objects.create( + credential_type=ct, + inputs=inputs + ) + + +def read_content(private_data_dir, env, inventory_update): + """Read the environmental data laid down by the task system + template out private and secret data so they will be readable and predictable + return a dictionary `content` with file contents, keyed off environment variable + that references the file + """ + references = {} + for key, value in env.items(): + references[value] = key + + cache_file_regex = re.compile(r'/tmp/awx_{0}_[a-zA-Z0-9_]+/{1}_cache[a-zA-Z0-9_]+'.format( + inventory_update.id, inventory_update.source) + ) + private_key_regex = re.compile(r'-----BEGIN ENCRYPTED PRIVATE KEY-----.*-----END ENCRYPTED PRIVATE KEY-----') + + dir_contents = {} + for filename in os.listdir(private_data_dir): + abs_file_path = os.path.join(private_data_dir, filename) + try: + with open(abs_file_path, 'r') as f: + dir_contents[abs_file_path] = f.read() + # Declare a reference to inventory plugin file if it exists + if abs_file_path.endswith('.yml') and 'plugin: ' in dir_contents[abs_file_path]: + references[abs_file_path] = filename # plugin filenames are universal + except IsADirectoryError: + dir_contents[abs_file_path] = '' + print('dir contents') + print(dir_contents) + + # Declare cross-file references, also use special keywords if it is the cache + cache_referenced = False + cache_present = False + for abs_file_path, file_content in dir_contents.copy().items(): + if cache_file_regex.match(file_content): + cache_referenced = True + for target_path in dir_contents.keys(): + if target_path in file_content: + if target_path in references: + raise AssertionError( + 'File {} is referenced by env var or other file as well as file {}:\n{}\n{}'.format( + target_path, abs_file_path, json.dumps(env, indent=4), json.dumps(dir_contents, indent=4))) + else: + if cache_file_regex.match(target_path): + cache_present = True + if os.path.isdir(target_path): + keyword = 'cache_dir' + else: + keyword = 'cache_file' + references[target_path] = keyword + new_file_content = cache_file_regex.sub('{{ ' + keyword + ' }}', file_content) + else: + references[target_path] = 'file_reference' + new_file_content = file_content.replace(target_path, '{{ file_reference }}') + dir_contents[abs_file_path] = new_file_content + if cache_referenced and not cache_present: + raise AssertionError( + 'A cache file was referenced but never created, files:\n{}'.format( + json.dumps(dir_contents, indent=4))) + print('dir contents') + print(dir_contents) + + content = {} + for abs_file_path, file_content in dir_contents.items(): + if abs_file_path not in references: + raise AssertionError( + "File {} is not referenced by any other file or environment variable:\n{}\n{}".format( + abs_file_path, json.dumps(env, indent=4), json.dumps(dir_contents, indent=4))) + reference_key = references[abs_file_path] + file_content = private_key_regex.sub('{{private_key}}', file_content) + content[reference_key] = file_content + print(' contents') + print(content) + + return content + + +def create_reference_data(ref_dir, content): + if not os.path.exists(ref_dir): + os.mkdir(ref_dir) + for env_name, content in content.items(): + with open(os.path.join(ref_dir, env_name), 'w') as f: + f.write(content) + + +@pytest.mark.django_db +@pytest.mark.parametrize('this_kind', CLOUD_PROVIDERS) +@pytest.mark.parametrize('script_or_plugin', ['scripts', 'plugins']) +def test_inventory_script_structure(this_kind, script_or_plugin, inventory): + src_vars = dict(base_source_var='value_of_var') + if this_kind in INI_TEST_VARS: + src_vars.update(INI_TEST_VARS[this_kind]) + extra_kwargs = {} + if this_kind in TEST_SOURCE_FIELDS: + extra_kwargs.update(TEST_SOURCE_FIELDS[this_kind]) + inventory_source = InventorySource.objects.create( + inventory=inventory, + source=this_kind, + source_vars=src_vars, + **extra_kwargs + ) + inventory_source.credentials.add(fake_credential_factory(this_kind)) + inventory_update = inventory_source.create_unified_job() + task = RunInventoryUpdate() + + use_plugin = bool(script_or_plugin == 'plugins') + if use_plugin: + if this_kind not in InventorySource.injectors: + pytest.skip('Injector class for this source is not written yet') + elif InventorySource.injectors[this_kind].initial_version is None: + pytest.skip('Use of inventory plugin is not enabled for this source') + + def substitute_run(args, cwd, env, stdout_handle, **_kw): + """This method will replace run_pexpect + instead of running, it will read the private data directory contents + It will make assertions that the contents are correct + If MAKE_INVENTORY_REFERENCE_FILES is set, it will produce reference files + """ + private_data_dir = env['AWX_PRIVATE_DATA_DIR'] + set_files = bool(os.getenv("MAKE_INVENTORY_REFERENCE_FILES", 'false').lower()[0] not in ['f', '0']) + content = read_content(private_data_dir, env, inventory_update) + base_dir = os.path.join(DATA, script_or_plugin) + if not os.path.exists(base_dir): + os.mkdir(base_dir) + ref_dir = os.path.join(base_dir, this_kind) + if set_files: + create_reference_data(ref_dir, content) + pytest.skip('You set MAKE_INVENTORY_REFERENCE_FILES, so this created files, unset to run actual test.') + else: + expected_file_list = os.listdir(ref_dir) + assert set(expected_file_list) == set(content.keys()), ( + 'Inventory update runtime environment does not have expected files' + ) + for f_name in expected_file_list: + with open(os.path.join(ref_dir, f_name), 'r') as f: + ref_content = f.read() + assert content[f_name] == ref_content + return ('successful', 0) + + # Mock this so that it will not send events to the callback receiver + # because doing so in pytest land creates large explosions + with mock.patch('awx.main.queue.CallbackQueueDispatcher.dispatch', lambda self, obj: None): + # Force the update to use the script injector + with mock.patch('awx.main.models.inventory.PluginFileInjector.should_use_plugin', return_value=use_plugin): + # Also do not send websocket status updates + with mock.patch.object(UnifiedJob, 'websocket_emit_status', mock.Mock()): + with mock.patch('awx.main.expect.run.run_pexpect', substitute_run): + task.run(inventory_update.pk) diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index fb9ddb0c94..e442643b8c 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -2035,11 +2035,12 @@ class TestInventoryUpdateCredentials(TestJobExecution): assert 'cache' in config.sections() assert config.getint('cache', 'cache_max_age') == 0 + # Change the initial version of the inventory plugin to force use of script + with mock.patch('awx.main.models.inventory.gce.initial_version', None): + run('') - run('') - - inventory_update.source_regions = 'us-east-4' - run('us-east-4') + self.instance.source_regions = 'us-east-4' + run('us-east-4') def test_openstack_source(self, inventory_update, private_data_dir): task = tasks.RunInventoryUpdate()