From fcf75af6a7b1a1d58164e0609e397f0c6ff7d185 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Mon, 9 Mar 2020 13:07:41 -0400 Subject: [PATCH] Get current cloud sources working from collection update test data files Adopt official vendor location openstack not published yet Add collections to show paths Add collections loc to installer settings Add vendored collections to show path again --- .../management/commands/inventory_import.py | 2 +- awx/main/models/inventory.py | 53 ++++++++++++++----- awx/main/tasks.py | 2 +- .../plugins/azure_rm/files/azure_rm.yml | 2 +- .../inventory/plugins/ec2/files/aws_ec2.yml | 2 +- .../plugins/gce/files/gcp_compute.yml | 2 +- .../plugins/openstack/files/openstack.yml | 2 +- .../plugins/satellite6/files/foreman.yml | 2 +- .../inventory/plugins/tower/files/tower.yml | 2 +- .../test_inventory_source_injectors.py | 13 ++--- awx/settings/defaults.py | 4 ++ awx/settings/local_settings.py.docker_compose | 3 ++ installer/roles/image_build/files/settings.py | 2 + .../kubernetes/templates/configmap.yml.j2 | 1 + requirements/collections_requirements.yml | 10 +++- 15 files changed, 74 insertions(+), 28 deletions(-) diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index b7ddbecf43..9dad2c9a39 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -169,7 +169,7 @@ class AnsibleInventoryLoader(object): self.tmp_private_dir = build_proot_temp_dir() logger.debug("Using fresh temporary directory '{}' for isolation.".format(self.tmp_private_dir)) kwargs['proot_temp_dir'] = self.tmp_private_dir - kwargs['proot_show_paths'] = [functioning_dir(self.source)] + kwargs['proot_show_paths'] = [functioning_dir(self.source), settings.INVENTORY_COLLECTIONS_ROOT] logger.debug("Running from `{}` working directory.".format(cwd)) if self.venv_path != settings.ANSIBLE_VENV_PATH: diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 61fe38998b..e70514fe71 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -1612,6 +1612,11 @@ class PluginFileInjector(object): # base injector should be one of None, "managed", or "template" # this dictates which logic to borrow from playbook injectors base_injector = None + # every source should have collection, but these are set here + # so that a source without a collection will have null values + namespace = None + collection = None + collection_migration = '2.10' # In this version, content moved to collections def __init__(self, ansible_version): # This is InventoryOptions instance, could be source or inventory update @@ -1638,7 +1643,11 @@ class PluginFileInjector(object): """ if self.plugin_name is None: raise NotImplementedError('At minimum the plugin name is needed for inventory plugin use.') - return {'plugin': self.plugin_name} + if self.initial_version is None or Version(self.ansible_version) >= Version(self.collection_migration): + proper_name = f'{self.namespace}.{self.collection}.{self.plugin_name}' + else: + proper_name = self.plugin_name + return {'plugin': proper_name} def inventory_contents(self, inventory_update, private_data_dir): """Returns a string that is the content for the inventory file for the inventory plugin @@ -1693,7 +1702,10 @@ class PluginFileInjector(object): return injected_env def get_plugin_env(self, inventory_update, private_data_dir, private_data_files): - return self._get_shared_env(inventory_update, private_data_dir, private_data_files) + env = self._get_shared_env(inventory_update, private_data_dir, private_data_files) + if self.initial_version is None or Version(self.ansible_version) >= Version(self.collection_migration): + env['ANSIBLE_COLLECTIONS_PATHS'] = settings.INVENTORY_COLLECTIONS_ROOT + return env def get_script_env(self, inventory_update, private_data_dir, private_data_files): injected_env = self._get_shared_env(inventory_update, private_data_dir, private_data_files) @@ -1738,6 +1750,8 @@ class azure_rm(PluginFileInjector): initial_version = '2.8' # Driven by unsafe group names issue, hostvars, host names ini_env_reference = 'AZURE_INI_PATH' base_injector = 'managed' + namespace = 'azure' + collection = 'azcollection' def get_plugin_env(self, *args, **kwargs): ret = super(azure_rm, self).get_plugin_env(*args, **kwargs) @@ -1872,6 +1886,8 @@ class ec2(PluginFileInjector): # initial_version = '2.8' # Driven by unsafe group names issue, parent_group templating, hostvars ini_env_reference = 'EC2_INI_PATH' base_injector = 'managed' + namespace = 'ansible' + collection = 'amazon' def get_plugin_env(self, *args, **kwargs): ret = super(ec2, self).get_plugin_env(*args, **kwargs) @@ -2108,6 +2124,8 @@ class gce(PluginFileInjector): initial_version = '2.8' # Driven by unsafe group names issue, hostvars ini_env_reference = 'GCE_INI_PATH' base_injector = 'managed' + namespace = 'google' + collection = 'cloud' def get_plugin_env(self, *args, **kwargs): ret = super(gce, self).get_plugin_env(*args, **kwargs) @@ -2211,6 +2229,8 @@ class vmware(PluginFileInjector): # plugin_name = 'vmware_vm_inventory' # FIXME: implement me ini_env_reference = 'VMWARE_INI_PATH' base_injector = 'managed' + namespace = 'community' + collection = 'vmware' @property def script_name(self): @@ -2246,6 +2266,8 @@ class openstack(PluginFileInjector): plugin_name = 'openstack' # minimum version of 2.7.8 may be theoretically possible initial_version = '2.8' # Driven by consistency with other sources + namespace = 'openstack' + collection = 'cloud' @property def script_name(self): @@ -2309,12 +2331,10 @@ class openstack(PluginFileInjector): else: return 'uuid' - ret = dict( - plugin=self.plugin_name, - fail_on_errors=True, - expand_hostvars=True, - inventory_hostname=use_host_name_for_name(False), - ) + ret = super(openstack, self).inventory_as_dict(inventory_update, private_data_dir) + ret['fail_on_errors'] = True + ret['expand_hostvars'] = True + ret['inventory_hostname'] = use_host_name_for_name(False) # Note: mucking with defaults will break import integrity # For the plugin, we need to use the same defaults as the old script # or else imports will conflict. To find script defaults you have @@ -2341,6 +2361,8 @@ class rhv(PluginFileInjector): """ # plugin_name = 'FIXME' # contribute inventory plugin to Ansible base_injector = 'template' + namespace = 'ovirt' + collection = 'ovirt_collection' @property def script_name(self): @@ -2352,6 +2374,8 @@ class satellite6(PluginFileInjector): ini_env_reference = 'FOREMAN_INI_PATH' # initial_version = '2.8' # FIXME: turn on after plugin is validated # No base injector, because this does not work in playbooks. Bug?? + namespace = 'theforeman' + collection = 'foreman' @property def script_name(self): @@ -2425,6 +2449,8 @@ class cloudforms(PluginFileInjector): # plugin_name = 'FIXME' # contribute inventory plugin to Ansible ini_env_reference = 'CLOUDFORMS_INI_PATH' # Also no base_injector because this does not work in playbooks + # namespace = '' # does not have a collection + # collection = '' def build_script_private_data(self, inventory_update, private_data_dir): cp = configparser.RawConfigParser() @@ -2460,6 +2486,8 @@ class tower(PluginFileInjector): plugin_name = 'tower' base_injector = 'template' initial_version = '2.8' # Driven by "include_metadata" hostvars + namespace = 'awx' + collection = 'awx' def get_script_env(self, inventory_update, private_data_dir, private_data_files): env = super(tower, self).get_script_env(inventory_update, private_data_dir, private_data_files) @@ -2468,6 +2496,7 @@ class tower(PluginFileInjector): return env def inventory_as_dict(self, inventory_update, private_data_dir): + ret = super(tower, self).inventory_as_dict(inventory_update, private_data_dir) # Credentials injected as env vars, same as script try: # plugin can take an actual int type @@ -2475,11 +2504,9 @@ class tower(PluginFileInjector): except ValueError: # inventory_id could be a named URL identifier = iri_to_uri(inventory_update.instance_filters) - return { - 'plugin': self.plugin_name, - 'inventory_id': identifier, - 'include_metadata': True # used for license check - } + ret['inventory_id'] = identifier + ret['include_metadata'] = True # used for license check + return ret for cls in PluginFileInjector.__subclasses__(): diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 95118c5751..dd27a7d849 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -2407,7 +2407,7 @@ class RunInventoryUpdate(BaseTask): @property def proot_show_paths(self): - return [self.get_path_to('..', 'plugins', 'inventory')] + return [self.get_path_to('..', 'plugins', 'inventory'), settings.INVENTORY_COLLECTIONS_ROOT] def build_private_data(self, inventory_update, private_data_dir): """ diff --git a/awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml b/awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml index 4479e79058..8d6c1dbfa7 100644 --- a/awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml +++ b/awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml @@ -39,5 +39,5 @@ keyed_groups: prefix: '' separator: '' plain_host_names: true -plugin: azure_rm +plugin: azure.azcollection.azure_rm use_contrib_script_compatible_sanitization: true diff --git a/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml b/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml index b228770d36..f214483dfd 100644 --- a/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml +++ b/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml @@ -75,7 +75,7 @@ keyed_groups: parent_group: '{{ placement.region }}' prefix: '' separator: '' -plugin: aws_ec2 +plugin: ansible.amazon.aws_ec2 regions: - us-east-2 - ap-south-1 diff --git a/awx/main/tests/data/inventory/plugins/gce/files/gcp_compute.yml b/awx/main/tests/data/inventory/plugins/gce/files/gcp_compute.yml index 9f1ea11c36..63f8a44f64 100644 --- a/awx/main/tests/data/inventory/plugins/gce/files/gcp_compute.yml +++ b/awx/main/tests/data/inventory/plugins/gce/files/gcp_compute.yml @@ -40,7 +40,7 @@ keyed_groups: - key: image prefix: '' separator: '' -plugin: gcp_compute +plugin: google.cloud.gcp_compute projects: - fooo retrieve_image_info: true diff --git a/awx/main/tests/data/inventory/plugins/openstack/files/openstack.yml b/awx/main/tests/data/inventory/plugins/openstack/files/openstack.yml index c2b6ee58f3..36e9024b54 100644 --- a/awx/main/tests/data/inventory/plugins/openstack/files/openstack.yml +++ b/awx/main/tests/data/inventory/plugins/openstack/files/openstack.yml @@ -1,4 +1,4 @@ expand_hostvars: true fail_on_errors: true inventory_hostname: uuid -plugin: openstack +plugin: openstack.cloud.openstack diff --git a/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml b/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml index 7528dbc9e9..4b950202b6 100644 --- a/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml +++ b/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml @@ -1 +1 @@ -plugin: foreman +plugin: theforeman.foreman.foreman diff --git a/awx/main/tests/data/inventory/plugins/tower/files/tower.yml b/awx/main/tests/data/inventory/plugins/tower/files/tower.yml index d8d0efdc9a..2c41f1b55d 100644 --- a/awx/main/tests/data/inventory/plugins/tower/files/tower.yml +++ b/awx/main/tests/data/inventory/plugins/tower/files/tower.yml @@ -1,3 +1,3 @@ include_metadata: true inventory_id: 42 -plugin: tower +plugin: awx.awx.tower diff --git a/awx/main/tests/functional/test_inventory_source_injectors.py b/awx/main/tests/functional/test_inventory_source_injectors.py index 656c9a1511..277ebd3c3a 100644 --- a/awx/main/tests/functional/test_inventory_source_injectors.py +++ b/awx/main/tests/functional/test_inventory_source_injectors.py @@ -315,9 +315,10 @@ def test_inventory_update_injected_content(this_kind, script_or_plugin, inventor with mock.patch('awx.main.models.inventory.PluginFileInjector.should_use_plugin', return_value=use_plugin): # Also do not send websocket status updates with mock.patch.object(UnifiedJob, 'websocket_emit_status', mock.Mock()): - # The point of this test is that we replace run with assertions - with mock.patch('awx.main.tasks.ansible_runner.interface.run', substitute_run): - # mocking the licenser is necessary for the tower source - with mock.patch('awx.main.models.inventory.get_licenser', mock_licenser): - # so this sets up everything for a run and then yields control over to substitute_run - task.run(inventory_update.pk) + with mock.patch.object(task, 'get_ansible_version', return_value='2.13'): + # The point of this test is that we replace run with assertions + with mock.patch('awx.main.tasks.ansible_runner.interface.run', substitute_run): + # mocking the licenser is necessary for the tower source + with mock.patch('awx.main.models.inventory.get_licenser', mock_licenser): + # so this sets up everything for a run and then yields control over to substitute_run + task.run(inventory_update.pk) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index b2e3c23bf0..577ced5050 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -120,6 +120,10 @@ LOGIN_URL = '/api/login/' # This directory should not be web-accessible. PROJECTS_ROOT = os.path.join(BASE_DIR, 'projects') +# Absolute filesystem path to the directory to host collections for +# running inventory imports +INVENTORY_COLLECTIONS_ROOT = os.path.join(BASE_DIR, 'vendor', 'inventory_collections') + # Absolute filesystem path to the directory for job status stdout (default for # development and tests, default for production defined in production.py). This # directory should not be web-accessible diff --git a/awx/settings/local_settings.py.docker_compose b/awx/settings/local_settings.py.docker_compose index 776b17a5de..301b7a9dfe 100644 --- a/awx/settings/local_settings.py.docker_compose +++ b/awx/settings/local_settings.py.docker_compose @@ -52,6 +52,9 @@ if "pytest" in sys.modules: # This directory should NOT be web-accessible. PROJECTS_ROOT = '/var/lib/awx/projects/' +# Location for cross-development of inventory plugins +# INVENTORY_COLLECTIONS_ROOT = '/awx_devel/awx/plugins/collections' + # Absolute filesystem path to the directory for job status stdout # This directory should not be web-accessible JOBOUTPUT_ROOT = os.path.join(BASE_DIR, 'job_status') diff --git a/installer/roles/image_build/files/settings.py b/installer/roles/image_build/files/settings.py index 1a178302fb..5bdb3a549a 100644 --- a/installer/roles/image_build/files/settings.py +++ b/installer/roles/image_build/files/settings.py @@ -14,6 +14,8 @@ STATIC_ROOT = '/var/lib/awx/public/static' PROJECTS_ROOT = '/var/lib/awx/projects' +INVENTORY_COLLECTIONS_ROOT = '/var/lib/awx/vendor/inventory_collections' + JOBOUTPUT_ROOT = '/var/lib/awx/job_status' SECRET_KEY = get_secret() diff --git a/installer/roles/kubernetes/templates/configmap.yml.j2 b/installer/roles/kubernetes/templates/configmap.yml.j2 index 30b0e3397b..65ff2e0ef9 100644 --- a/installer/roles/kubernetes/templates/configmap.yml.j2 +++ b/installer/roles/kubernetes/templates/configmap.yml.j2 @@ -153,6 +153,7 @@ data: STATIC_ROOT = '/var/lib/awx/public/static' PROJECTS_ROOT = '/var/lib/awx/projects' + INVENTORY_COLLECTIONS_ROOT = '/var/lib/awx/vendor/inventory_collections' JOBOUTPUT_ROOT = '/var/lib/awx/job_status' SECRET_KEY = open('/etc/tower/SECRET_KEY', 'rb').read().strip() ALLOWED_HOSTS = ['*'] diff --git a/requirements/collections_requirements.yml b/requirements/collections_requirements.yml index 6e4f78fd26..e6f61311d5 100644 --- a/requirements/collections_requirements.yml +++ b/requirements/collections_requirements.yml @@ -1,5 +1,13 @@ --- collections: - name: awx.awx - version: 9.2.0 + version: 9.3.0 source: https://galaxy.ansible.com + - name: azure.azcollection + version: 0.1.1 # https://github.com/ansible-collections/azure/issues/55 + # - name: ansible.amazon # needs to be published + - name: theforeman.foreman # needs inventory plugin published + - name: google.cloud # https://github.com/ansible-collections/ansible_collections_google/pull/167 + # - name: openstack.cloud # needs to be published + # - name: community.vmware # not published + - name: ovirt.ovirt_collection # new fix published