diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index b7ddbecf43..9dad2c9a39 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -169,7 +169,7 @@ class AnsibleInventoryLoader(object): self.tmp_private_dir = build_proot_temp_dir() logger.debug("Using fresh temporary directory '{}' for isolation.".format(self.tmp_private_dir)) kwargs['proot_temp_dir'] = self.tmp_private_dir - kwargs['proot_show_paths'] = [functioning_dir(self.source)] + kwargs['proot_show_paths'] = [functioning_dir(self.source), settings.INVENTORY_COLLECTIONS_ROOT] logger.debug("Running from `{}` working directory.".format(cwd)) if self.venv_path != settings.ANSIBLE_VENV_PATH: diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 61fe38998b..e70514fe71 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -1612,6 +1612,11 @@ class PluginFileInjector(object): # base injector should be one of None, "managed", or "template" # this dictates which logic to borrow from playbook injectors base_injector = None + # every source should have collection, but these are set here + # so that a source without a collection will have null values + namespace = None + collection = None + collection_migration = '2.10' # In this version, content moved to collections def __init__(self, ansible_version): # This is InventoryOptions instance, could be source or inventory update @@ -1638,7 +1643,11 @@ class PluginFileInjector(object): """ if self.plugin_name is None: raise NotImplementedError('At minimum the plugin name is needed for inventory plugin use.') - return {'plugin': self.plugin_name} + if self.initial_version is None or Version(self.ansible_version) >= Version(self.collection_migration): + proper_name = f'{self.namespace}.{self.collection}.{self.plugin_name}' + else: + proper_name = self.plugin_name + return {'plugin': proper_name} def inventory_contents(self, inventory_update, private_data_dir): """Returns a string that is the content for the inventory file for the inventory plugin @@ -1693,7 +1702,10 @@ class PluginFileInjector(object): return injected_env def get_plugin_env(self, inventory_update, private_data_dir, private_data_files): - return self._get_shared_env(inventory_update, private_data_dir, private_data_files) + env = self._get_shared_env(inventory_update, private_data_dir, private_data_files) + if self.initial_version is None or Version(self.ansible_version) >= Version(self.collection_migration): + env['ANSIBLE_COLLECTIONS_PATHS'] = settings.INVENTORY_COLLECTIONS_ROOT + return env def get_script_env(self, inventory_update, private_data_dir, private_data_files): injected_env = self._get_shared_env(inventory_update, private_data_dir, private_data_files) @@ -1738,6 +1750,8 @@ class azure_rm(PluginFileInjector): initial_version = '2.8' # Driven by unsafe group names issue, hostvars, host names ini_env_reference = 'AZURE_INI_PATH' base_injector = 'managed' + namespace = 'azure' + collection = 'azcollection' def get_plugin_env(self, *args, **kwargs): ret = super(azure_rm, self).get_plugin_env(*args, **kwargs) @@ -1872,6 +1886,8 @@ class ec2(PluginFileInjector): # initial_version = '2.8' # Driven by unsafe group names issue, parent_group templating, hostvars ini_env_reference = 'EC2_INI_PATH' base_injector = 'managed' + namespace = 'ansible' + collection = 'amazon' def get_plugin_env(self, *args, **kwargs): ret = super(ec2, self).get_plugin_env(*args, **kwargs) @@ -2108,6 +2124,8 @@ class gce(PluginFileInjector): initial_version = '2.8' # Driven by unsafe group names issue, hostvars ini_env_reference = 'GCE_INI_PATH' base_injector = 'managed' + namespace = 'google' + collection = 'cloud' def get_plugin_env(self, *args, **kwargs): ret = super(gce, self).get_plugin_env(*args, **kwargs) @@ -2211,6 +2229,8 @@ class vmware(PluginFileInjector): # plugin_name = 'vmware_vm_inventory' # FIXME: implement me ini_env_reference = 'VMWARE_INI_PATH' base_injector = 'managed' + namespace = 'community' + collection = 'vmware' @property def script_name(self): @@ -2246,6 +2266,8 @@ class openstack(PluginFileInjector): plugin_name = 'openstack' # minimum version of 2.7.8 may be theoretically possible initial_version = '2.8' # Driven by consistency with other sources + namespace = 'openstack' + collection = 'cloud' @property def script_name(self): @@ -2309,12 +2331,10 @@ class openstack(PluginFileInjector): else: return 'uuid' - ret = dict( - plugin=self.plugin_name, - fail_on_errors=True, - expand_hostvars=True, - inventory_hostname=use_host_name_for_name(False), - ) + ret = super(openstack, self).inventory_as_dict(inventory_update, private_data_dir) + ret['fail_on_errors'] = True + ret['expand_hostvars'] = True + ret['inventory_hostname'] = use_host_name_for_name(False) # Note: mucking with defaults will break import integrity # For the plugin, we need to use the same defaults as the old script # or else imports will conflict. To find script defaults you have @@ -2341,6 +2361,8 @@ class rhv(PluginFileInjector): """ # plugin_name = 'FIXME' # contribute inventory plugin to Ansible base_injector = 'template' + namespace = 'ovirt' + collection = 'ovirt_collection' @property def script_name(self): @@ -2352,6 +2374,8 @@ class satellite6(PluginFileInjector): ini_env_reference = 'FOREMAN_INI_PATH' # initial_version = '2.8' # FIXME: turn on after plugin is validated # No base injector, because this does not work in playbooks. Bug?? + namespace = 'theforeman' + collection = 'foreman' @property def script_name(self): @@ -2425,6 +2449,8 @@ class cloudforms(PluginFileInjector): # plugin_name = 'FIXME' # contribute inventory plugin to Ansible ini_env_reference = 'CLOUDFORMS_INI_PATH' # Also no base_injector because this does not work in playbooks + # namespace = '' # does not have a collection + # collection = '' def build_script_private_data(self, inventory_update, private_data_dir): cp = configparser.RawConfigParser() @@ -2460,6 +2486,8 @@ class tower(PluginFileInjector): plugin_name = 'tower' base_injector = 'template' initial_version = '2.8' # Driven by "include_metadata" hostvars + namespace = 'awx' + collection = 'awx' def get_script_env(self, inventory_update, private_data_dir, private_data_files): env = super(tower, self).get_script_env(inventory_update, private_data_dir, private_data_files) @@ -2468,6 +2496,7 @@ class tower(PluginFileInjector): return env def inventory_as_dict(self, inventory_update, private_data_dir): + ret = super(tower, self).inventory_as_dict(inventory_update, private_data_dir) # Credentials injected as env vars, same as script try: # plugin can take an actual int type @@ -2475,11 +2504,9 @@ class tower(PluginFileInjector): except ValueError: # inventory_id could be a named URL identifier = iri_to_uri(inventory_update.instance_filters) - return { - 'plugin': self.plugin_name, - 'inventory_id': identifier, - 'include_metadata': True # used for license check - } + ret['inventory_id'] = identifier + ret['include_metadata'] = True # used for license check + return ret for cls in PluginFileInjector.__subclasses__(): diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 95118c5751..dd27a7d849 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -2407,7 +2407,7 @@ class RunInventoryUpdate(BaseTask): @property def proot_show_paths(self): - return [self.get_path_to('..', 'plugins', 'inventory')] + return [self.get_path_to('..', 'plugins', 'inventory'), settings.INVENTORY_COLLECTIONS_ROOT] def build_private_data(self, inventory_update, private_data_dir): """ diff --git a/awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml b/awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml index 4479e79058..8d6c1dbfa7 100644 --- a/awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml +++ b/awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml @@ -39,5 +39,5 @@ keyed_groups: prefix: '' separator: '' plain_host_names: true -plugin: azure_rm +plugin: azure.azcollection.azure_rm use_contrib_script_compatible_sanitization: true diff --git a/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml b/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml index b228770d36..f214483dfd 100644 --- a/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml +++ b/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml @@ -75,7 +75,7 @@ keyed_groups: parent_group: '{{ placement.region }}' prefix: '' separator: '' -plugin: aws_ec2 +plugin: ansible.amazon.aws_ec2 regions: - us-east-2 - ap-south-1 diff --git a/awx/main/tests/data/inventory/plugins/gce/files/gcp_compute.yml b/awx/main/tests/data/inventory/plugins/gce/files/gcp_compute.yml index 9f1ea11c36..63f8a44f64 100644 --- a/awx/main/tests/data/inventory/plugins/gce/files/gcp_compute.yml +++ b/awx/main/tests/data/inventory/plugins/gce/files/gcp_compute.yml @@ -40,7 +40,7 @@ keyed_groups: - key: image prefix: '' separator: '' -plugin: gcp_compute +plugin: google.cloud.gcp_compute projects: - fooo retrieve_image_info: true diff --git a/awx/main/tests/data/inventory/plugins/openstack/files/openstack.yml b/awx/main/tests/data/inventory/plugins/openstack/files/openstack.yml index c2b6ee58f3..36e9024b54 100644 --- a/awx/main/tests/data/inventory/plugins/openstack/files/openstack.yml +++ b/awx/main/tests/data/inventory/plugins/openstack/files/openstack.yml @@ -1,4 +1,4 @@ expand_hostvars: true fail_on_errors: true inventory_hostname: uuid -plugin: openstack +plugin: openstack.cloud.openstack diff --git a/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml b/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml index 7528dbc9e9..4b950202b6 100644 --- a/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml +++ b/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml @@ -1 +1 @@ -plugin: foreman +plugin: theforeman.foreman.foreman diff --git a/awx/main/tests/data/inventory/plugins/tower/files/tower.yml b/awx/main/tests/data/inventory/plugins/tower/files/tower.yml index d8d0efdc9a..2c41f1b55d 100644 --- a/awx/main/tests/data/inventory/plugins/tower/files/tower.yml +++ b/awx/main/tests/data/inventory/plugins/tower/files/tower.yml @@ -1,3 +1,3 @@ include_metadata: true inventory_id: 42 -plugin: tower +plugin: awx.awx.tower diff --git a/awx/main/tests/functional/test_inventory_source_injectors.py b/awx/main/tests/functional/test_inventory_source_injectors.py index 656c9a1511..277ebd3c3a 100644 --- a/awx/main/tests/functional/test_inventory_source_injectors.py +++ b/awx/main/tests/functional/test_inventory_source_injectors.py @@ -315,9 +315,10 @@ def test_inventory_update_injected_content(this_kind, script_or_plugin, inventor with mock.patch('awx.main.models.inventory.PluginFileInjector.should_use_plugin', return_value=use_plugin): # Also do not send websocket status updates with mock.patch.object(UnifiedJob, 'websocket_emit_status', mock.Mock()): - # The point of this test is that we replace run with assertions - with mock.patch('awx.main.tasks.ansible_runner.interface.run', substitute_run): - # mocking the licenser is necessary for the tower source - with mock.patch('awx.main.models.inventory.get_licenser', mock_licenser): - # so this sets up everything for a run and then yields control over to substitute_run - task.run(inventory_update.pk) + with mock.patch.object(task, 'get_ansible_version', return_value='2.13'): + # The point of this test is that we replace run with assertions + with mock.patch('awx.main.tasks.ansible_runner.interface.run', substitute_run): + # mocking the licenser is necessary for the tower source + with mock.patch('awx.main.models.inventory.get_licenser', mock_licenser): + # so this sets up everything for a run and then yields control over to substitute_run + task.run(inventory_update.pk) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index b2e3c23bf0..577ced5050 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -120,6 +120,10 @@ LOGIN_URL = '/api/login/' # This directory should not be web-accessible. PROJECTS_ROOT = os.path.join(BASE_DIR, 'projects') +# Absolute filesystem path to the directory to host collections for +# running inventory imports +INVENTORY_COLLECTIONS_ROOT = os.path.join(BASE_DIR, 'vendor', 'inventory_collections') + # Absolute filesystem path to the directory for job status stdout (default for # development and tests, default for production defined in production.py). This # directory should not be web-accessible diff --git a/awx/settings/local_settings.py.docker_compose b/awx/settings/local_settings.py.docker_compose index 776b17a5de..301b7a9dfe 100644 --- a/awx/settings/local_settings.py.docker_compose +++ b/awx/settings/local_settings.py.docker_compose @@ -52,6 +52,9 @@ if "pytest" in sys.modules: # This directory should NOT be web-accessible. PROJECTS_ROOT = '/var/lib/awx/projects/' +# Location for cross-development of inventory plugins +# INVENTORY_COLLECTIONS_ROOT = '/awx_devel/awx/plugins/collections' + # Absolute filesystem path to the directory for job status stdout # This directory should not be web-accessible JOBOUTPUT_ROOT = os.path.join(BASE_DIR, 'job_status') diff --git a/installer/roles/image_build/files/settings.py b/installer/roles/image_build/files/settings.py index 1a178302fb..5bdb3a549a 100644 --- a/installer/roles/image_build/files/settings.py +++ b/installer/roles/image_build/files/settings.py @@ -14,6 +14,8 @@ STATIC_ROOT = '/var/lib/awx/public/static' PROJECTS_ROOT = '/var/lib/awx/projects' +INVENTORY_COLLECTIONS_ROOT = '/var/lib/awx/vendor/inventory_collections' + JOBOUTPUT_ROOT = '/var/lib/awx/job_status' SECRET_KEY = get_secret() diff --git a/installer/roles/kubernetes/templates/configmap.yml.j2 b/installer/roles/kubernetes/templates/configmap.yml.j2 index 30b0e3397b..65ff2e0ef9 100644 --- a/installer/roles/kubernetes/templates/configmap.yml.j2 +++ b/installer/roles/kubernetes/templates/configmap.yml.j2 @@ -153,6 +153,7 @@ data: STATIC_ROOT = '/var/lib/awx/public/static' PROJECTS_ROOT = '/var/lib/awx/projects' + INVENTORY_COLLECTIONS_ROOT = '/var/lib/awx/vendor/inventory_collections' JOBOUTPUT_ROOT = '/var/lib/awx/job_status' SECRET_KEY = open('/etc/tower/SECRET_KEY', 'rb').read().strip() ALLOWED_HOSTS = ['*'] diff --git a/requirements/collections_requirements.yml b/requirements/collections_requirements.yml index 6e4f78fd26..e6f61311d5 100644 --- a/requirements/collections_requirements.yml +++ b/requirements/collections_requirements.yml @@ -1,5 +1,13 @@ --- collections: - name: awx.awx - version: 9.2.0 + version: 9.3.0 source: https://galaxy.ansible.com + - name: azure.azcollection + version: 0.1.1 # https://github.com/ansible-collections/azure/issues/55 + # - name: ansible.amazon # needs to be published + - name: theforeman.foreman # needs inventory plugin published + - name: google.cloud # https://github.com/ansible-collections/ansible_collections_google/pull/167 + # - name: openstack.cloud # needs to be published + # - name: community.vmware # not published + - name: ovirt.ovirt_collection # new fix published