Get current cloud sources working from collection

update test data files

Adopt official vendor location

openstack not published yet

Add collections to show paths

Add collections loc to installer settings

Add vendored collections to show path again

awx/main/management/commands/inventory_import.py

@@ -169,7 +169,7 @@ class AnsibleInventoryLoader(object):
             self.tmp_private_dir = build_proot_temp_dir()
             logger.debug("Using fresh temporary directory '{}' for isolation.".format(self.tmp_private_dir))
             kwargs['proot_temp_dir'] = self.tmp_private_dir
-            kwargs['proot_show_paths'] = [functioning_dir(self.source)]
+            kwargs['proot_show_paths'] = [functioning_dir(self.source), settings.INVENTORY_COLLECTIONS_ROOT]
         logger.debug("Running from `{}` working directory.".format(cwd))
 
         if self.venv_path != settings.ANSIBLE_VENV_PATH:

awx/main/models/inventory.py

@@ -1612,6 +1612,11 @@ class PluginFileInjector(object):
     # base injector should be one of None, "managed", or "template"
     # this dictates which logic to borrow from playbook injectors
     base_injector = None
+    # every source should have collection, but these are set here
+    # so that a source without a collection will have null values
+    namespace = None
+    collection = None
+    collection_migration = '2.10'  # In this version, content moved to collections
 
     def __init__(self, ansible_version):
         # This is InventoryOptions instance, could be source or inventory update
@@ -1638,7 +1643,11 @@ class PluginFileInjector(object):
         """
         if self.plugin_name is None:
             raise NotImplementedError('At minimum the plugin name is needed for inventory plugin use.')
-        return {'plugin': self.plugin_name}
+        if self.initial_version is None or Version(self.ansible_version) >= Version(self.collection_migration):
+            proper_name = f'{self.namespace}.{self.collection}.{self.plugin_name}'
+        else:
+            proper_name = self.plugin_name
+        return {'plugin': proper_name}
 
     def inventory_contents(self, inventory_update, private_data_dir):
         """Returns a string that is the content for the inventory file for the inventory plugin
@@ -1693,7 +1702,10 @@ class PluginFileInjector(object):
         return injected_env
 
     def get_plugin_env(self, inventory_update, private_data_dir, private_data_files):
-        return self._get_shared_env(inventory_update, private_data_dir, private_data_files)
+        env = self._get_shared_env(inventory_update, private_data_dir, private_data_files)
+        if self.initial_version is None or Version(self.ansible_version) >= Version(self.collection_migration):
+            env['ANSIBLE_COLLECTIONS_PATHS'] = settings.INVENTORY_COLLECTIONS_ROOT
+        return env
 
     def get_script_env(self, inventory_update, private_data_dir, private_data_files):
         injected_env = self._get_shared_env(inventory_update, private_data_dir, private_data_files)
@@ -1738,6 +1750,8 @@ class azure_rm(PluginFileInjector):
     initial_version = '2.8'  # Driven by unsafe group names issue, hostvars, host names
     ini_env_reference = 'AZURE_INI_PATH'
     base_injector = 'managed'
+    namespace = 'azure'
+    collection = 'azcollection'
 
     def get_plugin_env(self, *args, **kwargs):
         ret = super(azure_rm, self).get_plugin_env(*args, **kwargs)
@@ -1872,6 +1886,8 @@ class ec2(PluginFileInjector):
     # initial_version = '2.8'  # Driven by unsafe group names issue, parent_group templating, hostvars
     ini_env_reference = 'EC2_INI_PATH'
     base_injector = 'managed'
+    namespace = 'ansible'
+    collection = 'amazon'
 
     def get_plugin_env(self, *args, **kwargs):
         ret = super(ec2, self).get_plugin_env(*args, **kwargs)
@@ -2108,6 +2124,8 @@ class gce(PluginFileInjector):
     initial_version = '2.8'  # Driven by unsafe group names issue, hostvars
     ini_env_reference = 'GCE_INI_PATH'
     base_injector = 'managed'
+    namespace = 'google'
+    collection = 'cloud'
 
     def get_plugin_env(self, *args, **kwargs):
         ret = super(gce, self).get_plugin_env(*args, **kwargs)
@@ -2211,6 +2229,8 @@ class vmware(PluginFileInjector):
     # plugin_name = 'vmware_vm_inventory'  # FIXME: implement me
     ini_env_reference = 'VMWARE_INI_PATH'
     base_injector = 'managed'
+    namespace = 'community'
+    collection = 'vmware'
 
     @property
     def script_name(self):
@@ -2246,6 +2266,8 @@ class openstack(PluginFileInjector):
     plugin_name = 'openstack'
     # minimum version of 2.7.8 may be theoretically possible
     initial_version = '2.8'  # Driven by consistency with other sources
+    namespace = 'openstack'
+    collection = 'cloud'
 
     @property
     def script_name(self):
@@ -2309,12 +2331,10 @@ class openstack(PluginFileInjector):
             else:
                 return 'uuid'
 
-        ret = dict(
+        ret = super(openstack, self).inventory_as_dict(inventory_update, private_data_dir)
-            plugin=self.plugin_name,
+        ret['fail_on_errors'] = True
-            fail_on_errors=True,
+        ret['expand_hostvars'] = True
-            expand_hostvars=True,
+        ret['inventory_hostname'] = use_host_name_for_name(False)
-            inventory_hostname=use_host_name_for_name(False),
-        )
         # Note: mucking with defaults will break import integrity
         # For the plugin, we need to use the same defaults as the old script
         # or else imports will conflict. To find script defaults you have
@@ -2341,6 +2361,8 @@ class rhv(PluginFileInjector):
     """
     # plugin_name = 'FIXME'  # contribute inventory plugin to Ansible
     base_injector = 'template'
+    namespace = 'ovirt'
+    collection = 'ovirt_collection'
 
     @property
     def script_name(self):
@@ -2352,6 +2374,8 @@ class satellite6(PluginFileInjector):
     ini_env_reference = 'FOREMAN_INI_PATH'
     # initial_version = '2.8'  # FIXME: turn on after plugin is validated
     # No base injector, because this does not work in playbooks. Bug??
+    namespace = 'theforeman'
+    collection = 'foreman'
 
     @property
     def script_name(self):
@@ -2425,6 +2449,8 @@ class cloudforms(PluginFileInjector):
     # plugin_name = 'FIXME'  # contribute inventory plugin to Ansible
     ini_env_reference = 'CLOUDFORMS_INI_PATH'
     # Also no base_injector because this does not work in playbooks
+    # namespace = ''  # does not have a collection
+    # collection = ''
 
     def build_script_private_data(self, inventory_update, private_data_dir):
         cp = configparser.RawConfigParser()
@@ -2460,6 +2486,8 @@ class tower(PluginFileInjector):
     plugin_name = 'tower'
     base_injector = 'template'
     initial_version = '2.8'  # Driven by "include_metadata" hostvars
+    namespace = 'awx'
+    collection = 'awx'
 
     def get_script_env(self, inventory_update, private_data_dir, private_data_files):
         env = super(tower, self).get_script_env(inventory_update, private_data_dir, private_data_files)
@@ -2468,6 +2496,7 @@ class tower(PluginFileInjector):
         return env
 
     def inventory_as_dict(self, inventory_update, private_data_dir):
+        ret = super(tower, self).inventory_as_dict(inventory_update, private_data_dir)
         # Credentials injected as env vars, same as script
         try:
             # plugin can take an actual int type
@@ -2475,11 +2504,9 @@ class tower(PluginFileInjector):
         except ValueError:
             # inventory_id could be a named URL
             identifier = iri_to_uri(inventory_update.instance_filters)
-        return {
+        ret['inventory_id'] = identifier
-            'plugin': self.plugin_name,
+        ret['include_metadata'] = True  # used for license check
-            'inventory_id': identifier,
+        return ret
-            'include_metadata': True  # used for license check
-        }
 
 
 for cls in PluginFileInjector.__subclasses__():

awx/main/tasks.py

@@ -2407,7 +2407,7 @@ class RunInventoryUpdate(BaseTask):
 
     @property
     def proot_show_paths(self):
-        return [self.get_path_to('..', 'plugins', 'inventory')]
+        return [self.get_path_to('..', 'plugins', 'inventory'), settings.INVENTORY_COLLECTIONS_ROOT]
 
     def build_private_data(self, inventory_update, private_data_dir):
         """

awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml

@@ -39,5 +39,5 @@ keyed_groups:
   prefix: ''
   separator: ''
 plain_host_names: true
-plugin: azure_rm
+plugin: azure.azcollection.azure_rm
 use_contrib_script_compatible_sanitization: true

awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml

@@ -75,7 +75,7 @@ keyed_groups:
   parent_group: '{{ placement.region }}'
   prefix: ''
   separator: ''
-plugin: aws_ec2
+plugin: ansible.amazon.aws_ec2
 regions:
 - us-east-2
 - ap-south-1

awx/main/tests/data/inventory/plugins/gce/files/gcp_compute.yml

@@ -40,7 +40,7 @@ keyed_groups:
 - key: image
   prefix: ''
   separator: ''
-plugin: gcp_compute
+plugin: google.cloud.gcp_compute
 projects:
 - fooo
 retrieve_image_info: true

awx/main/tests/data/inventory/plugins/openstack/files/openstack.yml

@@ -1,4 +1,4 @@
 expand_hostvars: true
 fail_on_errors: true
 inventory_hostname: uuid
-plugin: openstack
+plugin: openstack.cloud.openstack

awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml

@@ -1 +1 @@
-plugin: foreman
+plugin: theforeman.foreman.foreman

awx/main/tests/data/inventory/plugins/tower/files/tower.yml

@@ -1,3 +1,3 @@
 include_metadata: true
 inventory_id: 42
-plugin: tower
+plugin: awx.awx.tower

awx/main/tests/functional/test_inventory_source_injectors.py

@@ -315,9 +315,10 @@ def test_inventory_update_injected_content(this_kind, script_or_plugin, inventor
         with mock.patch('awx.main.models.inventory.PluginFileInjector.should_use_plugin', return_value=use_plugin):
             # Also do not send websocket status updates
             with mock.patch.object(UnifiedJob, 'websocket_emit_status', mock.Mock()):
-                # The point of this test is that we replace run with assertions
+                with mock.patch.object(task, 'get_ansible_version', return_value='2.13'):
-                with mock.patch('awx.main.tasks.ansible_runner.interface.run', substitute_run):
+                    # The point of this test is that we replace run with assertions
-                    # mocking the licenser is necessary for the tower source
+                    with mock.patch('awx.main.tasks.ansible_runner.interface.run', substitute_run):
-                    with mock.patch('awx.main.models.inventory.get_licenser', mock_licenser):
+                        # mocking the licenser is necessary for the tower source
-                        # so this sets up everything for a run and then yields control over to substitute_run
+                        with mock.patch('awx.main.models.inventory.get_licenser', mock_licenser):
-                        task.run(inventory_update.pk)
+                            # so this sets up everything for a run and then yields control over to substitute_run
+                            task.run(inventory_update.pk)

awx/settings/defaults.py

@@ -120,6 +120,10 @@ LOGIN_URL = '/api/login/'
 # This directory should not be web-accessible.
 PROJECTS_ROOT = os.path.join(BASE_DIR, 'projects')
 
+# Absolute filesystem path to the directory to host collections for
+# running inventory imports
+INVENTORY_COLLECTIONS_ROOT = os.path.join(BASE_DIR, 'vendor', 'inventory_collections')
+
 # Absolute filesystem path to the directory for job status stdout (default for
 # development and tests, default for production defined in production.py). This
 # directory should not be web-accessible

awx/settings/local_settings.py.docker_compose

@@ -52,6 +52,9 @@ if "pytest" in sys.modules:
 # This directory should NOT be web-accessible.
 PROJECTS_ROOT = '/var/lib/awx/projects/'
 
+# Location for cross-development of inventory plugins
+# INVENTORY_COLLECTIONS_ROOT = '/awx_devel/awx/plugins/collections'
+
 # Absolute filesystem path to the directory for job status stdout
 # This directory should not be web-accessible
 JOBOUTPUT_ROOT = os.path.join(BASE_DIR, 'job_status')

installer/roles/image_build/files/settings.py

@@ -14,6 +14,8 @@ STATIC_ROOT = '/var/lib/awx/public/static'
 
 PROJECTS_ROOT = '/var/lib/awx/projects'
 
+INVENTORY_COLLECTIONS_ROOT = '/var/lib/awx/vendor/inventory_collections'
+
 JOBOUTPUT_ROOT = '/var/lib/awx/job_status'
 
 SECRET_KEY = get_secret()

installer/roles/kubernetes/templates/configmap.yml.j2

@@ -153,6 +153,7 @@ data:
 
     STATIC_ROOT = '/var/lib/awx/public/static'
     PROJECTS_ROOT = '/var/lib/awx/projects'
+    INVENTORY_COLLECTIONS_ROOT = '/var/lib/awx/vendor/inventory_collections'
     JOBOUTPUT_ROOT = '/var/lib/awx/job_status'
     SECRET_KEY = open('/etc/tower/SECRET_KEY', 'rb').read().strip()
     ALLOWED_HOSTS = ['*']

requirements/collections_requirements.yml

@@ -1,5 +1,13 @@
 ---
 collections:
   - name: awx.awx
-    version: 9.2.0
+    version: 9.3.0
     source: https://galaxy.ansible.com
+  - name: azure.azcollection
+    version: 0.1.1  # https://github.com/ansible-collections/azure/issues/55
+  # - name: ansible.amazon  # needs to be published
+  - name: theforeman.foreman  # needs inventory plugin published
+  - name: google.cloud  # https://github.com/ansible-collections/ansible_collections_google/pull/167
+  # - name: openstack.cloud  # needs to be published
+  # - name: community.vmware  # not published
+  - name: ovirt.ovirt_collection  # new fix published