From 541b9607f52a5fd2e071019416386b2917d5192e Mon Sep 17 00:00:00 2001 From: Yanis Guenane Date: Wed, 11 Mar 2020 15:08:57 +0100 Subject: [PATCH 01/19] Collections: Adding a requirements.yml file --- MANIFEST.in | 1 + Makefile | 6 +++++- installer/roles/image_build/templates/Dockerfile.j2 | 2 ++ requirements/collections_requirements.yml | 5 +++++ tools/docker-compose/Dockerfile | 3 +++ 5 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 requirements/collections_requirements.yml diff --git a/MANIFEST.in b/MANIFEST.in index 3c687ce2da..53e1d8eebd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -10,6 +10,7 @@ recursive-include awx/playbooks *.yml recursive-include awx/lib/site-packages * recursive-include awx/plugins *.ps1 recursive-include requirements *.txt +recursive-include requirements *.yml recursive-include config * recursive-include docs/licenses * recursive-exclude awx devonly.py* diff --git a/Makefile b/Makefile index 80e695693b..7167d7d9b7 100644 --- a/Makefile +++ b/Makefile @@ -209,7 +209,11 @@ requirements_awx: virtualenv_awx requirements_awx_dev: $(VENV_BASE)/awx/bin/pip install -r requirements/requirements_dev.txt -requirements: requirements_ansible requirements_awx +requirements_collections: + mkdir -p $(COLLECTION_BASE) + ansible-galaxy collection install -r requirements/collections_requirements.yml -p $(COLLECTION_BASE) + +requirements: requirements_ansible requirements_awx requirements_collections requirements_dev: requirements_awx requirements_ansible_py3 requirements_awx_dev requirements_ansible_dev diff --git a/installer/roles/image_build/templates/Dockerfile.j2 b/installer/roles/image_build/templates/Dockerfile.j2 index 3ec1d91f55..b9b5d5e4fd 100644 --- a/installer/roles/image_build/templates/Dockerfile.j2 +++ b/installer/roles/image_build/templates/Dockerfile.j2 @@ -74,8 +74,10 @@ ADD requirements/requirements_ansible.txt \ requirements/requirements.txt \ requirements/requirements_tower_uninstall.txt \ requirements/requirements_git.txt \ + requirements/collections_requirements.yml \ /tmp/requirements/ RUN cd /tmp && VENV_BASE="/var/lib/awx/venv" make requirements_awx requirements_ansible_py3 +RUN cd /tmp && COLLECTION_BASE="/var/lib/awx/vendor/inventory_collections" make requirements_collections COPY {{ awx_sdist_file }} /tmp/{{ awx_sdist_file }} RUN echo "{{ awx_version }}" > /var/lib/awx/.tower_version && \ diff --git a/requirements/collections_requirements.yml b/requirements/collections_requirements.yml new file mode 100644 index 0000000000..6e4f78fd26 --- /dev/null +++ b/requirements/collections_requirements.yml @@ -0,0 +1,5 @@ +--- +collections: + - name: awx.awx + version: 9.2.0 + source: https://galaxy.ansible.com diff --git a/tools/docker-compose/Dockerfile b/tools/docker-compose/Dockerfile index 4611247231..1f8b290e64 100644 --- a/tools/docker-compose/Dockerfile +++ b/tools/docker-compose/Dockerfile @@ -92,9 +92,12 @@ ADD requirements/requirements.txt \ requirements/requirements_dev.txt \ requirements/requirements_ansible_uninstall.txt \ requirements/requirements_tower_uninstall.txt \ + requirements/collections_requirements.yml \ /tmp/requirements/ RUN mkdir -p /venv && chmod g+w /venv RUN cd /tmp && VENV_BASE="/venv" make requirements_dev +RUN mkdir -p /vendor/inventory_collections && chmod g+w /vendor/inventory_collections +RUN cd /tmp && COLLECTION_BASE="/vendor/inventory_collections" make requirements_collections # Use the distro provided npm to bootstrap our required version of node RUN npm install -g n && n 10.15.0 && dnf remove -y nodejs From fcf75af6a7b1a1d58164e0609e397f0c6ff7d185 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Mon, 9 Mar 2020 13:07:41 -0400 Subject: [PATCH 02/19] Get current cloud sources working from collection update test data files Adopt official vendor location openstack not published yet Add collections to show paths Add collections loc to installer settings Add vendored collections to show path again --- .../management/commands/inventory_import.py | 2 +- awx/main/models/inventory.py | 53 ++++++++++++++----- awx/main/tasks.py | 2 +- .../plugins/azure_rm/files/azure_rm.yml | 2 +- .../inventory/plugins/ec2/files/aws_ec2.yml | 2 +- .../plugins/gce/files/gcp_compute.yml | 2 +- .../plugins/openstack/files/openstack.yml | 2 +- .../plugins/satellite6/files/foreman.yml | 2 +- .../inventory/plugins/tower/files/tower.yml | 2 +- .../test_inventory_source_injectors.py | 13 ++--- awx/settings/defaults.py | 4 ++ awx/settings/local_settings.py.docker_compose | 3 ++ installer/roles/image_build/files/settings.py | 2 + .../kubernetes/templates/configmap.yml.j2 | 1 + requirements/collections_requirements.yml | 10 +++- 15 files changed, 74 insertions(+), 28 deletions(-) diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index b7ddbecf43..9dad2c9a39 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -169,7 +169,7 @@ class AnsibleInventoryLoader(object): self.tmp_private_dir = build_proot_temp_dir() logger.debug("Using fresh temporary directory '{}' for isolation.".format(self.tmp_private_dir)) kwargs['proot_temp_dir'] = self.tmp_private_dir - kwargs['proot_show_paths'] = [functioning_dir(self.source)] + kwargs['proot_show_paths'] = [functioning_dir(self.source), settings.INVENTORY_COLLECTIONS_ROOT] logger.debug("Running from `{}` working directory.".format(cwd)) if self.venv_path != settings.ANSIBLE_VENV_PATH: diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 61fe38998b..e70514fe71 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -1612,6 +1612,11 @@ class PluginFileInjector(object): # base injector should be one of None, "managed", or "template" # this dictates which logic to borrow from playbook injectors base_injector = None + # every source should have collection, but these are set here + # so that a source without a collection will have null values + namespace = None + collection = None + collection_migration = '2.10' # In this version, content moved to collections def __init__(self, ansible_version): # This is InventoryOptions instance, could be source or inventory update @@ -1638,7 +1643,11 @@ class PluginFileInjector(object): """ if self.plugin_name is None: raise NotImplementedError('At minimum the plugin name is needed for inventory plugin use.') - return {'plugin': self.plugin_name} + if self.initial_version is None or Version(self.ansible_version) >= Version(self.collection_migration): + proper_name = f'{self.namespace}.{self.collection}.{self.plugin_name}' + else: + proper_name = self.plugin_name + return {'plugin': proper_name} def inventory_contents(self, inventory_update, private_data_dir): """Returns a string that is the content for the inventory file for the inventory plugin @@ -1693,7 +1702,10 @@ class PluginFileInjector(object): return injected_env def get_plugin_env(self, inventory_update, private_data_dir, private_data_files): - return self._get_shared_env(inventory_update, private_data_dir, private_data_files) + env = self._get_shared_env(inventory_update, private_data_dir, private_data_files) + if self.initial_version is None or Version(self.ansible_version) >= Version(self.collection_migration): + env['ANSIBLE_COLLECTIONS_PATHS'] = settings.INVENTORY_COLLECTIONS_ROOT + return env def get_script_env(self, inventory_update, private_data_dir, private_data_files): injected_env = self._get_shared_env(inventory_update, private_data_dir, private_data_files) @@ -1738,6 +1750,8 @@ class azure_rm(PluginFileInjector): initial_version = '2.8' # Driven by unsafe group names issue, hostvars, host names ini_env_reference = 'AZURE_INI_PATH' base_injector = 'managed' + namespace = 'azure' + collection = 'azcollection' def get_plugin_env(self, *args, **kwargs): ret = super(azure_rm, self).get_plugin_env(*args, **kwargs) @@ -1872,6 +1886,8 @@ class ec2(PluginFileInjector): # initial_version = '2.8' # Driven by unsafe group names issue, parent_group templating, hostvars ini_env_reference = 'EC2_INI_PATH' base_injector = 'managed' + namespace = 'ansible' + collection = 'amazon' def get_plugin_env(self, *args, **kwargs): ret = super(ec2, self).get_plugin_env(*args, **kwargs) @@ -2108,6 +2124,8 @@ class gce(PluginFileInjector): initial_version = '2.8' # Driven by unsafe group names issue, hostvars ini_env_reference = 'GCE_INI_PATH' base_injector = 'managed' + namespace = 'google' + collection = 'cloud' def get_plugin_env(self, *args, **kwargs): ret = super(gce, self).get_plugin_env(*args, **kwargs) @@ -2211,6 +2229,8 @@ class vmware(PluginFileInjector): # plugin_name = 'vmware_vm_inventory' # FIXME: implement me ini_env_reference = 'VMWARE_INI_PATH' base_injector = 'managed' + namespace = 'community' + collection = 'vmware' @property def script_name(self): @@ -2246,6 +2266,8 @@ class openstack(PluginFileInjector): plugin_name = 'openstack' # minimum version of 2.7.8 may be theoretically possible initial_version = '2.8' # Driven by consistency with other sources + namespace = 'openstack' + collection = 'cloud' @property def script_name(self): @@ -2309,12 +2331,10 @@ class openstack(PluginFileInjector): else: return 'uuid' - ret = dict( - plugin=self.plugin_name, - fail_on_errors=True, - expand_hostvars=True, - inventory_hostname=use_host_name_for_name(False), - ) + ret = super(openstack, self).inventory_as_dict(inventory_update, private_data_dir) + ret['fail_on_errors'] = True + ret['expand_hostvars'] = True + ret['inventory_hostname'] = use_host_name_for_name(False) # Note: mucking with defaults will break import integrity # For the plugin, we need to use the same defaults as the old script # or else imports will conflict. To find script defaults you have @@ -2341,6 +2361,8 @@ class rhv(PluginFileInjector): """ # plugin_name = 'FIXME' # contribute inventory plugin to Ansible base_injector = 'template' + namespace = 'ovirt' + collection = 'ovirt_collection' @property def script_name(self): @@ -2352,6 +2374,8 @@ class satellite6(PluginFileInjector): ini_env_reference = 'FOREMAN_INI_PATH' # initial_version = '2.8' # FIXME: turn on after plugin is validated # No base injector, because this does not work in playbooks. Bug?? + namespace = 'theforeman' + collection = 'foreman' @property def script_name(self): @@ -2425,6 +2449,8 @@ class cloudforms(PluginFileInjector): # plugin_name = 'FIXME' # contribute inventory plugin to Ansible ini_env_reference = 'CLOUDFORMS_INI_PATH' # Also no base_injector because this does not work in playbooks + # namespace = '' # does not have a collection + # collection = '' def build_script_private_data(self, inventory_update, private_data_dir): cp = configparser.RawConfigParser() @@ -2460,6 +2486,8 @@ class tower(PluginFileInjector): plugin_name = 'tower' base_injector = 'template' initial_version = '2.8' # Driven by "include_metadata" hostvars + namespace = 'awx' + collection = 'awx' def get_script_env(self, inventory_update, private_data_dir, private_data_files): env = super(tower, self).get_script_env(inventory_update, private_data_dir, private_data_files) @@ -2468,6 +2496,7 @@ class tower(PluginFileInjector): return env def inventory_as_dict(self, inventory_update, private_data_dir): + ret = super(tower, self).inventory_as_dict(inventory_update, private_data_dir) # Credentials injected as env vars, same as script try: # plugin can take an actual int type @@ -2475,11 +2504,9 @@ class tower(PluginFileInjector): except ValueError: # inventory_id could be a named URL identifier = iri_to_uri(inventory_update.instance_filters) - return { - 'plugin': self.plugin_name, - 'inventory_id': identifier, - 'include_metadata': True # used for license check - } + ret['inventory_id'] = identifier + ret['include_metadata'] = True # used for license check + return ret for cls in PluginFileInjector.__subclasses__(): diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 95118c5751..dd27a7d849 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -2407,7 +2407,7 @@ class RunInventoryUpdate(BaseTask): @property def proot_show_paths(self): - return [self.get_path_to('..', 'plugins', 'inventory')] + return [self.get_path_to('..', 'plugins', 'inventory'), settings.INVENTORY_COLLECTIONS_ROOT] def build_private_data(self, inventory_update, private_data_dir): """ diff --git a/awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml b/awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml index 4479e79058..8d6c1dbfa7 100644 --- a/awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml +++ b/awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml @@ -39,5 +39,5 @@ keyed_groups: prefix: '' separator: '' plain_host_names: true -plugin: azure_rm +plugin: azure.azcollection.azure_rm use_contrib_script_compatible_sanitization: true diff --git a/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml b/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml index b228770d36..f214483dfd 100644 --- a/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml +++ b/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml @@ -75,7 +75,7 @@ keyed_groups: parent_group: '{{ placement.region }}' prefix: '' separator: '' -plugin: aws_ec2 +plugin: ansible.amazon.aws_ec2 regions: - us-east-2 - ap-south-1 diff --git a/awx/main/tests/data/inventory/plugins/gce/files/gcp_compute.yml b/awx/main/tests/data/inventory/plugins/gce/files/gcp_compute.yml index 9f1ea11c36..63f8a44f64 100644 --- a/awx/main/tests/data/inventory/plugins/gce/files/gcp_compute.yml +++ b/awx/main/tests/data/inventory/plugins/gce/files/gcp_compute.yml @@ -40,7 +40,7 @@ keyed_groups: - key: image prefix: '' separator: '' -plugin: gcp_compute +plugin: google.cloud.gcp_compute projects: - fooo retrieve_image_info: true diff --git a/awx/main/tests/data/inventory/plugins/openstack/files/openstack.yml b/awx/main/tests/data/inventory/plugins/openstack/files/openstack.yml index c2b6ee58f3..36e9024b54 100644 --- a/awx/main/tests/data/inventory/plugins/openstack/files/openstack.yml +++ b/awx/main/tests/data/inventory/plugins/openstack/files/openstack.yml @@ -1,4 +1,4 @@ expand_hostvars: true fail_on_errors: true inventory_hostname: uuid -plugin: openstack +plugin: openstack.cloud.openstack diff --git a/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml b/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml index 7528dbc9e9..4b950202b6 100644 --- a/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml +++ b/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml @@ -1 +1 @@ -plugin: foreman +plugin: theforeman.foreman.foreman diff --git a/awx/main/tests/data/inventory/plugins/tower/files/tower.yml b/awx/main/tests/data/inventory/plugins/tower/files/tower.yml index d8d0efdc9a..2c41f1b55d 100644 --- a/awx/main/tests/data/inventory/plugins/tower/files/tower.yml +++ b/awx/main/tests/data/inventory/plugins/tower/files/tower.yml @@ -1,3 +1,3 @@ include_metadata: true inventory_id: 42 -plugin: tower +plugin: awx.awx.tower diff --git a/awx/main/tests/functional/test_inventory_source_injectors.py b/awx/main/tests/functional/test_inventory_source_injectors.py index 656c9a1511..277ebd3c3a 100644 --- a/awx/main/tests/functional/test_inventory_source_injectors.py +++ b/awx/main/tests/functional/test_inventory_source_injectors.py @@ -315,9 +315,10 @@ def test_inventory_update_injected_content(this_kind, script_or_plugin, inventor with mock.patch('awx.main.models.inventory.PluginFileInjector.should_use_plugin', return_value=use_plugin): # Also do not send websocket status updates with mock.patch.object(UnifiedJob, 'websocket_emit_status', mock.Mock()): - # The point of this test is that we replace run with assertions - with mock.patch('awx.main.tasks.ansible_runner.interface.run', substitute_run): - # mocking the licenser is necessary for the tower source - with mock.patch('awx.main.models.inventory.get_licenser', mock_licenser): - # so this sets up everything for a run and then yields control over to substitute_run - task.run(inventory_update.pk) + with mock.patch.object(task, 'get_ansible_version', return_value='2.13'): + # The point of this test is that we replace run with assertions + with mock.patch('awx.main.tasks.ansible_runner.interface.run', substitute_run): + # mocking the licenser is necessary for the tower source + with mock.patch('awx.main.models.inventory.get_licenser', mock_licenser): + # so this sets up everything for a run and then yields control over to substitute_run + task.run(inventory_update.pk) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index b2e3c23bf0..577ced5050 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -120,6 +120,10 @@ LOGIN_URL = '/api/login/' # This directory should not be web-accessible. PROJECTS_ROOT = os.path.join(BASE_DIR, 'projects') +# Absolute filesystem path to the directory to host collections for +# running inventory imports +INVENTORY_COLLECTIONS_ROOT = os.path.join(BASE_DIR, 'vendor', 'inventory_collections') + # Absolute filesystem path to the directory for job status stdout (default for # development and tests, default for production defined in production.py). This # directory should not be web-accessible diff --git a/awx/settings/local_settings.py.docker_compose b/awx/settings/local_settings.py.docker_compose index 776b17a5de..301b7a9dfe 100644 --- a/awx/settings/local_settings.py.docker_compose +++ b/awx/settings/local_settings.py.docker_compose @@ -52,6 +52,9 @@ if "pytest" in sys.modules: # This directory should NOT be web-accessible. PROJECTS_ROOT = '/var/lib/awx/projects/' +# Location for cross-development of inventory plugins +# INVENTORY_COLLECTIONS_ROOT = '/awx_devel/awx/plugins/collections' + # Absolute filesystem path to the directory for job status stdout # This directory should not be web-accessible JOBOUTPUT_ROOT = os.path.join(BASE_DIR, 'job_status') diff --git a/installer/roles/image_build/files/settings.py b/installer/roles/image_build/files/settings.py index 1a178302fb..5bdb3a549a 100644 --- a/installer/roles/image_build/files/settings.py +++ b/installer/roles/image_build/files/settings.py @@ -14,6 +14,8 @@ STATIC_ROOT = '/var/lib/awx/public/static' PROJECTS_ROOT = '/var/lib/awx/projects' +INVENTORY_COLLECTIONS_ROOT = '/var/lib/awx/vendor/inventory_collections' + JOBOUTPUT_ROOT = '/var/lib/awx/job_status' SECRET_KEY = get_secret() diff --git a/installer/roles/kubernetes/templates/configmap.yml.j2 b/installer/roles/kubernetes/templates/configmap.yml.j2 index 30b0e3397b..65ff2e0ef9 100644 --- a/installer/roles/kubernetes/templates/configmap.yml.j2 +++ b/installer/roles/kubernetes/templates/configmap.yml.j2 @@ -153,6 +153,7 @@ data: STATIC_ROOT = '/var/lib/awx/public/static' PROJECTS_ROOT = '/var/lib/awx/projects' + INVENTORY_COLLECTIONS_ROOT = '/var/lib/awx/vendor/inventory_collections' JOBOUTPUT_ROOT = '/var/lib/awx/job_status' SECRET_KEY = open('/etc/tower/SECRET_KEY', 'rb').read().strip() ALLOWED_HOSTS = ['*'] diff --git a/requirements/collections_requirements.yml b/requirements/collections_requirements.yml index 6e4f78fd26..e6f61311d5 100644 --- a/requirements/collections_requirements.yml +++ b/requirements/collections_requirements.yml @@ -1,5 +1,13 @@ --- collections: - name: awx.awx - version: 9.2.0 + version: 9.3.0 source: https://galaxy.ansible.com + - name: azure.azcollection + version: 0.1.1 # https://github.com/ansible-collections/azure/issues/55 + # - name: ansible.amazon # needs to be published + - name: theforeman.foreman # needs inventory plugin published + - name: google.cloud # https://github.com/ansible-collections/ansible_collections_google/pull/167 + # - name: openstack.cloud # needs to be published + # - name: community.vmware # not published + - name: ovirt.ovirt_collection # new fix published From 99ae614a63ffc6d6b6c92583ad7ddaa6e5173832 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Tue, 17 Mar 2020 12:51:29 -0400 Subject: [PATCH 03/19] Vmware was published implement vmware inventory plugin Enable the previously broken properties --- awx/main/models/inventory.py | 224 +++++++++++++++++++++- requirements/collections_requirements.yml | 2 +- 2 files changed, 224 insertions(+), 2 deletions(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index e70514fe71..dc0b8f6797 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -2226,7 +2226,8 @@ class gce(PluginFileInjector): class vmware(PluginFileInjector): - # plugin_name = 'vmware_vm_inventory' # FIXME: implement me + plugin_name = 'vmware_vm_inventory' # FIXME: implement me + initial_version = '2.10' # Only tested with collection version, TODO: test 2.9 as candidate ini_env_reference = 'VMWARE_INI_PATH' base_injector = 'managed' namespace = 'community' @@ -2236,6 +2237,227 @@ class vmware(PluginFileInjector): def script_name(self): return 'vmware_inventory.py' # exception + + def inventory_as_dict(self, inventory_update, private_data_dir): + ret = super(vmware, self).inventory_as_dict(inventory_update, private_data_dir) + ret['properties'] = [ + "name", + "config.cpuHotAddEnabled", + "config.cpuHotRemoveEnabled", + "config.instanceUuid", + "config.hardware.numCPU", + "config.template", + "config.name", + "guest.hostName", + "guest.ipAddress", + "guest.guestId", + "guest.guestState", + "runtime.maxMemoryUsage", + "customValue", + # not in the defaults + "vpmcsupported", + "vpmcenabled", + "vmxconfigchecksum", + "vmstorageobjectid", + "vmnpivwwnupdatesupported", + "vmnpivwwnsupported", + "vmnpivwwndisablesupported", + "vmfsnativesnapshotsupported", + "vm", + "virtualmmuusagesupported", + "virtualmmuusageignored", + "virtualexecusageignored", + "vflashcachereservation", + "vflashcacheallocation", + "version", + "vassertsenabled", + "vappconfig", + "value", + "uuid", + 'triggeredalarmstate', + 'toolsversionstatus2', + 'toolsversionstatus', + 'toolsversion', + 'toolsupdatestatus', + 'toolssynctimesupported', + 'toolsstatus', + 'toolsrunningstatus', + 'toolsrebootpredictsupported', + 'toolsinstalltype', + 'toolsinstallermounted', + 'toolsautoupdatesupported', + 'tools', + 'timestamp', + 'template', + 'tag', + 'swapstorageobjectid', + 'swapplacementsupported', + 'swapplacement', + 'suspendtime', + 'suspendinterval', + 'summary', # vim.vm.Summary + 'storage', # vim.vm.StorageInfo + 'snapshotoperationssupported', + 'snapshotinbackground', + 'snapshotconfigsupported', + 'snapshot', + 'settingvideoramsizesupported', + 'settingscreenresolutionsupported', + 'settingdisplaytopologysupported', + 'settingdisplaytopologymodessupported', + 'sesparsedisksupported', + 'securebootsupported', + 'screen', + 'scheduledhardwareupgradeinfo', + 's1acpimanagementsupported', + 'runtime', # vim.vm.RuntimeInfo + 'rootsnapshot', + 'reverttosnapshotsupported', + 'resourcepool', + 'repconfig', + 'recordreplaysupported', + 'recordreplaystate', + 'recenttask', + 'quiescedsnapshotssupported', + 'quiescedforkparent', + 'quickstats', + 'question', + 'powerstate', + 'powerpolicy', + 'poweredonmonitortypechangesupported', + 'poweredoffsnapshotssupported', + 'pervmevcsupported', + 'permission', + 'perdatastoreusage', + 'paused', + 'parentvapp', + 'overallstatus', + 'onlinestandby', + 'offlinefeaturerequirement', + 'nummksconnections', + 'npivwwnonnonrdmvmsupported', + 'npivworldwidenametype', + 'npivtemporarydisabled', + 'npivportworldwidename', + 'npivonnonrdmdisks', + 'npivnodeworldwidename', + 'npivdesiredportwwns', + 'npivdesirednodewwns', + 'networkshaper', + 'network', # vim.dvs.DistributedVirtualPortgroup + 'net', + 'nestedhvsupported', + 'nestedhvenabled', + 'needsecondaryreason', + 'name', + 'multiplesnapshotssupported', + 'multiplecorespersocketsupported', + 'modified', + 'minrequiredevcmodekey', + 'migrateencryption', + 'messagebustunnelenabled', + 'messagebussupported', + 'memorysnapshotssupported', + 'memoryreservationlocksupported', + 'memoryreservationlockedtomax', + 'memoryoverhead', + 'memoryhotaddenabled', + 'memoryallocation', + 'memoryaffinity', + 'maxmksconnections', + 'maxmemoryusage', + 'maxcpuusage', + 'managedby', + 'locksnapshotssupported', + 'locationid', + 'layoutex', + 'layout', + 'latencysensitivity', + 'keyid', + 'ipstack', + 'ipaddress', + 'interactiveguestoperationsready', + 'instantclonefrozen', + 'instanceuuid', + 'initialoverhead', + 'hotplugmemorylimit', + 'hotplugmemoryincrementsize', + 'hostname', + 'hostbasedreplicationsupported', + 'host', + 'hardware', + 'gueststatechangesupported', + 'gueststate', + 'guestoperationsready', + 'guestkernelcrashed', + 'guestintegrityinfo', + 'guestid', + 'guestheartbeatstatus', + 'guestfullname', + 'guestfamily', + 'guestautolocksupported', + 'guestautolockenabled', + 'guest', # Object of type 'vim.vm.GuestInfo' is not JSON serializable + 'generationinfo', + 'ftinfo', + 'forkconfiginfo', + 'flags', + 'firmware', + 'files', + 'featurerequirementsupported', + 'featurerequirement', + 'featuremask', + 'faulttolerancestate', + 'extraconfig', + 'effectiverole', + 'dynamictype', + 'dynamicproperty', + 'disksharessupported', + 'diskonlysnapshotonsuspendedvmsupported', + 'disk', + 'disablesnapshotssupported', + 'device', + 'defaultpowerops', + 'datastoreurl', + 'datastore', # some kind of datastore object + 'dasvmprotection', # Object of type 'vim.Datastore' is not JSON serializable + 'customvalue', + 'cryptostate', + 'createdate', + 'cpuhotremoveenabled', + 'cpuhotaddenabled', + 'cpufeaturemasksupported', + 'cpufeaturemask', + 'cpuallocation', + 'cpuaffinity', + 'runtime.consolidationneeded', + 'consolepreferencessupported', + 'consolepreferences', + 'connectionstate', + 'configstatus', + 'configissue', + 'config', # Object of type 'vim.vm.ConfigInfo' is not JSON serializable + 'cleanpoweroff', + 'changeversion', + 'changetrackingsupported', + 'changetrackingenabled', + 'capability', # non-python object, bug + 'canconnectusbdevices', + 'boottime', + 'bootretryoptionssupported', + 'bootoptionssupported', + 'bootoptions', + 'availablefield', + 'appstate', + 'appheartbeatstatus', + 'ansible_uuid', + 'ansible_ssh_host', + 'ansible_host', + 'annotation', + 'alternateguestname' + ] + return ret + def build_script_private_data(self, inventory_update, private_data_dir): cp = configparser.RawConfigParser() credential = inventory_update.get_cloud_credential() diff --git a/requirements/collections_requirements.yml b/requirements/collections_requirements.yml index e6f61311d5..4fe18f8a33 100644 --- a/requirements/collections_requirements.yml +++ b/requirements/collections_requirements.yml @@ -9,5 +9,5 @@ collections: - name: theforeman.foreman # needs inventory plugin published - name: google.cloud # https://github.com/ansible-collections/ansible_collections_google/pull/167 # - name: openstack.cloud # needs to be published - # - name: community.vmware # not published + - name: community.vmware # needs patch https://github.com/ansible-collections/vmware/pull/58 - name: ovirt.ovirt_collection # new fix published From f38437b6bc77c996797ee0e2fac782552517abb5 Mon Sep 17 00:00:00 2001 From: Jim Ladd Date: Mon, 13 Jan 2020 12:40:59 -0800 Subject: [PATCH 04/19] foreman plugin updates --- awx/main/models/inventory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index dc0b8f6797..022d9a409c 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -2594,7 +2594,7 @@ class rhv(PluginFileInjector): class satellite6(PluginFileInjector): plugin_name = 'foreman' ini_env_reference = 'FOREMAN_INI_PATH' - # initial_version = '2.8' # FIXME: turn on after plugin is validated + initial_version = 2.10 # No base injector, because this does not work in playbooks. Bug?? namespace = 'theforeman' collection = 'foreman' From 0e2786d1f19f41d3d42a20e1b140fba904be5586 Mon Sep 17 00:00:00 2001 From: Jim Ladd Date: Mon, 17 Feb 2020 21:39:29 -0800 Subject: [PATCH 05/19] compat layer for foreman --- awx/main/models/inventory.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 022d9a409c..41373b2cb8 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -2594,7 +2594,7 @@ class rhv(PluginFileInjector): class satellite6(PluginFileInjector): plugin_name = 'foreman' ini_env_reference = 'FOREMAN_INI_PATH' - initial_version = 2.10 + initial_version = '2.10' # No base injector, because this does not work in playbooks. Bug?? namespace = 'theforeman' collection = 'foreman' @@ -2666,6 +2666,37 @@ class satellite6(PluginFileInjector): ret['FOREMAN_PASSWORD'] = credential.get_input('password', default='') return ret + def inventory_as_dict(self, inventory_update, private_data_dir): + ret = super(satellite6, self).inventory_as_dict(inventory_update, private_data_dir) + + # Compatibility content + group_by_hostvar = { + "environment": {"prefix": "foreman_environment_", + "separator": "", + "key": "foreman['environment_name'] | lower | regex_replace(' ', '') | " + "regex_replace('[^A-Za-z0-9\_]', '_') | regex_replace('none', '')"}, # NOQA: W605 + "location": {"prefix": "foreman_location_", + "separator": "", + "key": "foreman['location_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9\_]', '_')"}, + "organization": {"prefix": "foreman_organization_", + "separator": "", + "key": "foreman['organization_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9\_]', '_')"}, + "lifecycle_environment": {"prefix": "foreman_lifecycle_environment_", + "separator": "", + "key": "foreman['content_facet_attributes']['lifecycle_environment_name'] | " + "lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9\_]', '_')"}, + "content_view": {"prefix": "foreman_content_view_", + "separator": "", + "key": "foreman['content_facet_attributes']['content_view_name'] | " + "lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9\_]', '_')"} + } + ret['keyed_groups'] = [group_by_hostvar[grouping_name] for grouping_name in group_by_hostvar] + ret['legacy_hostvars'] = True + ret['want_facts'] = True + ret['want_params'] = True + + return ret + class cloudforms(PluginFileInjector): # plugin_name = 'FIXME' # contribute inventory plugin to Ansible From 8a20b5225bc1bc75ed0583413b2b889f08e7b923 Mon Sep 17 00:00:00 2001 From: Jim Ladd Date: Mon, 13 Jan 2020 12:39:43 -0800 Subject: [PATCH 06/19] enable aws_ec2 plugin --- awx/main/models/inventory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 41373b2cb8..ad1f4ddacc 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -1883,7 +1883,7 @@ class azure_rm(PluginFileInjector): class ec2(PluginFileInjector): plugin_name = 'aws_ec2' # blocked by https://github.com/ansible/ansible/issues/54059 - # initial_version = '2.8' # Driven by unsafe group names issue, parent_group templating, hostvars + initial_version = '2.9' # Driven by unsafe group names issue, parent_group templating, hostvars ini_env_reference = 'EC2_INI_PATH' base_injector = 'managed' namespace = 'ansible' From 96c6cf9f054bfb91fc0befccf98dcf8853004331 Mon Sep 17 00:00:00 2001 From: Jim Ladd Date: Mon, 24 Feb 2020 16:03:49 -0800 Subject: [PATCH 07/19] pass iam_role_arn through to aws inv. plugin --- awx/main/models/inventory.py | 3 +++ awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml | 1 + awx/main/tests/functional/test_inventory_source_injectors.py | 3 ++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index ad1f4ddacc..1c66c2eddc 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -2027,6 +2027,9 @@ class ec2(PluginFileInjector): grouping_data['key'] += ' | regex_replace("{rx}", "_")'.format(rx=legacy_regex) # end compatibility content + if source_vars.get('iam_role_arn', None): + ret['iam_role_arn'] = source_vars['iam_role_arn'] + # This was an allowed ec2.ini option, also plugin option, so pass through if source_vars.get('boto_profile', None): ret['boto_profile'] = source_vars['boto_profile'] diff --git a/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml b/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml index f214483dfd..a56f7a1ad0 100644 --- a/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml +++ b/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml @@ -1,4 +1,5 @@ boto_profile: /tmp/my_boto_stuff +iam_role_arn: arn:aws:iam::123456789012:role/test-role compose: ansible_host: public_ip_address ec2_account_id: owner_id diff --git a/awx/main/tests/functional/test_inventory_source_injectors.py b/awx/main/tests/functional/test_inventory_source_injectors.py index 277ebd3c3a..2557b158d6 100644 --- a/awx/main/tests/functional/test_inventory_source_injectors.py +++ b/awx/main/tests/functional/test_inventory_source_injectors.py @@ -38,7 +38,8 @@ TEST_SOURCE_FIELDS = { INI_TEST_VARS = { 'ec2': { - 'boto_profile': '/tmp/my_boto_stuff' + 'boto_profile': '/tmp/my_boto_stuff', + 'iam_role_arn': 'arn:aws:iam::123456789012:role/test-role' }, 'gce': {}, 'openstack': { From 6807878e2d6d7704ac9bf427b3c8ed3eff601df7 Mon Sep 17 00:00:00 2001 From: Jim Ladd Date: Tue, 25 Feb 2020 11:51:41 -0800 Subject: [PATCH 08/19] I think this is the right place for iam_role_arn in the tests? --- awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml b/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml index a56f7a1ad0..257da7d611 100644 --- a/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml +++ b/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml @@ -1,5 +1,4 @@ boto_profile: /tmp/my_boto_stuff -iam_role_arn: arn:aws:iam::123456789012:role/test-role compose: ansible_host: public_ip_address ec2_account_id: owner_id @@ -54,6 +53,7 @@ hostnames: - network-interface.addresses.association.public-ip - dns-name - private-dns-name +iam_role_arn: arn:aws:iam::123456789012:role/test-role keyed_groups: - key: placement.availability_zone parent_group: zones From c3f2b3e44dee4dfb02394b015cbe7d5467402e47 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Thu, 26 Mar 2020 08:14:53 -0400 Subject: [PATCH 09/19] bump versions of Galaxy collection requirements Implement 2.9 version policy Update ec2 collection name Enable ovirt, refresh test files Put in upstream forks to get it running for now pick up openstack.cloud fix --- awx/main/models/inventory.py | 14 +- .../inventory/plugins/ec2/files/aws_ec2.yml | 2 +- .../tests/data/inventory/plugins/rhv/env.json | 7 + .../plugins/rhv/files/file_reference | 5 + .../inventory/plugins/rhv/files/ovirt.yml | 1 + .../plugins/satellite6/files/foreman.yml | 19 ++ .../data/inventory/plugins/vmware/env.json | 7 + .../vmware/files/vmware_vm_inventory.yml | 215 ++++++++++++++++++ .../scripts/ec2/files/file_reference | 1 + requirements/collections_requirements.yml | 20 +- 10 files changed, 275 insertions(+), 16 deletions(-) create mode 100644 awx/main/tests/data/inventory/plugins/rhv/env.json create mode 100644 awx/main/tests/data/inventory/plugins/rhv/files/file_reference create mode 100644 awx/main/tests/data/inventory/plugins/rhv/files/ovirt.yml create mode 100644 awx/main/tests/data/inventory/plugins/vmware/env.json create mode 100644 awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 1c66c2eddc..b5c2f4f0d6 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -1616,7 +1616,7 @@ class PluginFileInjector(object): # so that a source without a collection will have null values namespace = None collection = None - collection_migration = '2.10' # In this version, content moved to collections + collection_migration = '2.9' # Starting with this version, we use collections def __init__(self, ansible_version): # This is InventoryOptions instance, could be source or inventory update @@ -1886,8 +1886,8 @@ class ec2(PluginFileInjector): initial_version = '2.9' # Driven by unsafe group names issue, parent_group templating, hostvars ini_env_reference = 'EC2_INI_PATH' base_injector = 'managed' - namespace = 'ansible' - collection = 'amazon' + namespace = 'amazon' + collection = 'aws' def get_plugin_env(self, *args, **kwargs): ret = super(ec2, self).get_plugin_env(*args, **kwargs) @@ -2230,10 +2230,10 @@ class gce(PluginFileInjector): class vmware(PluginFileInjector): plugin_name = 'vmware_vm_inventory' # FIXME: implement me - initial_version = '2.10' # Only tested with collection version, TODO: test 2.9 as candidate + initial_version = '2.9' # Only tested with collection version ini_env_reference = 'VMWARE_INI_PATH' base_injector = 'managed' - namespace = 'community' + namespace = 'alancoding' # FIXME collection = 'vmware' @property @@ -2584,7 +2584,7 @@ class openstack(PluginFileInjector): class rhv(PluginFileInjector): """ovirt uses the custom credential templating, and that is all """ - # plugin_name = 'FIXME' # contribute inventory plugin to Ansible + plugin_name = 'ovirt' base_injector = 'template' namespace = 'ovirt' collection = 'ovirt_collection' @@ -2597,7 +2597,7 @@ class rhv(PluginFileInjector): class satellite6(PluginFileInjector): plugin_name = 'foreman' ini_env_reference = 'FOREMAN_INI_PATH' - initial_version = '2.10' + initial_version = '2.9' # No base injector, because this does not work in playbooks. Bug?? namespace = 'theforeman' collection = 'foreman' diff --git a/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml b/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml index 257da7d611..fc7f9c2421 100644 --- a/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml +++ b/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml @@ -76,7 +76,7 @@ keyed_groups: parent_group: '{{ placement.region }}' prefix: '' separator: '' -plugin: ansible.amazon.aws_ec2 +plugin: amazon.aws.aws_ec2 regions: - us-east-2 - ap-south-1 diff --git a/awx/main/tests/data/inventory/plugins/rhv/env.json b/awx/main/tests/data/inventory/plugins/rhv/env.json new file mode 100644 index 0000000000..08477df169 --- /dev/null +++ b/awx/main/tests/data/inventory/plugins/rhv/env.json @@ -0,0 +1,7 @@ +{ + "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never", + "OVIRT_INI_PATH": "{{ file_reference }}", + "OVIRT_PASSWORD": "fooo", + "OVIRT_URL": "https://foo.invalid", + "OVIRT_USERNAME": "fooo" +} \ No newline at end of file diff --git a/awx/main/tests/data/inventory/plugins/rhv/files/file_reference b/awx/main/tests/data/inventory/plugins/rhv/files/file_reference new file mode 100644 index 0000000000..06c2180789 --- /dev/null +++ b/awx/main/tests/data/inventory/plugins/rhv/files/file_reference @@ -0,0 +1,5 @@ +[ovirt] +ovirt_url=https://foo.invalid +ovirt_username=fooo +ovirt_password=fooo +ovirt_ca_file=fooo \ No newline at end of file diff --git a/awx/main/tests/data/inventory/plugins/rhv/files/ovirt.yml b/awx/main/tests/data/inventory/plugins/rhv/files/ovirt.yml new file mode 100644 index 0000000000..a2aacf5656 --- /dev/null +++ b/awx/main/tests/data/inventory/plugins/rhv/files/ovirt.yml @@ -0,0 +1 @@ +plugin: ovirt.ovirt_collection.ovirt diff --git a/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml b/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml index 4b950202b6..20d868137a 100644 --- a/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml +++ b/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml @@ -1 +1,20 @@ +keyed_groups: +- key: foreman['environment_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9\_]', '_') | regex_replace('none', '') + prefix: foreman_environment_ + separator: '' +- key: foreman['location_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9\_]', '_') + prefix: foreman_location_ + separator: '' +- key: foreman['organization_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9\_]', '_') + prefix: foreman_organization_ + separator: '' +- key: foreman['content_facet_attributes']['lifecycle_environment_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9\_]', '_') + prefix: foreman_lifecycle_environment_ + separator: '' +- key: foreman['content_facet_attributes']['content_view_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9\_]', '_') + prefix: foreman_content_view_ + separator: '' +legacy_hostvars: true plugin: theforeman.foreman.foreman +want_facts: true +want_params: true diff --git a/awx/main/tests/data/inventory/plugins/vmware/env.json b/awx/main/tests/data/inventory/plugins/vmware/env.json new file mode 100644 index 0000000000..97563377c0 --- /dev/null +++ b/awx/main/tests/data/inventory/plugins/vmware/env.json @@ -0,0 +1,7 @@ +{ + "ANSIBLE_TRANSFORM_INVALID_GROUP_CHARS": "never", + "VMWARE_HOST": "https://foo.invalid", + "VMWARE_PASSWORD": "fooo", + "VMWARE_USER": "fooo", + "VMWARE_VALIDATE_CERTS": "False" +} \ No newline at end of file diff --git a/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml b/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml new file mode 100644 index 0000000000..b83f8a77f7 --- /dev/null +++ b/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml @@ -0,0 +1,215 @@ +plugin: community.vmware.vmware_vm_inventory +properties: +- name +- config.cpuHotAddEnabled +- config.cpuHotRemoveEnabled +- config.instanceUuid +- config.hardware.numCPU +- config.template +- config.name +- guest.hostName +- guest.ipAddress +- guest.guestId +- guest.guestState +- runtime.maxMemoryUsage +- customValue +- vpmcsupported +- vpmcenabled +- vmxconfigchecksum +- vmstorageobjectid +- vmnpivwwnupdatesupported +- vmnpivwwnsupported +- vmnpivwwndisablesupported +- vmfsnativesnapshotsupported +- vm +- virtualmmuusagesupported +- virtualmmuusageignored +- virtualexecusageignored +- vflashcachereservation +- vflashcacheallocation +- version +- vassertsenabled +- vappconfig +- value +- uuid +- triggeredalarmstate +- toolsversionstatus2 +- toolsversionstatus +- toolsversion +- toolsupdatestatus +- toolssynctimesupported +- toolsstatus +- toolsrunningstatus +- toolsrebootpredictsupported +- toolsinstalltype +- toolsinstallermounted +- toolsautoupdatesupported +- tools +- timestamp +- template +- tag +- swapstorageobjectid +- swapplacementsupported +- swapplacement +- suspendtime +- suspendinterval +- summary +- storage +- snapshotoperationssupported +- snapshotinbackground +- snapshotconfigsupported +- snapshot +- settingvideoramsizesupported +- settingscreenresolutionsupported +- settingdisplaytopologysupported +- settingdisplaytopologymodessupported +- sesparsedisksupported +- securebootsupported +- screen +- scheduledhardwareupgradeinfo +- s1acpimanagementsupported +- runtime +- rootsnapshot +- reverttosnapshotsupported +- resourcepool +- repconfig +- recordreplaysupported +- recordreplaystate +- recenttask +- quiescedsnapshotssupported +- quiescedforkparent +- quickstats +- question +- powerstate +- powerpolicy +- poweredonmonitortypechangesupported +- poweredoffsnapshotssupported +- pervmevcsupported +- permission +- perdatastoreusage +- paused +- parentvapp +- overallstatus +- onlinestandby +- offlinefeaturerequirement +- nummksconnections +- npivwwnonnonrdmvmsupported +- npivworldwidenametype +- npivtemporarydisabled +- npivportworldwidename +- npivonnonrdmdisks +- npivnodeworldwidename +- npivdesiredportwwns +- npivdesirednodewwns +- networkshaper +- network +- net +- nestedhvsupported +- nestedhvenabled +- needsecondaryreason +- name +- multiplesnapshotssupported +- multiplecorespersocketsupported +- modified +- minrequiredevcmodekey +- migrateencryption +- messagebustunnelenabled +- messagebussupported +- memorysnapshotssupported +- memoryreservationlocksupported +- memoryreservationlockedtomax +- memoryoverhead +- memoryhotaddenabled +- memoryallocation +- memoryaffinity +- maxmksconnections +- maxmemoryusage +- maxcpuusage +- managedby +- locksnapshotssupported +- locationid +- layoutex +- layout +- latencysensitivity +- keyid +- ipstack +- ipaddress +- interactiveguestoperationsready +- instantclonefrozen +- instanceuuid +- initialoverhead +- hotplugmemorylimit +- hotplugmemoryincrementsize +- hostname +- hostbasedreplicationsupported +- host +- hardware +- gueststatechangesupported +- gueststate +- guestoperationsready +- guestkernelcrashed +- guestintegrityinfo +- guestid +- guestheartbeatstatus +- guestfullname +- guestfamily +- guestautolocksupported +- guestautolockenabled +- guest +- generationinfo +- ftinfo +- forkconfiginfo +- flags +- firmware +- files +- featurerequirementsupported +- featurerequirement +- featuremask +- faulttolerancestate +- extraconfig +- effectiverole +- dynamictype +- dynamicproperty +- disksharessupported +- diskonlysnapshotonsuspendedvmsupported +- disk +- disablesnapshotssupported +- device +- defaultpowerops +- datastoreurl +- datastore +- dasvmprotection +- customvalue +- cryptostate +- createdate +- cpuhotremoveenabled +- cpuhotaddenabled +- cpufeaturemasksupported +- cpufeaturemask +- cpuallocation +- cpuaffinity +- runtime.consolidationneeded +- consolepreferencessupported +- consolepreferences +- connectionstate +- configstatus +- configissue +- config +- cleanpoweroff +- changeversion +- changetrackingsupported +- changetrackingenabled +- capability +- canconnectusbdevices +- boottime +- bootretryoptionssupported +- bootoptionssupported +- bootoptions +- availablefield +- appstate +- appheartbeatstatus +- ansible_uuid +- ansible_ssh_host +- ansible_host +- annotation +- alternateguestname diff --git a/awx/main/tests/data/inventory/scripts/ec2/files/file_reference b/awx/main/tests/data/inventory/scripts/ec2/files/file_reference index aef5c1441c..564e631ba0 100644 --- a/awx/main/tests/data/inventory/scripts/ec2/files/file_reference +++ b/awx/main/tests/data/inventory/scripts/ec2/files/file_reference @@ -1,6 +1,7 @@ [ec2] base_source_var = value_of_var boto_profile = /tmp/my_boto_stuff +iam_role_arn = arn:aws:iam::123456789012:role/test-role regions = us-east-2,ap-south-1 regions_exclude = us-gov-west-1,cn-north-1 destination_variable = public_dns_name diff --git a/requirements/collections_requirements.yml b/requirements/collections_requirements.yml index 4fe18f8a33..3f1b5bfe9b 100644 --- a/requirements/collections_requirements.yml +++ b/requirements/collections_requirements.yml @@ -2,12 +2,16 @@ collections: - name: awx.awx version: 9.3.0 - source: https://galaxy.ansible.com - name: azure.azcollection - version: 0.1.1 # https://github.com/ansible-collections/azure/issues/55 - # - name: ansible.amazon # needs to be published - - name: theforeman.foreman # needs inventory plugin published - - name: google.cloud # https://github.com/ansible-collections/ansible_collections_google/pull/167 - # - name: openstack.cloud # needs to be published - - name: community.vmware # needs patch https://github.com/ansible-collections/vmware/pull/58 - - name: ovirt.ovirt_collection # new fix published + version: 0.1.1 # questionable https://github.com/ansible-collections/azure/issues/55 + - name: amazon.aws + version: 0.1.0 + - name: theforeman.foreman + version: 0.7.0 # contains the inventory plugin, but more patches are needed + - name: google.cloud + version: 0.0.9 # contains PR 167, should be good to go + - name: openstack.cloud + version: 0.0.1-dev85 # earlier had checksum mismatch + - name: alancoding.vmware # FIXME needs patch https://github.com/ansible-collections/vmware/pull/58 + - name: ovirt.ovirt_collection + version: 1.0.1 # new fix published, should be good to go From 6b015c9d8140b40430abad417e0fd8bf63e37dda Mon Sep 17 00:00:00 2001 From: Jim Ladd Date: Tue, 31 Mar 2020 12:51:33 -0700 Subject: [PATCH 10/19] Call parent get_plugin_env for foreman --- awx/main/models/inventory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index b5c2f4f0d6..4a9e3962ec 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -2662,7 +2662,7 @@ class satellite6(PluginFileInjector): # this assumes that this is merged # https://github.com/ansible/ansible/pull/52693 credential = inventory_update.get_cloud_credential() - ret = {} + ret = super(satellite6, self).get_plugin_env(inventory_update, private_data_dir, private_data_files) if credential: ret['FOREMAN_SERVER'] = credential.get_input('host', default='') ret['FOREMAN_USER'] = credential.get_input('username', default='') From 47bdf86dfa2753c0ad16583f2e43e296107c7e57 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Tue, 31 Mar 2020 20:07:22 -0400 Subject: [PATCH 11/19] Pick up collections paths in openstack inventory update prep Update amazon.aws version Update vmware to new release Pin some more tests due to transitioning to inventory plugins Remove more temporary hacks --- awx/main/models/inventory.py | 9 ++++++--- awx/main/tests/unit/test_tasks.py | 15 +++++++++------ requirements/collections_requirements.yml | 5 +++-- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 4a9e3962ec..ddd9c1a224 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -2229,11 +2229,11 @@ class gce(PluginFileInjector): class vmware(PluginFileInjector): - plugin_name = 'vmware_vm_inventory' # FIXME: implement me + plugin_name = 'vmware_vm_inventory' initial_version = '2.9' # Only tested with collection version ini_env_reference = 'VMWARE_INI_PATH' base_injector = 'managed' - namespace = 'alancoding' # FIXME + namespace = 'community' collection = 'vmware' @property @@ -2544,7 +2544,10 @@ class openstack(PluginFileInjector): return self.build_script_private_data(inventory_update, private_data_dir, mk_cache=False) def get_plugin_env(self, inventory_update, private_data_dir, private_data_files): - return self.get_script_env(inventory_update, private_data_dir, private_data_files) + env = super(openstack, self).get_plugin_env(inventory_update, private_data_dir, private_data_files) + script_env = self.get_script_env(inventory_update, private_data_dir, private_data_files) + env.update(script_env) + return env def inventory_as_dict(self, inventory_update, private_data_dir): def use_host_name_for_name(a_bool_maybe): diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 78852cbbf2..e7e208d9a3 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -1807,8 +1807,9 @@ class TestInventoryUpdateCredentials(TestJobExecution): inventory_update.get_cloud_credential = mocker.Mock(return_value=None) inventory_update.get_extra_credentials = mocker.Mock(return_value=[]) - private_data_files = task.build_private_data_files(inventory_update, private_data_dir) - env = task.build_env(inventory_update, private_data_dir, False, private_data_files) + with mocker.patch('awx.main.tasks._get_ansible_version', mocker.MagicMock(return_value='2.7')): + private_data_files = task.build_private_data_files(inventory_update, private_data_dir) + env = task.build_env(inventory_update, private_data_dir, False, private_data_files) assert 'AWS_ACCESS_KEY_ID' not in env assert 'AWS_SECRET_ACCESS_KEY' not in env @@ -1915,8 +1916,9 @@ class TestInventoryUpdateCredentials(TestJobExecution): inventory_update.get_cloud_credential = get_cred inventory_update.get_extra_credentials = mocker.Mock(return_value=[]) - private_data_files = task.build_private_data_files(inventory_update, private_data_dir) - env = task.build_env(inventory_update, private_data_dir, False, private_data_files) + with mocker.patch('awx.main.tasks._get_ansible_version', mocker.MagicMock(return_value='2.7')): + private_data_files = task.build_private_data_files(inventory_update, private_data_dir) + env = task.build_env(inventory_update, private_data_dir, False, private_data_files) safe_env = {} credentials = task.build_credentials_list(inventory_update) @@ -2153,8 +2155,9 @@ class TestInventoryUpdateCredentials(TestJobExecution): 'satellite6_want_facts': False } - private_data_files = task.build_private_data_files(inventory_update, private_data_dir) - env = task.build_env(inventory_update, private_data_dir, False, private_data_files) + with mocker.patch('awx.main.tasks._get_ansible_version', mocker.MagicMock(return_value='2.7')): + private_data_files = task.build_private_data_files(inventory_update, private_data_dir) + env = task.build_env(inventory_update, private_data_dir, False, private_data_files) config = configparser.ConfigParser() config.read(env['FOREMAN_INI_PATH']) diff --git a/requirements/collections_requirements.yml b/requirements/collections_requirements.yml index 3f1b5bfe9b..cec16c491f 100644 --- a/requirements/collections_requirements.yml +++ b/requirements/collections_requirements.yml @@ -5,13 +5,14 @@ collections: - name: azure.azcollection version: 0.1.1 # questionable https://github.com/ansible-collections/azure/issues/55 - name: amazon.aws - version: 0.1.0 + version: 0.1.1 # version 0.1.0 seems to have gone missing - name: theforeman.foreman version: 0.7.0 # contains the inventory plugin, but more patches are needed - name: google.cloud version: 0.0.9 # contains PR 167, should be good to go - name: openstack.cloud version: 0.0.1-dev85 # earlier had checksum mismatch - - name: alancoding.vmware # FIXME needs patch https://github.com/ansible-collections/vmware/pull/58 + - name: community.vmware + version: 0.3.1-dev1 - name: ovirt.ovirt_collection version: 1.0.1 # new fix published, should be good to go From 84d863ff9d3600a36789ef98738703b6751ced61 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Thu, 9 Apr 2020 14:23:08 -0400 Subject: [PATCH 12/19] Shorten list of VMWare inventory plugin properties Do not include summaries decision on vmware keyed groups Handle first set of custom user vmware options --- awx/main/models/inventory.py | 298 +++++------------- .../vmware/files/vmware_vm_inventory.yml | 245 ++------------ 2 files changed, 121 insertions(+), 422 deletions(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index ddd9c1a224..ac6ab5acc0 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -2243,222 +2243,96 @@ class vmware(PluginFileInjector): def inventory_as_dict(self, inventory_update, private_data_dir): ret = super(vmware, self).inventory_as_dict(inventory_update, private_data_dir) + ret['strict'] = True ret['properties'] = [ "name", - "config.cpuHotAddEnabled", - "config.cpuHotRemoveEnabled", - "config.instanceUuid", - "config.hardware.numCPU", - "config.template", - "config.name", - "guest.hostName", - "guest.ipAddress", - "guest.guestId", - "guest.guestState", - "runtime.maxMemoryUsage", - "customValue", - # not in the defaults - "vpmcsupported", - "vpmcenabled", - "vmxconfigchecksum", - "vmstorageobjectid", - "vmnpivwwnupdatesupported", - "vmnpivwwnsupported", - "vmnpivwwndisablesupported", - "vmfsnativesnapshotsupported", - "vm", - "virtualmmuusagesupported", - "virtualmmuusageignored", - "virtualexecusageignored", - "vflashcachereservation", - "vflashcacheallocation", - "version", - "vassertsenabled", - "vappconfig", + "ansible_ssh_host", + "ansible_host", + "ansible_uuid", + "availablefield", + "capability", # nested properties + "config", # nested properties + "configissue", + "configstatus", + "customvalue", + "datastore", + "effectiverole", + "guest", # nested properties + "guestheartbeatstatus", + "layout", + "layoutex", + "name", + "network", + "overallstatus", + "parentvapp", + "permission", + "recenttask", + "resourcepool", + "rootsnapshot", + "runtime", # nested properties + "snapshot", + "storage", # nested properties + "summary", # repeat of other properties + "tag", + "triggeredalarmstate", "value", - "uuid", - 'triggeredalarmstate', - 'toolsversionstatus2', - 'toolsversionstatus', - 'toolsversion', - 'toolsupdatestatus', - 'toolssynctimesupported', - 'toolsstatus', - 'toolsrunningstatus', - 'toolsrebootpredictsupported', - 'toolsinstalltype', - 'toolsinstallermounted', - 'toolsautoupdatesupported', - 'tools', - 'timestamp', - 'template', - 'tag', - 'swapstorageobjectid', - 'swapplacementsupported', - 'swapplacement', - 'suspendtime', - 'suspendinterval', - 'summary', # vim.vm.Summary - 'storage', # vim.vm.StorageInfo - 'snapshotoperationssupported', - 'snapshotinbackground', - 'snapshotconfigsupported', - 'snapshot', - 'settingvideoramsizesupported', - 'settingscreenresolutionsupported', - 'settingdisplaytopologysupported', - 'settingdisplaytopologymodessupported', - 'sesparsedisksupported', - 'securebootsupported', - 'screen', - 'scheduledhardwareupgradeinfo', - 's1acpimanagementsupported', - 'runtime', # vim.vm.RuntimeInfo - 'rootsnapshot', - 'reverttosnapshotsupported', - 'resourcepool', - 'repconfig', - 'recordreplaysupported', - 'recordreplaystate', - 'recenttask', - 'quiescedsnapshotssupported', - 'quiescedforkparent', - 'quickstats', - 'question', - 'powerstate', - 'powerpolicy', - 'poweredonmonitortypechangesupported', - 'poweredoffsnapshotssupported', - 'pervmevcsupported', - 'permission', - 'perdatastoreusage', - 'paused', - 'parentvapp', - 'overallstatus', - 'onlinestandby', - 'offlinefeaturerequirement', - 'nummksconnections', - 'npivwwnonnonrdmvmsupported', - 'npivworldwidenametype', - 'npivtemporarydisabled', - 'npivportworldwidename', - 'npivonnonrdmdisks', - 'npivnodeworldwidename', - 'npivdesiredportwwns', - 'npivdesirednodewwns', - 'networkshaper', - 'network', # vim.dvs.DistributedVirtualPortgroup - 'net', - 'nestedhvsupported', - 'nestedhvenabled', - 'needsecondaryreason', - 'name', - 'multiplesnapshotssupported', - 'multiplecorespersocketsupported', - 'modified', - 'minrequiredevcmodekey', - 'migrateencryption', - 'messagebustunnelenabled', - 'messagebussupported', - 'memorysnapshotssupported', - 'memoryreservationlocksupported', - 'memoryreservationlockedtomax', - 'memoryoverhead', - 'memoryhotaddenabled', - 'memoryallocation', - 'memoryaffinity', - 'maxmksconnections', - 'maxmemoryusage', - 'maxcpuusage', - 'managedby', - 'locksnapshotssupported', - 'locationid', - 'layoutex', - 'layout', - 'latencysensitivity', - 'keyid', - 'ipstack', - 'ipaddress', - 'interactiveguestoperationsready', - 'instantclonefrozen', - 'instanceuuid', - 'initialoverhead', - 'hotplugmemorylimit', - 'hotplugmemoryincrementsize', - 'hostname', - 'hostbasedreplicationsupported', - 'host', - 'hardware', - 'gueststatechangesupported', - 'gueststate', - 'guestoperationsready', - 'guestkernelcrashed', - 'guestintegrityinfo', - 'guestid', - 'guestheartbeatstatus', - 'guestfullname', - 'guestfamily', - 'guestautolocksupported', - 'guestautolockenabled', - 'guest', # Object of type 'vim.vm.GuestInfo' is not JSON serializable - 'generationinfo', - 'ftinfo', - 'forkconfiginfo', - 'flags', - 'firmware', - 'files', - 'featurerequirementsupported', - 'featurerequirement', - 'featuremask', - 'faulttolerancestate', - 'extraconfig', - 'effectiverole', - 'dynamictype', - 'dynamicproperty', - 'disksharessupported', - 'diskonlysnapshotonsuspendedvmsupported', - 'disk', - 'disablesnapshotssupported', - 'device', - 'defaultpowerops', - 'datastoreurl', - 'datastore', # some kind of datastore object - 'dasvmprotection', # Object of type 'vim.Datastore' is not JSON serializable - 'customvalue', - 'cryptostate', - 'createdate', - 'cpuhotremoveenabled', - 'cpuhotaddenabled', - 'cpufeaturemasksupported', - 'cpufeaturemask', - 'cpuallocation', - 'cpuaffinity', - 'runtime.consolidationneeded', - 'consolepreferencessupported', - 'consolepreferences', - 'connectionstate', - 'configstatus', - 'configissue', - 'config', # Object of type 'vim.vm.ConfigInfo' is not JSON serializable - 'cleanpoweroff', - 'changeversion', - 'changetrackingsupported', - 'changetrackingenabled', - 'capability', # non-python object, bug - 'canconnectusbdevices', - 'boottime', - 'bootretryoptionssupported', - 'bootoptionssupported', - 'bootoptions', - 'availablefield', - 'appstate', - 'appheartbeatstatus', - 'ansible_uuid', - 'ansible_ssh_host', - 'ansible_host', - 'annotation', - 'alternateguestname' ] + # ret['properties'] = ["all"] # causes UnknownWsdlTypeError exception + ret['with_nested_properties'] = True + # ret['with_sanitized_property_name'] = True # needs adjustment + # ret['filters'] = [ # needs implemented + # 'runtime.powerstate == "poweredOn"' + # ] + # these are the defaults, so we don't put them in + # ret['keyed_groups'] = [ + # { + # 'prefix': '', 'separator': '', + # # 'key': '"None" if not config.guest_id else config.guest_id' + # # 'key': 'config.get("guest_id") | default([])' # also in guest dict, depending on branch + # 'key': 'config.guestId' + # }, + # { + # 'prefix': '', 'separator': '', + # 'key': "'templates' if config.template else 'guests'" + # } + # ] + ret['property_name_format'] = 'lower_case' + + # process custom options + vmware_opts = dict(inventory_update.source_vars_dict.items()) + if inventory_update.instance_filters: + vmware_opts.setdefault('host_filters', inventory_update.instance_filters) + if inventory_update.group_by: + vmware_opts.setdefault('groupby_patterns', inventory_update.group_by) + + alias_pattern = vmware_opts.get('alias_pattern') + if alias_pattern: + ret.setdefault('hostnames', []) + for alias in alias_pattern.split(','): # make best effort + striped_alias = alias.replace('{', '').replace('}', '').strip() # make best effort + if not striped_alias: + continue + ret['hostnames'].append(striped_alias) + + host_pattern = vmware_opts.get('host_pattern') # not working in script + if host_pattern: + pass # does not appear to have an option for this + + host_filters = vmware_opts.get('host_filters') + if host_filters: + pass + # ret['resources'] = host_filters.split(',') # does not follow same pattern + + groupby_patterns = vmware_opts.get('groupby_patterns') + if groupby_patterns: + ret.setdefault('keyed_groups', []) + for pattern in groupby_patterns.split(','): + stripped_pattern = pattern.replace('{', '').replace('}', '').strip() # make best effort + ret['keyed_groups'].append({ + 'prefix': '', 'separator': '', + 'key': stripped_pattern + }) + return ret def build_script_private_data(self, inventory_update, private_data_dir): diff --git a/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml b/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml index b83f8a77f7..feb2150190 100644 --- a/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml +++ b/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml @@ -1,215 +1,40 @@ +keyed_groups: +- key: fouo + prefix: '' + separator: '' plugin: community.vmware.vmware_vm_inventory properties: - name -- config.cpuHotAddEnabled -- config.cpuHotRemoveEnabled -- config.instanceUuid -- config.hardware.numCPU -- config.template -- config.name -- guest.hostName -- guest.ipAddress -- guest.guestId -- guest.guestState -- runtime.maxMemoryUsage -- customValue -- vpmcsupported -- vpmcenabled -- vmxconfigchecksum -- vmstorageobjectid -- vmnpivwwnupdatesupported -- vmnpivwwnsupported -- vmnpivwwndisablesupported -- vmfsnativesnapshotsupported -- vm -- virtualmmuusagesupported -- virtualmmuusageignored -- virtualexecusageignored -- vflashcachereservation -- vflashcacheallocation -- version -- vassertsenabled -- vappconfig -- value -- uuid -- triggeredalarmstate -- toolsversionstatus2 -- toolsversionstatus -- toolsversion -- toolsupdatestatus -- toolssynctimesupported -- toolsstatus -- toolsrunningstatus -- toolsrebootpredictsupported -- toolsinstalltype -- toolsinstallermounted -- toolsautoupdatesupported -- tools -- timestamp -- template -- tag -- swapstorageobjectid -- swapplacementsupported -- swapplacement -- suspendtime -- suspendinterval -- summary -- storage -- snapshotoperationssupported -- snapshotinbackground -- snapshotconfigsupported -- snapshot -- settingvideoramsizesupported -- settingscreenresolutionsupported -- settingdisplaytopologysupported -- settingdisplaytopologymodessupported -- sesparsedisksupported -- securebootsupported -- screen -- scheduledhardwareupgradeinfo -- s1acpimanagementsupported -- runtime -- rootsnapshot -- reverttosnapshotsupported -- resourcepool -- repconfig -- recordreplaysupported -- recordreplaystate -- recenttask -- quiescedsnapshotssupported -- quiescedforkparent -- quickstats -- question -- powerstate -- powerpolicy -- poweredonmonitortypechangesupported -- poweredoffsnapshotssupported -- pervmevcsupported -- permission -- perdatastoreusage -- paused -- parentvapp -- overallstatus -- onlinestandby -- offlinefeaturerequirement -- nummksconnections -- npivwwnonnonrdmvmsupported -- npivworldwidenametype -- npivtemporarydisabled -- npivportworldwidename -- npivonnonrdmdisks -- npivnodeworldwidename -- npivdesiredportwwns -- npivdesirednodewwns -- networkshaper -- network -- net -- nestedhvsupported -- nestedhvenabled -- needsecondaryreason -- name -- multiplesnapshotssupported -- multiplecorespersocketsupported -- modified -- minrequiredevcmodekey -- migrateencryption -- messagebustunnelenabled -- messagebussupported -- memorysnapshotssupported -- memoryreservationlocksupported -- memoryreservationlockedtomax -- memoryoverhead -- memoryhotaddenabled -- memoryallocation -- memoryaffinity -- maxmksconnections -- maxmemoryusage -- maxcpuusage -- managedby -- locksnapshotssupported -- locationid -- layoutex -- layout -- latencysensitivity -- keyid -- ipstack -- ipaddress -- interactiveguestoperationsready -- instantclonefrozen -- instanceuuid -- initialoverhead -- hotplugmemorylimit -- hotplugmemoryincrementsize -- hostname -- hostbasedreplicationsupported -- host -- hardware -- gueststatechangesupported -- gueststate -- guestoperationsready -- guestkernelcrashed -- guestintegrityinfo -- guestid -- guestheartbeatstatus -- guestfullname -- guestfamily -- guestautolocksupported -- guestautolockenabled -- guest -- generationinfo -- ftinfo -- forkconfiginfo -- flags -- firmware -- files -- featurerequirementsupported -- featurerequirement -- featuremask -- faulttolerancestate -- extraconfig -- effectiverole -- dynamictype -- dynamicproperty -- disksharessupported -- diskonlysnapshotonsuspendedvmsupported -- disk -- disablesnapshotssupported -- device -- defaultpowerops -- datastoreurl -- datastore -- dasvmprotection -- customvalue -- cryptostate -- createdate -- cpuhotremoveenabled -- cpuhotaddenabled -- cpufeaturemasksupported -- cpufeaturemask -- cpuallocation -- cpuaffinity -- runtime.consolidationneeded -- consolepreferencessupported -- consolepreferences -- connectionstate -- configstatus -- configissue -- config -- cleanpoweroff -- changeversion -- changetrackingsupported -- changetrackingenabled -- capability -- canconnectusbdevices -- boottime -- bootretryoptionssupported -- bootoptionssupported -- bootoptions -- availablefield -- appstate -- appheartbeatstatus -- ansible_uuid - ansible_ssh_host - ansible_host -- annotation -- alternateguestname +- ansible_uuid +- availablefield +- capability +- config +- configissue +- configstatus +- customvalue +- datastore +- effectiverole +- guest +- guestheartbeatstatus +- layout +- layoutex +- name +- network +- overallstatus +- parentvapp +- permission +- recenttask +- resourcepool +- rootsnapshot +- runtime +- snapshot +- storage +- summary +- tag +- triggeredalarmstate +- value +property_name_format: lower_case +strict: true +with_nested_properties: true From e2f5aa987d71213e1be7d01acdec154e24e66d90 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Fri, 17 Apr 2020 14:52:38 -0400 Subject: [PATCH 13/19] Implement vmware plugin host filtering --- awx/main/models/inventory.py | 8 ++++++-- .../plugins/vmware/files/vmware_vm_inventory.yml | 3 +++ .../data/inventory/scripts/vmware/files/file_reference | 2 +- .../tests/functional/test_inventory_source_injectors.py | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index ac6ab5acc0..04895217b3 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -2320,8 +2320,12 @@ class vmware(PluginFileInjector): host_filters = vmware_opts.get('host_filters') if host_filters: - pass - # ret['resources'] = host_filters.split(',') # does not follow same pattern + ret.setdefault('host_filters', []) + for hf in host_filters.split(','): + striped_hf = hf.replace('{', '').replace('}', '').strip() # make best effort + if not striped_hf: + continue + ret['host_filters'].append(striped_hf) groupby_patterns = vmware_opts.get('groupby_patterns') if groupby_patterns: diff --git a/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml b/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml index feb2150190..3bc5269dd8 100644 --- a/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml +++ b/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml @@ -1,3 +1,6 @@ +host_filters: +- config.name == "only_my_server" +- somevar == "bar" keyed_groups: - key: fouo prefix: '' diff --git a/awx/main/tests/data/inventory/scripts/vmware/files/file_reference b/awx/main/tests/data/inventory/scripts/vmware/files/file_reference index 8a4d8a7700..5282c2db71 100644 --- a/awx/main/tests/data/inventory/scripts/vmware/files/file_reference +++ b/awx/main/tests/data/inventory/scripts/vmware/files/file_reference @@ -5,6 +5,6 @@ username = fooo password = fooo server = https://foo.invalid base_source_var = value_of_var -host_filters = foobaa +host_filters = {{ config.name == "only_my_server" }},{{ somevar == "bar"}} groupby_patterns = fouo diff --git a/awx/main/tests/functional/test_inventory_source_injectors.py b/awx/main/tests/functional/test_inventory_source_injectors.py index 2557b158d6..e776753274 100644 --- a/awx/main/tests/functional/test_inventory_source_injectors.py +++ b/awx/main/tests/functional/test_inventory_source_injectors.py @@ -16,7 +16,7 @@ DATA = os.path.join(os.path.dirname(data.__file__), 'inventory') TEST_SOURCE_FIELDS = { 'vmware': { - 'instance_filters': 'foobaa', + 'instance_filters': '{{ config.name == "only_my_server" }},{{ somevar == "bar"}}', 'group_by': 'fouo' }, 'ec2': { From fff34f7227791fc6a0681b2683ad48def68c58ec Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Fri, 17 Apr 2020 15:43:03 -0400 Subject: [PATCH 14/19] Make inventory non-strict for production and remove comments --- awx/main/models/inventory.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 04895217b3..e47b21b001 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -2243,7 +2243,7 @@ class vmware(PluginFileInjector): def inventory_as_dict(self, inventory_update, private_data_dir): ret = super(vmware, self).inventory_as_dict(inventory_update, private_data_dir) - ret['strict'] = True + ret['strict'] = False ret['properties'] = [ "name", "ansible_ssh_host", @@ -2279,23 +2279,6 @@ class vmware(PluginFileInjector): ] # ret['properties'] = ["all"] # causes UnknownWsdlTypeError exception ret['with_nested_properties'] = True - # ret['with_sanitized_property_name'] = True # needs adjustment - # ret['filters'] = [ # needs implemented - # 'runtime.powerstate == "poweredOn"' - # ] - # these are the defaults, so we don't put them in - # ret['keyed_groups'] = [ - # { - # 'prefix': '', 'separator': '', - # # 'key': '"None" if not config.guest_id else config.guest_id' - # # 'key': 'config.get("guest_id") | default([])' # also in guest dict, depending on branch - # 'key': 'config.guestId' - # }, - # { - # 'prefix': '', 'separator': '', - # 'key': "'templates' if config.template else 'guests'" - # } - # ] ret['property_name_format'] = 'lower_case' # process custom options From de0122d64ea569bf4aca1992cbc629ebd2319618 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Sat, 18 Apr 2020 19:32:18 -0400 Subject: [PATCH 15/19] Respect the ec2 script nested groups option --- awx/main/models/inventory.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index e47b21b001..3308c84b8b 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -2038,6 +2038,10 @@ class ec2(PluginFileInjector): # Using the plugin, but still want dashes whitelisted ret['use_contrib_script_compatible_sanitization'] = True + if source_vars.get('nested_groups') is False: + for this_keyed_group in keyed_groups: + this_keyed_group.pop('parent_group', None) + if keyed_groups: ret['keyed_groups'] = keyed_groups From 50197c6a12c0928dab80eba1b31d0061bb630be3 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Sat, 18 Apr 2020 23:17:59 -0400 Subject: [PATCH 16/19] Handle custom script options of hostnames and destination variable --- awx/main/models/inventory.py | 29 +++++++++++++++---- .../inventory/plugins/ec2/files/aws_ec2.yml | 4 +-- .../vmware/files/vmware_vm_inventory.yml | 2 +- .../scripts/ec2/files/file_reference | 3 +- .../test_inventory_source_injectors.py | 4 ++- 5 files changed, 30 insertions(+), 12 deletions(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 3308c84b8b..a32a24da44 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -2053,18 +2053,35 @@ class ec2(PluginFileInjector): compose_dict.update(self._compat_compose_vars()) # plugin provides "aws_ec2", but not this which the script gave ret['groups'] = {'ec2': True} - # public_ip as hostname is non-default plugin behavior, script behavior - ret['hostnames'] = [ - 'network-interface.addresses.association.public-ip', - 'dns-name', - 'private-dns-name' - ] + if source_vars.get('hostname_variable') is not None: + hnames = [] + for expr in source_vars.get('hostname_variable').split(','): + if expr == 'public_dns_name': + hnames.append('dns-name') + elif not expr.startswith('tag:') and '_' in expr: + hnames.append(expr.replace('_', '-')) + else: + hnames.append(expr) + ret['hostnames'] = hnames + else: + # public_ip as hostname is non-default plugin behavior, script behavior + ret['hostnames'] = [ + 'network-interface.addresses.association.public-ip', + 'dns-name', + 'private-dns-name' + ] # The script returned only running state by default, the plugin does not # https://docs.aws.amazon.com/cli/latest/reference/ec2/describe-instances.html#options # options: pending | running | shutting-down | terminated | stopping | stopped inst_filters['instance-state-name'] = ['running'] # end compatibility content + if source_vars.get('destination_variable') or source_vars.get('vpc_destination_variable'): + for fd in ('destination_variable', 'vpc_destination_variable'): + if source_vars.get(fd): + compose_dict['ansible_host'] = source_vars.get(fd) + break + if compose_dict: ret['compose'] = compose_dict diff --git a/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml b/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml index fc7f9c2421..8984d4cb56 100644 --- a/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml +++ b/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml @@ -1,6 +1,6 @@ boto_profile: /tmp/my_boto_stuff compose: - ansible_host: public_ip_address + ansible_host: public_dns_name ec2_account_id: owner_id ec2_ami_launch_index: ami_launch_index | string ec2_architecture: architecture @@ -50,9 +50,7 @@ filters: groups: ec2: true hostnames: -- network-interface.addresses.association.public-ip - dns-name -- private-dns-name iam_role_arn: arn:aws:iam::123456789012:role/test-role keyed_groups: - key: placement.availability_zone diff --git a/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml b/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml index 3bc5269dd8..18b96b9e78 100644 --- a/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml +++ b/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml @@ -39,5 +39,5 @@ properties: - triggeredalarmstate - value property_name_format: lower_case -strict: true +strict: false with_nested_properties: true diff --git a/awx/main/tests/data/inventory/scripts/ec2/files/file_reference b/awx/main/tests/data/inventory/scripts/ec2/files/file_reference index 564e631ba0..4ffb85656f 100644 --- a/awx/main/tests/data/inventory/scripts/ec2/files/file_reference +++ b/awx/main/tests/data/inventory/scripts/ec2/files/file_reference @@ -2,9 +2,10 @@ base_source_var = value_of_var boto_profile = /tmp/my_boto_stuff iam_role_arn = arn:aws:iam::123456789012:role/test-role +hostname_variable = public_dns_name +destination_variable = public_dns_name regions = us-east-2,ap-south-1 regions_exclude = us-gov-west-1,cn-north-1 -destination_variable = public_dns_name vpc_destination_variable = ip_address route53 = False all_instances = True diff --git a/awx/main/tests/functional/test_inventory_source_injectors.py b/awx/main/tests/functional/test_inventory_source_injectors.py index e776753274..50510e3c16 100644 --- a/awx/main/tests/functional/test_inventory_source_injectors.py +++ b/awx/main/tests/functional/test_inventory_source_injectors.py @@ -39,7 +39,9 @@ TEST_SOURCE_FIELDS = { INI_TEST_VARS = { 'ec2': { 'boto_profile': '/tmp/my_boto_stuff', - 'iam_role_arn': 'arn:aws:iam::123456789012:role/test-role' + 'iam_role_arn': 'arn:aws:iam::123456789012:role/test-role', + 'hostname_variable': 'public_dns_name', + 'destination_variable': 'public_dns_name' }, 'gce': {}, 'openstack': { From 68f5482c42b97d536efaee98f75046dd1aaebfb7 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Tue, 21 Apr 2020 22:49:20 -0400 Subject: [PATCH 17/19] Update vmware implementation to new agreements --- awx/main/models/inventory.py | 70 +++++++++++-------- .../vmware/files/vmware_vm_inventory.yml | 64 ++++++++++------- .../scripts/vmware/files/file_reference | 5 +- .../test_inventory_source_injectors.py | 3 + 4 files changed, 84 insertions(+), 58 deletions(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index a32a24da44..1930a89ae6 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -2265,42 +2265,50 @@ class vmware(PluginFileInjector): def inventory_as_dict(self, inventory_update, private_data_dir): ret = super(vmware, self).inventory_as_dict(inventory_update, private_data_dir) ret['strict'] = False - ret['properties'] = [ - "name", + # Documentation of props, see + # https://github.com/ansible/ansible/blob/devel/docs/docsite/rst/scenario_guides/vmware_scenarios/vmware_inventory_vm_attributes.rst + UPPERCASE_PROPS = [ "ansible_ssh_host", "ansible_host", "ansible_uuid", - "availablefield", - "capability", # nested properties - "config", # nested properties - "configissue", - "configstatus", - "customvalue", + "availableField", # optional? + "configIssue", # optional? + "configStatus", # optional? + "customValue", # optional? "datastore", - "effectiverole", - "guest", # nested properties - "guestheartbeatstatus", - "layout", - "layoutex", + "effectiveRole", # optional? + "guestHeartbeatStatus", # optonal? + "layout", # optional? + "layoutEx", # optional? "name", "network", - "overallstatus", - "parentvapp", + "overallStatus", # optional? + "parentVApp", # optional? "permission", - "recenttask", - "resourcepool", - "rootsnapshot", - "runtime", # nested properties - "snapshot", - "storage", # nested properties - "summary", # repeat of other properties + "recentTask", # optional? + "resourcePool", + "rootSnapshot", # optional? + "snapshot", # optional "tag", - "triggeredalarmstate", - "value", + "triggeredAlarmState", + "value" ] - # ret['properties'] = ["all"] # causes UnknownWsdlTypeError exception - ret['with_nested_properties'] = True - ret['property_name_format'] = 'lower_case' + NESTED_PROPS = [ + "capability", + "config", + "guest", + "runtime", + "storage", + "summary", # repeat of other properties + ] + ret['properties'] = UPPERCASE_PROPS + NESTED_PROPS + ret['compose'] = {} + for prop in UPPERCASE_PROPS: + if prop == prop.lower(): + continue + ret['compose'][prop.lower()] = prop + # ret['with_nested_properties'] = True # only dacrystal/topic/vmware-inventory-plugin-enhancements + # ret['property_name_format'] = 'lower_case' # only dacrystal/topic/vmware-inventory-plugin-property-format # process custom options vmware_opts = dict(inventory_update.source_vars_dict.items()) @@ -2320,16 +2328,18 @@ class vmware(PluginFileInjector): host_pattern = vmware_opts.get('host_pattern') # not working in script if host_pattern: - pass # does not appear to have an option for this + stripped_hp = host_pattern.replace('{', '').replace('}', '').strip() # make best effort + ret['compose']['ansible_host'] = stripped_hp + ret['compose']['ansible_ssh_host'] = stripped_hp host_filters = vmware_opts.get('host_filters') if host_filters: - ret.setdefault('host_filters', []) + ret.setdefault('filters', []) for hf in host_filters.split(','): striped_hf = hf.replace('{', '').replace('}', '').strip() # make best effort if not striped_hf: continue - ret['host_filters'].append(striped_hf) + ret['filters'].append(striped_hf) groupby_patterns = vmware_opts.get('groupby_patterns') if groupby_patterns: diff --git a/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml b/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml index 18b96b9e78..36796f1779 100644 --- a/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml +++ b/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml @@ -1,43 +1,55 @@ -host_filters: -- config.name == "only_my_server" -- somevar == "bar" +compose: + availablefield: availableField + configissue: configIssue + configstatus: configStatus + customvalue: customValue + effectiverole: effectiveRole + guestheartbeatstatus: guestHeartbeatStatus + layoutex: layoutEx + overallstatus: overallStatus + parentvapp: parentVApp + recenttask: recentTask + resourcepool: resourcePool + rootsnapshot: rootSnapshot + triggeredalarmstate: triggeredAlarmState +filters: +- config.zoo == "DC0_H0_VM0" +hostnames: +- config.foo keyed_groups: -- key: fouo +- key: config.asdf prefix: '' separator: '' plugin: community.vmware.vmware_vm_inventory properties: -- name - ansible_ssh_host - ansible_host - ansible_uuid -- availablefield -- capability -- config -- configissue -- configstatus -- customvalue +- availableField +- configIssue +- configStatus +- customValue - datastore -- effectiverole -- guest -- guestheartbeatstatus +- effectiveRole +- guestHeartbeatStatus - layout -- layoutex +- layoutEx - name - network -- overallstatus -- parentvapp +- overallStatus +- parentVApp - permission -- recenttask -- resourcepool -- rootsnapshot -- runtime +- recentTask +- resourcePool +- rootSnapshot - snapshot +- tag +- triggeredAlarmState +- value +- capability +- config +- guest +- runtime - storage - summary -- tag -- triggeredalarmstate -- value -property_name_format: lower_case strict: false -with_nested_properties: true diff --git a/awx/main/tests/data/inventory/scripts/vmware/files/file_reference b/awx/main/tests/data/inventory/scripts/vmware/files/file_reference index 5282c2db71..56b3c1c920 100644 --- a/awx/main/tests/data/inventory/scripts/vmware/files/file_reference +++ b/awx/main/tests/data/inventory/scripts/vmware/files/file_reference @@ -5,6 +5,7 @@ username = fooo password = fooo server = https://foo.invalid base_source_var = value_of_var -host_filters = {{ config.name == "only_my_server" }},{{ somevar == "bar"}} -groupby_patterns = fouo +alias_pattern = {{ config.foo }} +host_filters = {{ config.zoo == "DC0_H0_VM0" }} +groupby_patterns = {{ config.asdf }} diff --git a/awx/main/tests/functional/test_inventory_source_injectors.py b/awx/main/tests/functional/test_inventory_source_injectors.py index 50510e3c16..c3c70d0c4a 100644 --- a/awx/main/tests/functional/test_inventory_source_injectors.py +++ b/awx/main/tests/functional/test_inventory_source_injectors.py @@ -53,6 +53,9 @@ INI_TEST_VARS = { 'rhv': {}, # there are none 'tower': {}, # there are none 'vmware': { + 'alias_pattern': "{{ config.foo }}", + 'host_filters': '{{ config.zoo == "DC0_H0_VM0" }}', + 'groupby_patterns': "{{ config.asdf }}", # setting VMWARE_VALIDATE_CERTS is duplicated with env var }, 'azure_rm': { From 8c657e210cdca66203f0e24c3920fd284c1a8c55 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Tue, 21 Apr 2020 23:00:15 -0400 Subject: [PATCH 18/19] VMWare collection ready but waiting for release turn off use of VMWare inventory plugin for now Attempt to roll back general migration to Ansible 2.8 failed, kept at 2.9 because of compatibility --- awx/main/models/inventory.py | 33 ++++++++++--------- .../vmware/files/vmware_vm_inventory.yml | 4 +++ 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 1930a89ae6..b0e5e6700f 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -2251,7 +2251,7 @@ class gce(PluginFileInjector): class vmware(PluginFileInjector): plugin_name = 'vmware_vm_inventory' - initial_version = '2.9' # Only tested with collection version + # initial_version = '2.9' # Ready 4/22/2020, waiting for release ini_env_reference = 'VMWARE_INI_PATH' base_injector = 'managed' namespace = 'community' @@ -2271,23 +2271,23 @@ class vmware(PluginFileInjector): "ansible_ssh_host", "ansible_host", "ansible_uuid", - "availableField", # optional? - "configIssue", # optional? - "configStatus", # optional? - "customValue", # optional? + "availableField", + "configIssue", + "configStatus", + "customValue", # optional "datastore", - "effectiveRole", # optional? - "guestHeartbeatStatus", # optonal? - "layout", # optional? - "layoutEx", # optional? + "effectiveRole", + "guestHeartbeatStatus", # optonal + "layout", # optional + "layoutEx", # optional "name", "network", - "overallStatus", # optional? - "parentVApp", # optional? + "overallStatus", + "parentVApp", # optional "permission", - "recentTask", # optional? + "recentTask", "resourcePool", - "rootSnapshot", # optional? + "rootSnapshot", "snapshot", # optional "tag", "triggeredAlarmState", @@ -2302,12 +2302,15 @@ class vmware(PluginFileInjector): "summary", # repeat of other properties ] ret['properties'] = UPPERCASE_PROPS + NESTED_PROPS - ret['compose'] = {} + ret['compose'] = {'ansible_host': 'guest.ipAddress'} # default value + ret['compose']['ansible_ssh_host'] = ret['compose']['ansible_host'] + # the ansible_uuid was unique every host, every import, from the script + ret['compose']['ansible_uuid'] = '99999999 | random | to_uuid' for prop in UPPERCASE_PROPS: if prop == prop.lower(): continue ret['compose'][prop.lower()] = prop - # ret['with_nested_properties'] = True # only dacrystal/topic/vmware-inventory-plugin-enhancements + ret['with_nested_properties'] = True # ret['property_name_format'] = 'lower_case' # only dacrystal/topic/vmware-inventory-plugin-property-format # process custom options diff --git a/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml b/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml index 36796f1779..6363d6c26b 100644 --- a/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml +++ b/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml @@ -1,4 +1,7 @@ compose: + ansible_host: guest.ipAddress + ansible_ssh_host: guest.ipAddress + ansible_uuid: 99999999 | random | to_uuid availablefield: availableField configissue: configIssue configstatus: configStatus @@ -53,3 +56,4 @@ properties: - storage - summary strict: false +with_nested_properties: true From ab703e2a321aad071fc78349499859103e6f6ea6 Mon Sep 17 00:00:00 2001 From: AlanCoding Date: Wed, 22 Apr 2020 21:39:02 -0400 Subject: [PATCH 19/19] Add feature docs and document development tooling --- docs/inventory/inventory_plugins.md | 32 ++++++++----- tools/collections/README.md | 34 ++++++++++++++ tools/collections/clone_vendor.sh | 71 +++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 11 deletions(-) create mode 100644 tools/collections/README.md create mode 100644 tools/collections/clone_vendor.sh diff --git a/docs/inventory/inventory_plugins.md b/docs/inventory/inventory_plugins.md index 00a763b361..01bf66fea6 100644 --- a/docs/inventory/inventory_plugins.md +++ b/docs/inventory/inventory_plugins.md @@ -1,8 +1,12 @@ # Transition to Ansible Inventory Plugins -Inventory updates have changed from using scripts, which are vendored as executable Python scripts, to using dynamically-generated YAML files which conform to the specifications of the `auto` inventory plugin. These are then parsed by their respective inventory plugin. +Inventory updates have changed from using deprecated inventory scripts, to using dynamically-generated YAML files which are parsed by their respective inventory plugin. -The major organizational change is that the inventory plugins are part of the Ansible core distribution, whereas the same logic used to be a part of AWX source. +In Ansible 2.8, the inventory plugins which are part of the Ansible core distribution were used. +This only applied to a few select sources. + +In all other circumstances, inventory imports make use of the inventory plugin from vendored collections. +Those collections are downloaded from Ansible Galaxy at the time of packaging building the container image. ## Prior Background for Transition @@ -14,11 +18,10 @@ AWX used to maintain logic that parsed `.ini` inventory file contents, in additi The CLI entry point `ansible-inventory` was introduced in Ansible 2.4. In Tower 3.2, inventory imports began running this command as an intermediary between the inventory and the import's logic to save content to the database. Using `ansible-inventory` eliminates the need to maintain source-specific logic, relying on Ansible's code instead. This also enables consistent data structure output from `ansible-inventory`. There are many valid structures that a script can provide, but the output from `ansible-inventory` will always be the same, thus the AWX logic to parse the content is simplified. This is why even scripts must be ran through the `ansible-inventory` CLI. -Along with this switchover, a backported version of `ansible-inventory` was provided, which supports Ansible versions 2.2 and 2.3. - ### Removal of Backport +Along with the `ansible-inventory` switchover, a backported version of `ansible-inventory` was provided, which supported Ansible versions 2.2 and 2.3. In AWX 3.0.0 (and Tower 3.5), the backport of `ansible-inventory` was removed, and support for using custom virtual environments was added. This set the minimum version of Ansible necessary to run _any_ inventory update to 2.4. @@ -30,9 +33,21 @@ In AWX 4.0.0 (and Tower 3.5) inventory source types start to switch over to plug To see in which version the plugin transition will happen, see `awx/main/models/inventory.py` and look for the source name as a subclass of `PluginFileInjector`, and there should be an `initial_version`, which is the first version that was deemed (via testing) to have sufficient parity in the content for its inventory plugin returns. For example, `openstack` will begin using the inventory plugin in Ansible version 2.8. If you run an OpenStack inventory update with Ansible 2.7.x or lower, it will use the script. -The eventual goal is for all source types to have moved to plugins. For any given source, after the `initial_version` for plugin use is higher than the lowest supported Ansible version, the script can be removed and the logic for script credential injection will also be removed. +At some point, scripts will be removed and the script-related (for credentials and configuration) logic will also be removed. -For example, after AWX no longer supports Ansible 2.7, the script `awx/plugins/openstack_inventory.py` will be removed. + +### Management of Collections + +Collections are used for inventory imports starting in Ansible 2.9, and each collection has its own versioning independently from Ansible. +Versions for those collections are set in the requirements file `requirements/collections_requirements.yml`. + +The location of vendored collections is set by the file-only setting `INVENTORY_COLLECTIONS_ROOT`. +For development purposes, this can be changed so that you can test against development versions of those collections. +Instructions for doing this are in `tools/collections`. + +If, for some reason, you need to change the version of a particular collection used in inventory imports, +you can use the `ansible-galaxy` tool to update the collection inside of the `INVENTORY_COLLECTIONS_ROOT`. +Note that the logic for building the inventory file is written and tested only for the version pinned in the requirements file. ## Changes to Expect in Imports @@ -54,11 +69,6 @@ More `hostvars` will appear if the inventory plugins are used. To maintain backw A small number of `hostvars` will be lost because of general deprecation needs. -#### Host Names - -In many cases, the host names will change. In all cases, accurate host tracking will still be maintained via the host `instance_id`. - - ## Writing Your Own Inventory File If you do not want any of this compatibility-related functionality, then you can add an SCM inventory source that points to your own file. You can also apply a credential of a `managed_by_tower` type to that inventory source that matches the credential you are using, as long as it is not `gce` or `openstack`. diff --git a/tools/collections/README.md b/tools/collections/README.md new file mode 100644 index 0000000000..932a846014 --- /dev/null +++ b/tools/collections/README.md @@ -0,0 +1,34 @@ +### Inventory Updates Cross-Development with Collections + +Inventory updates in production use vendored collections baked into the image, +which are downloaded from Ansible Galaxy in the build steps. + +This gives instructions to short-circuit that process for a faster development process. + +Running this script will do a `git clone` for all the relevant collections +into the folder `awx/plugins/collections`. + +``` +source tools/collections/clone_vendor.sh +``` + +After this is completed, you must change the path where the server looks +for the vendored inventory collections. +Add this line to your local settings: + +``` +INVENTORY_COLLECTIONS_ROOT = '/awx_devel/awx/plugins/collections' +``` + +Then when you run an inventory update of a particular type, it should +use the cloned collection. +This allows you to cd into a particular collection, add remotes, +change branches, etc. + +#### Extra Build Steps + +This will not work correctly in all circumstances. +Some collections make changes at build-time. + +In particular, the foreman inventory plugin needs the NAME attribute changed to +the fully-qualified collection name, and will fail if this is not done. diff --git a/tools/collections/clone_vendor.sh b/tools/collections/clone_vendor.sh new file mode 100644 index 0000000000..06f6e19956 --- /dev/null +++ b/tools/collections/clone_vendor.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +base_dir=awx/plugins/collections/ansible_collections + +if [ ! -d "$base_dir/azure/azcollection" ] +then + mkdir -p $base_dir/azure + git clone https://github.com/ansible-collections/azure.git $base_dir/azure/azcollection +else + echo "Azure collection already exists" +fi + +if [ ! -d "$base_dir/ansible/amazon" ] +then + mkdir -p $base_dir/ansible + git clone https://github.com/ansible-collections/ansible.amazon.git $base_dir/ansible/amazon +else + echo "Amazon collection already exists" +fi + +if [ ! -d "$base_dir/theforeman/foreman" ] +then + mkdir -p $base_dir/theforeman + git clone https://github.com/theforeman/foreman-ansible-modules.git $base_dir/theforeman/foreman +else + echo "foreman collection already exists" +fi + +if [ ! -d "$base_dir/google/cloud" ] +then + mkdir -p $base_dir/google + git clone https://github.com/ansible-collections/ansible_collections_google.git $base_dir/google/cloud +else + echo "google collection already exists" +fi + +if [ ! -d "$base_dir/openstack/cloud" ] +then + mkdir -p $base_dir/openstack + git clone https://github.com/openstack/ansible-collections-openstack.git $base_dir/openstack/cloud +else + echo "openstack collection already exists" +fi + +if [ ! -d "$base_dir/community/vmware" ] +then + mkdir -p $base_dir/community + git clone https://github.com/ansible-collections/vmware.git $base_dir/community/vmware +else + echo "VMWare collection already exists" +fi + +if [ ! -d "$base_dir/ovirt/ovirt_collection" ] +then + mkdir -p $base_dir/ovirt + git clone $base_dir/ovirt/ovirt_collection +else + echo "Ovirt collection already exists" +fi + +if [ ! -d "$base_dir/awx/awx" ] +then + mkdir -p $base_dir/awx + ln -s $(shell pwd)/awx_collection $base_dir/awx/awx + git clone $base_dir/awx/awx +else + echo "awx collection already exists" +fi + +echo "-- confirmation of what is installed --" +ANSIBLE_COLLECTIONS_PATHS=awx/plugins/collections ansible-galaxy collection list