Merge pull request #6283 from AlanCoding/vendoring_collections

Use vendored collections for inventory imports

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

tools/collections/README.md

@@ -0,0 +1,34 @@
+### Inventory Updates Cross-Development with Collections
+
+Inventory updates in production use vendored collections baked into the image,
+which are downloaded from Ansible Galaxy in the build steps.
+
+This gives instructions to short-circuit that process for a faster development process.
+
+Running this script will do a `git clone` for all the relevant collections
+into the folder `awx/plugins/collections`.
+
+```
+source tools/collections/clone_vendor.sh
+```
+
+After this is completed, you must change the path where the server looks
+for the vendored inventory collections.
+Add this line to your local settings:
+
+```
+INVENTORY_COLLECTIONS_ROOT = '/awx_devel/awx/plugins/collections'
+```
+
+Then when you run an inventory update of a particular type, it should
+use the cloned collection.
+This allows you to cd into a particular collection, add remotes,
+change branches, etc.
+
+#### Extra Build Steps
+
+This will not work correctly in all circumstances.
+Some collections make changes at build-time.
+
+In particular, the foreman inventory plugin needs the NAME attribute changed to
+the fully-qualified collection name, and will fail if this is not done.

tools/collections/clone_vendor.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+base_dir=awx/plugins/collections/ansible_collections
+
+if [ ! -d "$base_dir/azure/azcollection" ]
+then
+	mkdir -p $base_dir/azure
+	git clone https://github.com/ansible-collections/azure.git $base_dir/azure/azcollection
+else
+  echo "Azure collection already exists"
+fi
+
+if [ ! -d "$base_dir/ansible/amazon" ]
+then
+	mkdir -p $base_dir/ansible
+	git clone https://github.com/ansible-collections/ansible.amazon.git $base_dir/ansible/amazon
+else
+  echo "Amazon collection already exists"
+fi
+
+if [ ! -d "$base_dir/theforeman/foreman" ]
+then
+	mkdir -p $base_dir/theforeman
+	git clone https://github.com/theforeman/foreman-ansible-modules.git $base_dir/theforeman/foreman
+else
+  echo "foreman collection already exists"
+fi
+
+if [ ! -d "$base_dir/google/cloud" ]
+then
+	mkdir -p $base_dir/google
+	git clone https://github.com/ansible-collections/ansible_collections_google.git $base_dir/google/cloud
+else
+  echo "google collection already exists"
+fi
+
+if [ ! -d "$base_dir/openstack/cloud" ]
+then
+	mkdir -p $base_dir/openstack
+	git clone https://github.com/openstack/ansible-collections-openstack.git $base_dir/openstack/cloud
+else
+  echo "openstack collection already exists"
+fi
+
+if [ ! -d "$base_dir/community/vmware" ]
+then
+	mkdir -p $base_dir/community
+	git clone https://github.com/ansible-collections/vmware.git $base_dir/community/vmware
+else
+  echo "VMWare collection already exists"
+fi
+
+if [ ! -d "$base_dir/ovirt/ovirt_collection" ]
+then
+	mkdir -p $base_dir/ovirt
+	git clone  $base_dir/ovirt/ovirt_collection
+else
+  echo "Ovirt collection already exists"
+fi
+
+if [ ! -d "$base_dir/awx/awx" ]
+then
+	mkdir -p $base_dir/awx
+  ln -s $(shell pwd)/awx_collection $base_dir/awx/awx
+	git clone  $base_dir/awx/awx
+else
+  echo "awx collection already exists"
+fi
+
+echo "-- confirmation of what is installed --"
+ANSIBLE_COLLECTIONS_PATHS=awx/plugins/collections ansible-galaxy collection list

tools/docker-compose/Dockerfile

@@ -92,9 +92,12 @@ ADD requirements/requirements.txt \
     requirements/requirements_dev.txt \
     requirements/requirements_ansible_uninstall.txt \
     requirements/requirements_tower_uninstall.txt \
+    requirements/collections_requirements.yml \
     /tmp/requirements/
 RUN mkdir -p /venv && chmod g+w /venv
 RUN cd /tmp && VENV_BASE="/venv" make requirements_dev
+RUN mkdir -p /vendor/inventory_collections && chmod g+w /vendor/inventory_collections
+RUN cd /tmp && COLLECTION_BASE="/vendor/inventory_collections" make requirements_collections
 
 # Use the distro provided npm to bootstrap our required version of node
 RUN npm install -g n && n 10.15.0 && dnf remove -y nodejs