Merge pull request #7308 from AlanCoding/in_place_everything

Allow for SCM installs of the AWX collection

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

.gitignore

@@ -139,8 +139,8 @@ use_dev_supervisor.txt
 # Ansible module tests
 /awx_collection_test_venv/
 /awx_collection/*.tar.gz
-/awx_collection/galaxy.yml
 /sanity/
+/awx_collection_build/
 
 .idea/*
 *.unison.tmp

Makefile

@@ -401,11 +401,11 @@ symlink_collection:
 
 build_collection:
 	ansible-playbook -i localhost, awx_collection/tools/template_galaxy.yml -e collection_package=$(COLLECTION_PACKAGE) -e collection_namespace=$(COLLECTION_NAMESPACE) -e collection_version=$(VERSION) -e '{"awx_template_version":false}'
-	ansible-galaxy collection build awx_collection --force --output-path=awx_collection
+	ansible-galaxy collection build awx_collection_build --force --output-path=awx_collection_build
 
 install_collection: build_collection
 	rm -rf $(COLLECTION_INSTALL)
-	ansible-galaxy collection install awx_collection/$(COLLECTION_NAMESPACE)-$(COLLECTION_PACKAGE)-$(VERSION).tar.gz
+	ansible-galaxy collection install awx_collection_build/$(COLLECTION_NAMESPACE)-$(COLLECTION_PACKAGE)-$(VERSION).tar.gz
 
 test_collection_sanity: install_collection
 	cd $(COLLECTION_INSTALL) && ansible-test sanity

awx_collection/README.md

@@ -1,7 +1,18 @@
 # AWX Ansible Collection
 
-This Ansible collection allows for easy interaction with an AWX or Ansible Tower
-server via Ansible playbooks.
+[comment]: # (*******************************************************)
+[comment]: # (*                                                     *)
+[comment]: # (*             WARNING                                 *)
+[comment]: # (*                                                     *)
+[comment]: # (*  This file is templated and not to be               *)
+[comment]: # (*  edited directly! Instead modify:                   *)
+[comment]: # (*  tools/roles/template_galaxy/templates/README.md.j2 *)
+[comment]: # (*                                                     *)
+[comment]: # (*  Changes to the base README.md file are refreshed   *)
+[comment]: # (*  upon build of the collection                       *)
+[comment]: # (*******************************************************)
+
+This Ansible collection allows for easy interaction with an AWX server via Ansible playbooks.
 
 This source for this collection lives in the `awx_collection` folder inside of the
 AWX source.
@@ -56,6 +67,7 @@ Notable releases of the `awx.awx` collection:
 
  - 7.0.0 is intended to be identical to the content prior to the migration, aside from changes necessary to function as a collection.
  - 11.0.0 has no non-deprecated modules that depend on the deprecated `tower-cli` [PyPI](https://pypi.org/project/ansible-tower-cli/).
+ - 0.0.1-devel is the version you should see if installing from source, which is intended for development and expected to be unstable.
 
 The following notes are changes that may require changes to playbooks:
 

awx_collection/galaxy.yml

@@ -0,0 +1,38 @@
+# (********************************************************)
+# (*                                                      *)
+# (*             WARNING                                  *)
+# (*                                                      *)
+# (*  This file is managed by Ansible and not to be       *)
+# (*  edited directly! Instead modify:                    *)
+# (*  tools/roles/template_galaxy/templates/galaxy.yml.j2 *)
+# (*                                                      *)
+# (*  Changes to the base galaxy.yml file are refreshed   *)
+# (*  upon build of the collection                        *)
+# (********************************************************)
+---
+authors:
+  - AWX Project Contributors <awx-project@googlegroups.com>
+dependencies: {}
+description: Ansible content that interacts with the AWX or Ansible Tower API.
+documentation: https://github.com/ansible/awx/blob/devel/awx_collection/README.md
+homepage: https://www.ansible.com/
+issues: https://github.com/ansible/awx/issues?q=is%3Aissue+label%3Acomponent%3Aawx_collection
+license:
+  - GPL-3.0-only
+name: awx
+namespace: awx
+readme: README.md
+repository: https://github.com/ansible/awx
+tags:
+  - cloud
+  - infrastructure
+  - awx
+  - ansible
+  - automation
+version: 0.0.1-devel
+build_ignore:
+  - tools
+  - setup.cfg
+  - galaxy.yml.j2
+  - template_galaxy.yml
+  - '*.tar.gz'

awx_collection/plugins/module_utils/tower_api.py

@@ -33,7 +33,7 @@ class ItemNotDefined(Exception):
 
 class TowerModule(AnsibleModule):
     # This gets set by the make process so whatever is in here is irrelevant
-    _COLLECTION_VERSION = "devel"
+    _COLLECTION_VERSION = "0.0.1-devel"
     _COLLECTION_TYPE = "awx"
     # This maps the collections type (awx/tower) to the values returned by the API
     # Those values can be found in awx/api/generics.py line 204

awx_collection/test/awx/conftest.py

@@ -108,7 +108,7 @@ def run_module(request, collection_import):
             sanitize_dict(py_data)
             resp._content = bytes(json.dumps(django_response.data), encoding='utf8')
             resp.status_code = django_response.status_code
-            resp.headers = {'X-API-Product-Name': 'AWX', 'X-API-Product-Version': 'devel'}
+            resp.headers = {'X-API-Product-Name': 'AWX', 'X-API-Product-Version': '0.0.1-devel'}
 
             if request.config.getoption('verbose') > 0:
                 logger.info(

awx_collection/test/awx/test_module_utils.py

@@ -93,9 +93,9 @@ def test_no_templated_values(collection_import):
     checked into source.
     """
     TowerModule = collection_import('plugins.module_utils.tower_api').TowerModule
-    assert TowerModule._COLLECTION_VERSION == "devel", (
+    assert TowerModule._COLLECTION_VERSION == "0.0.1-devel", (
         'The collection version is templated when the collection is built '
-        'and the code should retain the placeholder of "devel".'
+        'and the code should retain the placeholder of "0.0.1-devel".'
     )
     InventoryModule = collection_import('plugins.inventory.tower').InventoryModule
     assert InventoryModule.NAME == 'awx.awx.tower', (

awx_collection/tools/roles/template_galaxy/tasks/main.yml

@@ -45,3 +45,9 @@
     src: "{{ collection_path }}/tools/roles/template_galaxy/templates/galaxy.yml.j2"
     dest: "{{ collection_path }}/galaxy.yml"
     force: true
+
+- name: Template the README.md file
+  template:
+    src: "{{ collection_path }}/tools/roles/template_galaxy/templates/README.md.j2"
+    dest: "{{ collection_path }}/README.md"
+    force: true

awx_collection/tools/roles/template_galaxy/templates/README.md.j2

@@ -0,0 +1,146 @@
+# {% if collection_package | lower() == 'awx' %}AWX{% else %}Tower{% endif %} Ansible Collection
+
+[comment]: # (*******************************************************)
+[comment]: # (*                                                     *)
+[comment]: # (*             WARNING                                 *)
+[comment]: # (*                                                     *)
+[comment]: # (*  This file is templated and not to be               *)
+[comment]: # (*  edited directly! Instead modify:                   *)
+[comment]: # (*  tools/roles/template_galaxy/templates/README.md.j2 *)
+[comment]: # (*                                                     *)
+[comment]: # (*  Changes to the base README.md file are refreshed   *)
+[comment]: # (*  upon build of the collection                       *)
+[comment]: # (*******************************************************)
+
+This Ansible collection allows for easy interaction with an {% if collection_package | lower() == 'awx' %}AWX{% else %}Ansible Tower{% endif %} server via Ansible playbooks.
+
+This source for this collection lives in the `awx_collection` folder inside of the
+AWX source.
+The previous home for this collection was inside the folder [lib/ansible/modules/web_infrastructure/ansible_tower](https://github.com/ansible/ansible/tree/stable-2.9/lib/ansible/modules/web_infrastructure/ansible_tower) in the Ansible repo,
+as well as other places for the inventory plugin, module utils, and
+doc fragment.
+
+## Building and Installing
+
+{% if collection_package | lower() == 'awx' %}
+This collection templates the `galaxy.yml` file it uses.
+Run `make build_collection` from the root folder of the AWX source tree.
+This will create the `tar.gz` file inside the `awx_collection` folder
+with the current AWX version, for example: `awx_collection/awx-awx-9.2.0.tar.gz`.
+
+Installing the `tar.gz` involves no special instructions.
+
+{% else %}
+This collection should be installed from [Content Hub][https://cloud.redhat.com/ansible/automation-hub/ansible/tower/]
+
+{% endif %}
+## Running
+
+Non-deprecated modules in this collection have no Python requirements, but
+may require the official [AWX CLI](https://docs.ansible.com/ansible-tower/latest/html/towercli/index.html)
+in the future. The `DOCUMENTATION` for each module will report this.
+
+You can specify authentication by a combination of either:
+
+ - host, username, password
+ - host, OAuth2 token
+
+The OAuth2 token is the preferred method. You can obtain a token via the
+AWX CLI [login](https://docs.ansible.com/ansible-tower/latest/html/towercli/reference.html#awx-login)
+command.
+
+These can be specified via (from highest to lowest precedence):
+
+ - direct module parameters
+ - environment variables (most useful when running against localhost)
+ - a config file path specified by the `tower_config_file` parameter
+ - a config file at `~/.tower_cli.cfg`
+ - a config file at `/etc/tower/tower_cli.cfg`
+
+Config file syntax looks like this:
+
+```
+[general]
+host = https://localhost:8043
+verify_ssl = true
+oauth_token = LEdCpKVKc4znzffcpQL5vLG8oyeku6
+```
+
+## Release and Upgrade Notes
+
+Notable releases of the `{{ collection_namespace }}.{{ collection_package }}` collection:
+
+{% if collection_package | lower() == "awx" %}
+ - 7.0.0 is intended to be identical to the content prior to the migration, aside from changes necessary to function as a collection.
+ - 11.0.0 has no non-deprecated modules that depend on the deprecated `tower-cli` [PyPI](https://pypi.org/project/ansible-tower-cli/).
+ - 0.0.1-devel is the version you should see if installing from source, which is intended for development and expected to be unstable.
+{% else %}
+ - 3.7.0 initial release
+{% endif %}
+
+The following notes are changes that may require changes to playbooks:
+
+ - When a project is created, it will wait for the update/sync to finish by default; this can be turned off with the `wait` parameter, if desired.
+ - Creating a "scan" type job template is no longer supported.
+ - Specifying a custom certificate via the `TOWER_CERTIFICATE` environment variable no longer works.
+ - Type changes of variable fields:
+
+   - `extra_vars` in the `tower_job_launch` module worked with a `list` previously, but now only works with a `dict` type
+   - `extra_vars` in the `tower_workflow_job_template` module worked with a `string` previously but now expects a `dict`
+   - When the `extra_vars` parameter is used with the `tower_job_launch` module, the launch will fail unless `ask_extra_vars` or `survey_enabled` is explicitly set to `True` on the Job Template
+   - The `variables` parameter in the `tower_group`, `tower_host` and `tower_inventory` modules now expects a `dict` type and no longer supports the use of `@` syntax for a file
+
+
+ - Type changes of other types of fields:
+
+   - `inputs` or `injectors` in the `tower_credential_type` module worked with a string previously but now expects a `dict`
+   - `schema` in the `tower_workflow_job_template` module worked with a `string` previously but not expects a `list` of `dict`s
+
+ - `tower_group` used to also service inventory sources, but this functionality has been removed from this module; use `tower_inventory_source` instead.
+ - Specified `tower_config` file used to handle `k=v` pairs on a single line; this is no longer supported. Please use a file formatted as `yaml`, `json` or `ini` only.
+ - Some return values (e.g., `credential_type`) have been removed. Use of `id` is recommended.
+ - `tower_job_template` no longer supports the deprecated `extra_vars_path` parameter, please use `extra_vars` with the lookup plugin to replace this functionality.
+ - The `notification_configuration` parameter of `tower_notification` has changed from a string to a dict. Please use the `lookup` plugin to read an existing file into a dict.
+ - `tower_credential` no longer supports passing a file name to ssh_key_data.
+ - The HipChat `notification_type` has been removed and can no longer be created using the `tower_notification` module.
+
+{% if collection_package | lower() == "awx" %}
+## Running Unit Tests
+
+Tests to verify compatibility with the most recent AWX code are in `awx_collection/test/awx`.
+These can be ran by `make test_collection` in the development container.
+
+To run outside of the development container, or to run against
+Ansible source, set up a working environment:
+
+```
+mkvirtualenv my_new_venv
+# may need to replace psycopg2 with psycopg2-binary in requirements/requirements.txt
+pip install -r requirements/requirements.txt -r requirements/requirements_dev.txt -r requirements/requirements_git.txt
+make clean-api
+pip install -e <path to your Ansible>
+pip install -e .
+pip install -e awxkit
+py.test awx_collection/test/awx/
+```
+
+## Running Integration Tests
+
+The integration tests require a virtualenv with `ansible` >= 2.9 and `tower_cli`.
+The collection must first be installed, which can be done using `make install_collection`.
+You also need a configuration file, as described in the running section.
+
+Run the tests:
+
+```
+# ansible-test must be run from the directory in which the collection is installed
+cd ~/.ansible/collections/ansible_collections/awx/awx/
+ansible-test integration
+```
+{% endif %}
+
+## Licensing
+
+All content in this folder is licensed under the same license as Ansible,
+which is the same as license that applied before the split into an
+independent collection.

awx_collection/tools/roles/template_galaxy/templates/galaxy.yml.j2

@@ -1,26 +1,38 @@
+# (********************************************************)
+# (*                                                      *)
+# (*             WARNING                                  *)
+# (*                                                      *)
+# (*  This file is managed by Ansible and not to be       *)
+# (*  edited directly! Instead modify:                    *)
+# (*  tools/roles/template_galaxy/templates/galaxy.yml.j2 *)
+# (*                                                      *)
+# (*  Changes to the base galaxy.yml file are refreshed   *)
+# (*  upon build of the collection                        *)
+# (********************************************************)
+---
 authors:
-- AWX Project Contributors <awx-project@googlegroups.com>
+  - AWX Project Contributors <awx-project@googlegroups.com>
 dependencies: {}
 description: Ansible content that interacts with the AWX or Ansible Tower API.
 documentation: https://github.com/ansible/awx/blob/devel/awx_collection/README.md
 homepage: https://www.ansible.com/
 issues: https://github.com/ansible/awx/issues?q=is%3Aissue+label%3Acomponent%3Aawx_collection
 license:
-- GPL-3.0-only
+  - GPL-3.0-only
 name: {{ collection_package }}
 namespace: {{ collection_namespace }}
 readme: README.md
 repository: https://github.com/ansible/awx
 tags:
-- cloud
-- infrastructure
-- awx
-- ansible
-- automation
-version: {{ collection_version }}
+  - cloud
+  - infrastructure
+  - awx
+  - ansible
+  - automation
+version: {{ collection_version_override | default(collection_version) }}
 build_ignore:
-- tools
-- setup.cfg
-- galaxy.yml.j2
-- template_galaxy.yml
-- '*.tar.gz'
+  - tools
+  - setup.cfg
+  - galaxy.yml.j2
+  - template_galaxy.yml
+  - '*.tar.gz'

awx_collection/tools/template_galaxy.yml

@@ -7,6 +7,34 @@
     collection_package: awx
     collection_namespace: awx
     collection_version: 0.0.1  # not for updating, pass in extra_vars
-    collection_path: "{{ playbook_dir }}/../"
+    collection_source: "{{ playbook_dir }}/../"
+    collection_path: "{{ playbook_dir }}/../../awx_collection_build"
+  pre_tasks:
+    - file:
+        path: "{{ collection_path }}"
+        state: absent
+
+    - copy:
+        src: "{{ collection_source }}"
+        dest: "{{ collection_path }}"
+        remote_src: true
+
   roles:
     - template_galaxy
+
+  tasks:
+    - name: Make substitutions in source to sync with templates
+      set_fact:
+        collection_version_override: 0.0.1-devel
+
+    - name: Template the galaxy.yml source file (should be commited with your changes)
+      template:
+        src: "{{ collection_source }}/tools/roles/template_galaxy/templates/galaxy.yml.j2"
+        dest: "{{ collection_source }}/galaxy.yml"
+        force: true
+
+    - name: Template the README.md source file (should be commited with your changes)
+      template:
+        src: "{{ collection_source }}/tools/roles/template_galaxy/templates/README.md.j2"
+        dest: "{{ collection_source }}/README.md"
+        force: true