Integrate Ansible core tower modules content into AWX

This commit includes all the changes involved in
converting the old Ansible Tower modules from commits
in Ansible core into the AWX collection that replaces it.
Also includes work needed to integrate it into the
AWX processes like tests, docs, and the Makefile.

Apply changes from content_collector tool

Add integrated module tests
  operate via run_module fixture
  add makefile target for them

Add flake8 target and fix flake8 errors

Update README

Make consolidated target for testing modules

.gitignore

@@ -134,3 +134,7 @@ venv/*
 use_dev_supervisor.txt
 
 .idea/*
+
+# Ansible module tests
+awx_modules_test_venv/
+awx-awx-*.tar.gz

Makefile

@@ -18,6 +18,7 @@ COMPOSE_TAG ?= $(GIT_BRANCH)
 COMPOSE_HOST ?= $(shell hostname)
 
 VENV_BASE ?= /venv
+MODULES_VENV ?= /awx_devel/awx_modules_test_venv
 SCL_PREFIX ?=
 CELERY_SCHEDULE_FILE ?= /var/lib/awx/beat.db
 
@@ -373,8 +374,31 @@ test:
 	fi; \
 	PYTHONDONTWRITEBYTECODE=1 py.test -p no:cacheprovider -n auto $(TEST_DIRS)
 	cd awxkit && $(VENV_BASE)/awx/bin/tox -re py2,py3
+	make test_modules_all
 	awx-manage check_migrations --dry-run --check  -n 'vNNN_missing_migration_file'
 
+prepare_modules_venv:
+	cd /awx_devel
+	rm -rf $(MODULES_VENV)
+	mkdir $(MODULES_VENV)
+	ln -s /usr/lib/python2.7/site-packages/ansible $(MODULES_VENV)/ansible
+	$(VENV_BASE)/awx/bin/pip install --target=$(MODULES_VENV) git+https://github.com/ansible/tower-cli.git
+
+MODULES_TEST_DIRS ?= awx_modules/test/awx
+
+test_modules:
+	@if [ "$(VENV_BASE)" ]; then \
+		. $(VENV_BASE)/awx/bin/activate; \
+	fi; \
+	PYTHONPATH=$(MODULES_VENV):/awx_devel/awx_modules:$PYTHONPATH py.test $(MODULES_TEST_DIRS)
+
+flake8_modules:
+	flake8 awx_modules/  # Different settings, in main exclude list
+
+prepare_test_modules: prepare_modules_venv test_modules  # deprecated
+
+test_modules_all: prepare_modules_venv test_modules flake8_modules
+
 test_unit:
 	@if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \

awx_modules/README.md

@@ -0,0 +1,52 @@
+# AWX Ansible Modules
+
+These modules allow for easy interaction with an AWX or Ansible Tower server
+in Ansible playbooks.
+
+The previous home for these modules was in https://github.com/ansible/ansible
+inside the folder `lib/ansible/modules/web_infrastructure/ansible_tower`.
+
+## Running
+
+To use these modules, the "old" tower-cli needs to be installed
+in the virtual environment where the modules run.
+You can install it from either:
+
+ - https://github.com/ansible/tower-cli/
+ - https://pypi.org/project/ansible-tower-cli/
+
+To use these modules in AWX, you should create a custom virtual environment
+to install the requirement into. NOTE: you will also probably still need
+to set the job template extra_vars to include `ansible_python_interpreter`
+to be the python in that virtual environment.
+
+## Running Tests
+
+Tests to verify compatibility with the most recent AWX code exist
+in `awx_modules/test/awx`. These tests require that python packages
+are available for all of `awx`, `ansible`, `tower_cli`, and the modules
+themselves.
+
+The target `make prepare_modules_venv` will prepare some requirements
+in the `awx_modules_test_venv` folder so that `make test_modules` can
+be ran to actually run the tests. A single test can be ran via:
+
+```
+make test_modules MODULE_TEST_DIRS=awx_modules/test/awx/test_organization.py
+```
+
+## Building
+
+To build, you should not be in the AWX virtual environment.
+This should work on any machine that has a sufficiently recent version
+of Ansible installed.
+
+```
+cd awx_modules
+ansible-galaxy build
+```
+
+This will leave a tar file in the awx_modules directory.
+
+This process may be amended in the future to template components of `galaxy.yml`
+from values (such as version) taken from AWX.

awx_modules/galaxy.yml

@@ -0,0 +1,16 @@
+authors:
+- Wayne Witzel III (@wwitzel3)
+- Chris Meyers <cmeyers@redhat.com>
+dependencies: {}
+description: Ansible modules that interact with the AWX API.
+documentation: https://docs.ansible.com/ansible/latest/modules/list_of_web_infrastructure_modules.html#ansible-tower
+homepage: https://ansible.com
+issues: https://github.com/ansible/awx/issues
+license:
+- GPL-3.0-only
+name: awx
+namespace: awx
+readme: README.md
+repository: https://github.com/ansible/awx
+tags: []
+version: 0.0.1

awx_modules/plugins/modules/tower_credential.py

@@ -190,7 +190,7 @@ EXAMPLES = '''
 import os
 
 from ansible.module_utils._text import to_text
-from ansible.module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
+from ..module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
 
 try:
     import tower_cli

awx_modules/plugins/modules/tower_credential_type.py

@@ -87,7 +87,7 @@ EXAMPLES = '''
 RETURN = ''' # '''
 
 
-from ansible.module_utils.ansible_tower import (
+from ..module_utils.ansible_tower import (
     TowerModule,
     tower_auth_config,
     tower_check_mode

awx_modules/plugins/modules/tower_group.py

@@ -93,7 +93,7 @@ EXAMPLES = '''
 
 import os
 
-from ansible.module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
+from ..module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
 
 try:
     import tower_cli

awx_modules/plugins/modules/tower_host.py

@@ -66,7 +66,7 @@ EXAMPLES = '''
 
 import os
 
-from ansible.module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode, HAS_TOWER_CLI
+from ..module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
 
 try:
     import tower_cli

awx_modules/plugins/modules/tower_inventory.py

@@ -67,7 +67,7 @@ EXAMPLES = '''
 '''
 
 
-from ansible.module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
+from ..module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
 
 try:
     import tower_cli

awx_modules/plugins/modules/tower_inventory_source.py

@@ -166,7 +166,7 @@ EXAMPLES = '''
 RETURN = ''' # '''
 
 
-from ansible.module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
+from ..module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
 
 try:
     import tower_cli

awx_modules/plugins/modules/tower_job_cancel.py

@@ -55,7 +55,7 @@ status:
 '''
 
 
-from ansible.module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
+from ..module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
 
 try:
     import tower_cli

awx_modules/plugins/modules/tower_job_launch.py

@@ -96,7 +96,7 @@ status:
 '''
 
 
-from ansible.module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
+from ..module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
 
 try:
     import tower_cli

awx_modules/plugins/modules/tower_job_list.py

@@ -78,7 +78,7 @@ results:
 '''
 
 
-from ansible.module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
+from ..module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
 
 try:
     import tower_cli

awx_modules/plugins/modules/tower_job_template.py

@@ -199,7 +199,7 @@ EXAMPLES = '''
     survey_spec: "{{ lookup('file', 'my_survey.json') }}"
 '''
 
-from ansible.module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
+from ..module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
 
 try:
     import tower_cli

awx_modules/plugins/modules/tower_job_wait.py

@@ -82,7 +82,7 @@ status:
 '''
 
 
-from ansible.module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
+from ..module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
 from ansible.module_utils.six.moves import cStringIO as StringIO
 
 
@@ -125,7 +125,7 @@ def main():
         job_id = params.get('job_id')
         try:
             result = job.monitor(job_id, **params)
-        except exc.Timeout as excinfo:
+        except exc.Timeout:
             result = job.status(job_id)
             result['id'] = job_id
             json_output['msg'] = 'Timeout waiting for job to finish.'

awx_modules/plugins/modules/tower_label.py

@@ -49,7 +49,7 @@ EXAMPLES = '''
     tower_config_file: "~/tower_cli.cfg"
 '''
 
-from ansible.module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
+from ..module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
 
 try:
     import tower_cli

awx_modules/plugins/modules/tower_notification.py

@@ -268,7 +268,7 @@ EXAMPLES = '''
 RETURN = ''' # '''
 
 
-from ansible.module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
+from ..module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
 
 try:
     import tower_cli

awx_modules/plugins/modules/tower_organization.py

@@ -48,7 +48,7 @@ EXAMPLES = '''
     tower_config_file: "~/tower_cli.cfg"
 '''
 
-from ansible.module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
+from ..module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
 
 try:
     import tower_cli

awx_modules/plugins/modules/tower_project.py

@@ -110,7 +110,7 @@ EXAMPLES = '''
     tower_config_file: "~/tower_cli.cfg"
 '''
 
-from ansible.module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
+from ..module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
 
 try:
     import tower_cli
@@ -171,7 +171,7 @@ def main():
                 try:
                     org_res = tower_cli.get_resource('organization')
                     org = org_res.get(name=organization)
-                except (exc.NotFound) as excinfo:
+                except exc.NotFound:
                     module.fail_json(msg='Failed to update project, organization not found: {0}'.format(organization), changed=False)
 
                 if scm_credential:
@@ -179,11 +179,11 @@ def main():
                         cred_res = tower_cli.get_resource('credential')
                         try:
                             cred = cred_res.get(name=scm_credential)
-                        except (tower_cli.exceptions.MultipleResults) as multi_res_excinfo:
+                        except tower_cli.exceptions.MultipleResults:
                             module.warn('Multiple credentials found for {0}, falling back looking in project organization'.format(scm_credential))
                             cred = cred_res.get(name=scm_credential, organization=org['id'])
                         scm_credential = cred['id']
-                    except (exc.NotFound) as excinfo:
+                    except exc.NotFound:
                         module.fail_json(msg='Failed to update project, credential not found: {0}'.format(scm_credential), changed=False)
 
                 if (scm_update_cache_timeout is not None) and (scm_update_on_launch is not True):

awx_modules/plugins/modules/tower_receive.py

@@ -109,7 +109,7 @@ assets:
     sample: [ {}, {} ]
 '''
 
-from ansible.module_utils.ansible_tower import TowerModule, tower_auth_config, HAS_TOWER_CLI
+from ..module_utils.ansible_tower import TowerModule, tower_auth_config, HAS_TOWER_CLI
 
 try:
     from tower_cli.cli.transfer.receive import Receiver

awx_modules/plugins/modules/tower_role.py

@@ -72,7 +72,7 @@ EXAMPLES = '''
     tower_config_file: "~/tower_cli.cfg"
 '''
 
-from ansible.module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
+from ..module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
 
 try:
     import tower_cli

awx_modules/plugins/modules/tower_send.py

@@ -76,7 +76,7 @@ import os
 import sys
 
 from ansible.module_utils.six.moves import StringIO
-from ansible.module_utils.ansible_tower import TowerModule, tower_auth_config, HAS_TOWER_CLI
+from ..module_utils.ansible_tower import TowerModule, tower_auth_config, HAS_TOWER_CLI
 
 from tempfile import mkstemp
 
@@ -138,7 +138,7 @@ def main():
             sys.stdout = captured_stdout = StringIO()
             try:
                 sender.send(files, prevent, password_management)
-            except TypeError as e:
+            except TypeError:
                 # Newer versions of TowerCLI require 4 parameters
                 sender.send(files, prevent, [], password_management)
 

awx_modules/plugins/modules/tower_settings.py

@@ -56,7 +56,7 @@ EXAMPLES = '''
   no_log: true
 '''
 
-from ansible.module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
+from ..module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
 
 try:
     import tower_cli

awx_modules/plugins/modules/tower_team.py

@@ -50,7 +50,7 @@ EXAMPLES = '''
     tower_config_file: "~/tower_cli.cfg"
 '''
 
-from ansible.module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
+from ..module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
 
 try:
     import tower_cli

awx_modules/plugins/modules/tower_user.py

@@ -100,7 +100,7 @@ EXAMPLES = '''
     tower_config_file: "~/tower_cli.cfg"
 '''
 
-from ansible.module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
+from ..module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode
 
 try:
     import tower_cli

awx_modules/plugins/modules/tower_workflow_launch.py

@@ -75,7 +75,7 @@ EXAMPLES = '''
   register: workflow_task_info
 '''
 
-from ansible.module_utils.ansible_tower import TowerModule, tower_auth_config
+from ..module_utils.ansible_tower import TowerModule, tower_auth_config
 
 try:
     import tower_cli

awx_modules/plugins/modules/tower_workflow_template.py

@@ -93,8 +93,7 @@ EXAMPLES = '''
 RETURN = ''' # '''
 
 
-from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.ansible_tower import (
+from ..module_utils.ansible_tower import (
     TowerModule,
     tower_auth_config,
     tower_check_mode

awx_modules/setup.cfg

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length=160
+ignore=E402

awx_modules/test/awx/conftest.py

@@ -0,0 +1,121 @@
+import io
+import json
+import datetime
+import importlib
+from contextlib import redirect_stdout
+from unittest import mock
+
+from requests.models import Response
+
+import pytest
+
+from awx.main.tests.functional.conftest import _request
+from awx.main.models import Organization, Project, Inventory, Credential, CredentialType
+
+
+def sanitize_dict(din):
+    '''Sanitize Django response data to purge it of internal types
+    so it may be used to cast a requests response object
+    '''
+    if isinstance(din, (int, str, type(None), bool)):
+        return din  # native JSON types, no problem
+    elif isinstance(din, datetime.datetime):
+        return din.isoformat()
+    elif isinstance(din, list):
+        for i in range(len(din)):
+            din[i] = sanitize_dict(din[i])
+        return din
+    elif isinstance(din, dict):
+        for k in din.copy().keys():
+            din[k] = sanitize_dict(din[k])
+        return din
+    else:
+        return str(din)  # translation proxies often not string but stringlike
+
+
+@pytest.fixture
+def run_module():
+    def rf(module_name, module_params, request_user):
+
+        def new_request(self, method, url, **kwargs):
+            kwargs_copy = kwargs.copy()
+            if 'data' in kwargs:
+                kwargs_copy['data'] = json.loads(kwargs['data'])
+
+            # make request
+            rf = _request(method.lower())
+            django_response = rf(url, user=request_user, expect=None, **kwargs_copy)
+
+            # requests library response object is different from the Django response, but they are the same concept
+            # this converts the Django response object into a requests response object for consumption
+            resp = Response()
+            py_data = django_response.data
+            sanitize_dict(py_data)
+            resp._content = bytes(json.dumps(django_response.data), encoding='utf8')
+            resp.status_code = django_response.status_code
+            return resp
+
+        stdout_buffer = io.StringIO()
+        # Requies specific PYTHONPATH, see docs
+        # Note that a proper Ansiballz explosion of the modules will have an import path like:
+        # ansible_collections.awx.awx.plugins.modules.{}
+        # We should consider supporting that in the future
+        resource_module = importlib.import_module('plugins.modules.{}'.format(module_name))
+
+        # Ansible params can be passed as an invocation argument or over stdin
+        # this short circuits within the AnsibleModule interface
+        def mock_load_params(self):
+            self.params = module_params
+
+        with mock.patch.object(resource_module.TowerModule, '_load_params', new=mock_load_params):
+            # Call the test utility (like a mock server) instead of issuing HTTP requests
+            with mock.patch('tower_cli.api.Session.request', new=new_request):
+                # Ansible modules return data to the mothership over stdout
+                with redirect_stdout(stdout_buffer):
+                    try:
+                        resource_module.main()
+                    except SystemExit:
+                        pass  # A system exit indicates successful execution
+
+        module_stdout = stdout_buffer.getvalue().strip()
+        result = json.loads(module_stdout)
+        return result
+
+    return rf
+
+
+@pytest.fixture
+def organization():
+    return Organization.objects.create(name='Default')
+
+
+@pytest.fixture
+def project(organization):
+    return Project.objects.create(
+        name="test-proj",
+        description="test-proj-desc",
+        organization=organization,
+        playbook_files=['helloworld.yml'],
+        local_path='_92__test_proj',
+        scm_revision='1234567890123456789012345678901234567890',
+        scm_url='localhost',
+        scm_type='git'
+    )
+
+
+@pytest.fixture
+def inventory(organization):
+    return Inventory.objects.create(
+        name='test-inv',
+        organization=organization
+    )
+
+
+@pytest.fixture
+def machine_credential(organization):
+    ssh_type = CredentialType.defaults['ssh']()
+    ssh_type.save()
+    return Credential.objects.create(
+        credential_type=ssh_type, name='machine-cred',
+        inputs={'username': 'test_user', 'password': 'pas4word'}
+    )

awx_modules/test/awx/test_job_template.py

@@ -0,0 +1,59 @@
+import pytest
+
+from awx.main.models import JobTemplate
+
+
+@pytest.mark.django_db
+def test_create_job_template(run_module, admin_user, project, inventory):
+
+    module_args = {
+        'name': 'foo', 'playbook': 'helloworld.yml',
+        'project': project.name, 'inventory': inventory.name,
+        'job_type': 'run',
+        'state': 'present'
+    }
+
+    result = run_module('tower_job_template', module_args, admin_user)
+
+    jt = JobTemplate.objects.get(name='foo')
+
+    assert result == {
+        "job_template": "foo",
+        "state": "present",
+        "id": jt.id,
+        "changed": True,
+        "invocation": {
+            "module_args": module_args
+        }
+    }
+
+    assert jt.project_id == project.id
+    assert jt.inventory_id == inventory.id
+
+
+@pytest.mark.django_db
+@pytest.mark.xfail(reason='Known limitation and needs to be fixed.')
+def test_create_job_template_with_old_machine_cred(run_module, admin_user, project, inventory, machine_credential):
+
+    module_args = {
+        'name': 'foo', 'playbook': 'helloworld.yml',
+        'project': project.name, 'inventory': inventory.name, 'credential': machine_credential.name,
+        'job_type': 'run',
+        'state': 'present'
+    }
+
+    result = run_module('tower_job_template', module_args, admin_user)
+
+    jt = JobTemplate.objects.get(name='foo')
+
+    assert result == {
+        "job_template": "foo",
+        "state": "present",
+        "id": jt.id,
+        "changed": True,
+        "invocation": {
+            "module_args": module_args
+        }
+    }
+
+    assert machine_credential.id in [cred.pk for cred in jt.credentials.all()]

awx_modules/test/awx/test_organization.py

@@ -0,0 +1,25 @@
+import pytest
+
+from awx.main.models import Organization
+
+
+@pytest.mark.django_db
+def test_create_organization(run_module, admin_user):
+
+    module_args = {'name': 'foo', 'description': 'barfoo', 'state': 'present'}
+
+    result = run_module('tower_organization', module_args, admin_user)
+
+    org = Organization.objects.get(name='foo')
+
+    assert result == {
+        "organization": "foo",
+        "state": "present",
+        "id": org.id,
+        "changed": True,
+        "invocation": {
+            "module_args": module_args
+        }
+    }
+
+    assert org.description == 'barfoo'

setup.cfg

@@ -18,4 +18,4 @@ exclude=.tox,venv,awx/lib/site-packages,awx/plugins/inventory/ec2.py,awx/plugins
 [flake8]
 max-line-length=160
 ignore=E201,E203,E221,E225,E231,E241,E251,E261,E265,E303,W291,W391,W293,E731,W504
-exclude=.tox,venv,awx/lib/site-packages,awx/plugins/inventory,awx/ui,awx/api/urls.py,awx/main/migrations,awx/main/tests/data,node_modules/,awx/projects/,tools/docker,awx/settings/local_*.py,installer/openshift/settings.py,build/,installer/,awxkit/test
+exclude=.tox,venv,awx/lib/site-packages,awx/plugins/inventory,awx/ui,awx/api/urls.py,awx/main/migrations,awx/main/tests/data,node_modules/,awx/projects/,tools/docker,awx/settings/local_*.py,installer/openshift/settings.py,build/,installer/,awxkit/test,awx_modules/