diff --git a/Makefile b/Makefile index cf5c2e15be..2d9371762d 100644 --- a/Makefile +++ b/Makefile @@ -393,7 +393,7 @@ symlink_collection: ln -s $(shell pwd)/awx_collection $(COLLECTION_INSTALL) build_collection: - ansible-playbook -i localhost, awx_collection/template_galaxy.yml -e collection_package=$(COLLECTION_PACKAGE) -e collection_namespace=$(COLLECTION_NAMESPACE) -e collection_version=$(VERSION) + ansible-playbook -i localhost, awx_collection/tools/template_galaxy.yml -e collection_package=$(COLLECTION_PACKAGE) -e collection_namespace=$(COLLECTION_NAMESPACE) -e collection_version=$(VERSION) -e '{"awx_template_version":false}' ansible-galaxy collection build awx_collection --force --output-path=awx_collection install_collection: build_collection diff --git a/awx_collection/plugins/doc_fragments/auth.py b/awx_collection/plugins/doc_fragments/auth.py index 9018f8f26d..bf0ea28832 100644 --- a/awx_collection/plugins/doc_fragments/auth.py +++ b/awx_collection/plugins/doc_fragments/auth.py @@ -39,6 +39,7 @@ options: tower_config_file: description: - Path to the Tower or AWX config file. + - If provided, the other locations for config files will not be considered. type: path notes: diff --git a/awx_collection/plugins/module_utils/tower_api.py b/awx_collection/plugins/module_utils/tower_api.py index 44bd59cebd..7029922a4e 100644 --- a/awx_collection/plugins/module_utils/tower_api.py +++ b/awx_collection/plugins/module_utils/tower_api.py @@ -32,6 +32,15 @@ class ItemNotDefined(Exception): class TowerModule(AnsibleModule): + # This gets set by the make process so whatever is in here is irrelevant + _COLLECTION_VERSION = "devel" + _COLLECTION_TYPE = "awx" + # This maps the collections type (awx/tower) to the values returned by the API + # Those values can be found in awx/api/generics.py line 204 + collection_to_version = { + 'awx': 'AWX', + 'tower': 'Red Hat Ansible Tower', + } url = None honorred_settings = ('host', 'username', 'password', 'verify_ssl', 'oauth_token') host = '127.0.0.1' @@ -45,6 +54,7 @@ class TowerModule(AnsibleModule): authenticated = False config_name = 'tower_cli.cfg' ENCRYPTED_STRING = "$encrypted$" + version_checked = False def __init__(self, argument_spec, **kwargs): args = dict( @@ -104,14 +114,6 @@ class TowerModule(AnsibleModule): local_dir = split(local_dir)[0] config_files.insert(2, join(local_dir, ".{0}".format(self.config_name))) - for config_file in config_files: - if exists(config_file) and not isdir(config_file): - # Only throw a formatting error if the file exists and is not a directory - try: - self.load_config(config_file) - except ConfigFileException: - self.fail_json('The config file {0} is not properly formatted'.format(config_file)) - # If we have a specified tower config, load it if self.params.get('tower_config_file'): duplicated_params = [] @@ -129,6 +131,14 @@ class TowerModule(AnsibleModule): except ConfigFileException as cfe: # Since we were told specifically to load this we want it to fail if we have an error self.fail_json(msg=cfe) + else: + for config_file in config_files: + if exists(config_file) and not isdir(config_file): + # Only throw a formatting error if the file exists and is not a directory + try: + self.load_config(config_file) + except ConfigFileException: + self.fail_json('The config file {0} is not properly formatted'.format(config_file)) def load_config(self, config_path): # Validate the config file is an actual file @@ -374,6 +384,26 @@ class TowerModule(AnsibleModule): finally: self.url = self.url._replace(query=None) + if not self.version_checked: + # In PY2 we get back an HTTPResponse object but PY2 is returning an addinfourl + # First try to get the headers in PY3 format and then drop down to PY2. + try: + tower_type = response.getheader('X-API-Product-Name', None) + tower_version = response.getheader('X-API-Product-Version', None) + except Exception: + tower_type = response.info().getheader('X-API-Product-Name', None) + tower_version = response.info().getheader('X-API-Product-Version', None) + + if self._COLLECTION_TYPE not in self.collection_to_version or self.collection_to_version[self._COLLECTION_TYPE] != tower_type: + self.warn("You are using the {0} version of this collection but connecting to {1}".format( + self._COLLECTION_TYPE, tower_type + )) + elif self._COLLECTION_VERSION != tower_version: + self.warn("You are running collection version {0} but connecting to tower version {1}".format( + self._COLLECTION_VERSION, tower_version + )) + self.version_checked = True + response_body = '' try: response_body = response.read() @@ -434,15 +464,6 @@ class TowerModule(AnsibleModule): # If we have neither of these, then we can try un-authenticated access self.authenticated = True - def default_check_mode(self): - '''Execute check mode logic for Ansible Tower modules''' - if self.check_mode: - try: - result = self.get_endpoint('ping') - self.exit_json(**{'changed': True, 'tower_version': '{0}'.format(result['json']['version'])}) - except(Exception) as excinfo: - self.fail_json(changed=False, msg='Failed check mode: {0}'.format(excinfo)) - def delete_if_needed(self, existing_item, on_delete=None): # This will exit from the module on its own. # If the method successfully deletes an item and on_delete param is defined, diff --git a/awx_collection/template_galaxy.yml b/awx_collection/template_galaxy.yml deleted file mode 100644 index fbfb89c451..0000000000 --- a/awx_collection/template_galaxy.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- -- hosts: localhost - gather_facts: false - connection: local - vars: - collection_package: awx - collection_namespace: awx - collection_version: 0.0.1 # not for updating, pass in extra_vars - - tasks: - - name: Do file content replacements for non-default namespace or package name - block: - - - name: Change module doc_fragments to support desired namespace and package names - replace: - path: "{{ item }}" - regexp: '^extends_documentation_fragment: awx.awx.auth$' - replace: 'extends_documentation_fragment: {{ collection_namespace }}.{{ collection_package }}.auth' - with_fileglob: "{{ playbook_dir }}/plugins/modules/tower_*.py" - loop_control: - label: "{{ item | basename }}" - - - name: Change inventory file to support desired namespace and package names - replace: - path: "{{ playbook_dir }}/plugins/inventory/tower.py" - regexp: "^ NAME = 'awx.awx.tower' # REPLACE$" - replace: " NAME = '{{ collection_namespace }}.{{ collection_package }}.tower' # REPLACE" - when: - - (collection_package != 'awx') or (collection_namespace != 'awx') - - - name: Template the galaxy.yml file - template: - src: "{{ playbook_dir }}/galaxy.yml.j2" - dest: "{{ playbook_dir }}/galaxy.yml" - force: true diff --git a/awx_collection/test/awx/conftest.py b/awx_collection/test/awx/conftest.py index 30e8866320..5b08f220a7 100644 --- a/awx_collection/test/awx/conftest.py +++ b/awx_collection/test/awx/conftest.py @@ -108,6 +108,7 @@ def run_module(request, collection_import): sanitize_dict(py_data) resp._content = bytes(json.dumps(django_response.data), encoding='utf8') resp.status_code = django_response.status_code + resp.headers = {'X-API-Product-Name': 'AWX', 'X-API-Product-Version': 'devel'} if request.config.getoption('verbose') > 0: logger.info( @@ -120,7 +121,11 @@ def run_module(request, collection_import): def new_open(self, method, url, **kwargs): r = new_request(self, method, url, **kwargs) - return mock.MagicMock(read=mock.MagicMock(return_value=r._content), status=r.status_code) + m = mock.MagicMock(read=mock.MagicMock(return_value=r._content), + status=r.status_code, + getheader=mock.MagicMock(side_effect=r.headers.get) + ) + return m stdout_buffer = io.StringIO() # Requies specific PYTHONPATH, see docs @@ -245,7 +250,7 @@ def silence_deprecation(): yield this_mock -@pytest.fixture +@pytest.fixture(autouse=True) def silence_warning(): """Warnings use global variable, same as deprecations.""" with mock.patch('ansible.module_utils.basic.AnsibleModule.warn') as this_mock: diff --git a/awx_collection/test/awx/test_credential.py b/awx_collection/test/awx/test_credential.py index 43619576bb..754133de19 100644 --- a/awx_collection/test/awx/test_credential.py +++ b/awx_collection/test/awx/test_credential.py @@ -152,7 +152,7 @@ def test_make_use_of_custom_credential_type(run_module, organization, admin_user @pytest.mark.django_db -def test_secret_field_write_twice(run_module, organization, admin_user, cred_type, silence_warning): +def test_secret_field_write_twice(run_module, organization, admin_user, cred_type): val1 = '7rEZK38DJl58A7RxA6EC7lLvUHbBQ1' result = run_module('tower_credential', dict( name='Galaxy Token for Steve', diff --git a/awx_collection/test/awx/test_job_template.py b/awx_collection/test/awx/test_job_template.py index 497d93dfcc..4b480ed45d 100644 --- a/awx_collection/test/awx/test_job_template.py +++ b/awx_collection/test/awx/test_job_template.py @@ -163,8 +163,7 @@ def test_job_template_with_survey_encrypted_default(run_module, admin_user, proj silence_warning.assert_called_once_with( "The field survey_spec of job_template {0} has encrypted data and " - "may inaccurately report task is changed.".format(result['id']) - ) + "may inaccurately report task is changed.".format(result['id'])) @pytest.mark.django_db diff --git a/awx_collection/test/awx/test_module_utils.py b/awx_collection/test/awx/test_module_utils.py index c282489490..3c3cdf61c8 100644 --- a/awx_collection/test/awx/test_module_utils.py +++ b/awx_collection/test/awx/test_module_utils.py @@ -1,14 +1,67 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type +import json import sys +from requests.models import Response from unittest import mock -import json + +def getheader(self, header_name, default): + mock_headers = {'X-API-Product-Name': 'not-junk', 'X-API-Product-Version': '1.2.3'} + return mock_headers.get(header_name, default) -def test_duplicate_config(collection_import): +def read(self): + return json.dumps({}) + + +def status(self): + return 200 + + +def mock_ping_response(self, method, url, **kwargs): + r = Response() + r.getheader = getheader.__get__(r) + r.read = read.__get__(r) + r.status = status.__get__(r) + return r + + +def test_version_warning(collection_import, silence_warning): + TowerModule = collection_import('plugins.module_utils.tower_api').TowerModule + cli_data = {'ANSIBLE_MODULE_ARGS': {}} + testargs = ['module_file2.py', json.dumps(cli_data)] + with mock.patch.object(sys, 'argv', testargs): + with mock.patch('ansible.module_utils.urls.Request.open', new=mock_ping_response): + my_module = TowerModule(argument_spec=dict()) + my_module._COLLECTION_VERSION = "1.0.0" + my_module._COLLECTION_TYPE = "not-junk" + my_module.collection_to_version['not-junk'] = 'not-junk' + my_module.get_endpoint('ping') + silence_warning.assert_called_once_with( + 'You are running collection version 1.0.0 but connecting to tower version 1.2.3' + ) + + +def test_type_warning(collection_import, silence_warning): + TowerModule = collection_import('plugins.module_utils.tower_api').TowerModule + cli_data = {'ANSIBLE_MODULE_ARGS': {}} + testargs = ['module_file2.py', json.dumps(cli_data)] + with mock.patch.object(sys, 'argv', testargs): + with mock.patch('ansible.module_utils.urls.Request.open', new=mock_ping_response): + my_module = TowerModule(argument_spec={}) + my_module._COLLECTION_VERSION = "1.2.3" + my_module._COLLECTION_TYPE = "junk" + my_module.collection_to_version['junk'] = 'junk' + my_module.get_endpoint('ping') + silence_warning.assert_called_once_with( + 'You are using the junk version of this collection but connecting to not-junk' + ) + + +def test_duplicate_config(collection_import, silence_warning): # imports done here because of PATH issues unique to this test suite TowerModule = collection_import('plugins.module_utils.tower_api').TowerModule data = { @@ -17,19 +70,42 @@ def test_duplicate_config(collection_import): 'tower_username': 'bob', 'tower_config_file': 'my_config' } + + class DuplicateTestTowerModule(TowerModule): + def load_config(self, config_path): + assert config_path == 'my_config' + + def _load_params(self): + self.params = data + cli_data = {'ANSIBLE_MODULE_ARGS': data} testargs = ['module_file.py', json.dumps(cli_data)] - with mock.patch('ansible.module_utils.basic.AnsibleModule.warn') as mock_warn: - with mock.patch.object(sys, 'argv', testargs): - with mock.patch.object(TowerModule, 'load_config') as mock_load: - argument_spec = dict( - name=dict(required=True), - zig=dict(type='str'), - ) - TowerModule(argument_spec=argument_spec) - mock_load.mock_calls[-1] == mock.call('my_config') - mock_warn.assert_called_once_with( + with mock.patch.object(sys, 'argv', testargs): + argument_spec = dict( + name=dict(required=True), + zig=dict(type='str'), + ) + DuplicateTestTowerModule(argument_spec=argument_spec) + silence_warning.assert_called_once_with( 'The parameter(s) tower_username were provided at the same time as ' 'tower_config_file. Precedence may be unstable, ' 'we suggest either using config file or params.' ) + + +def test_no_templated_values(collection_import): + """This test corresponds to replacements done by + awx_collection/tools/roles/template_galaxy/tasks/main.yml + Those replacements should happen at build time, so they should not be + checked into source. + """ + TowerModule = collection_import('plugins.module_utils.tower_api').TowerModule + assert TowerModule._COLLECTION_VERSION == "devel", ( + 'The collection version is templated when the collection is built ' + 'and the code should retain the placeholder of "devel".' + ) + InventoryModule = collection_import('plugins.inventory.tower').InventoryModule + assert InventoryModule.NAME == 'awx.awx.tower', ( + 'The inventory plugin FQCN is templated when the collection is built ' + 'and the code should retain the default of awx.awx.' + ) diff --git a/awx_collection/test/awx/test_project.py b/awx_collection/test/awx/test_project.py index babe2edf54..9ef1596d3f 100644 --- a/awx_collection/test/awx/test_project.py +++ b/awx_collection/test/awx/test_project.py @@ -3,23 +3,23 @@ __metaclass__ = type import pytest -from unittest import mock - from awx.main.models import Project @pytest.mark.django_db -def test_create_project(run_module, admin_user, organization): - with mock.patch('ansible.module_utils.basic.AnsibleModule.warn') as mock_warn: - result = run_module('tower_project', dict( - name='foo', - organization=organization.name, - scm_type='git', - scm_url='https://foo.invalid', - wait=False, - scm_update_cache_timeout=5 - ), admin_user) - mock_warn.assert_called_once_with('scm_update_cache_timeout will be ignored since scm_update_on_launch was not set to true') +def test_create_project(run_module, admin_user, organization, silence_warning): + result = run_module('tower_project', dict( + name='foo', + organization=organization.name, + scm_type='git', + scm_url='https://foo.invalid', + wait=False, + scm_update_cache_timeout=5 + ), admin_user) + silence_warning.assert_called_once_with( + 'scm_update_cache_timeout will be ignored since scm_update_on_launch ' + 'was not set to true') + assert result.pop('changed', None), result proj = Project.objects.get(name='foo') diff --git a/awx_collection/test/awx/test_send_receive.py b/awx_collection/test/awx/test_send_receive.py index c907e185eb..14f3c89426 100644 --- a/awx_collection/test/awx/test_send_receive.py +++ b/awx_collection/test/awx/test_send_receive.py @@ -17,7 +17,7 @@ from awx.main.models import ( # warns based on password_management param, but not security issue @pytest.mark.django_db -def test_receive_send_jt(run_module, admin_user, mocker, silence_deprecation, silence_warning): +def test_receive_send_jt(run_module, admin_user, mocker, silence_deprecation): org = Organization.objects.create(name='SRtest') proj = Project.objects.create( name='SRtest', diff --git a/awx_collection/test/awx/test_user.py b/awx_collection/test/awx/test_user.py index bb0821ccd2..db705bd5be 100644 --- a/awx_collection/test/awx/test_user.py +++ b/awx_collection/test/awx/test_user.py @@ -43,5 +43,4 @@ def test_password_no_op_warning(run_module, admin_user, mock_auth_stuff, silence silence_warning.assert_called_once_with( "The field password of user {0} has encrypted data and " - "may inaccurately report task is changed.".format(result['id']) - ) + "may inaccurately report task is changed.".format(result['id'])) diff --git a/awx_collection/tools/generate.yml b/awx_collection/tools/generate.yml index a65f64b402..4a592ae87f 100644 --- a/awx_collection/tools/generate.yml +++ b/awx_collection/tools/generate.yml @@ -17,68 +17,5 @@ force_basic_auth: true url_username: "{{ lookup('env', 'TOWER_USERNAME') }}" url_password: "{{ lookup('env', 'TOWER_PASSWORD') }}" - - tasks: - - name: Get date time data - setup: - gather_subset: min - - - name: Create module directory - file: - state: directory - name: "modules" - - - name: Load api/v2 - uri: - method: GET - url: "{{ api_url }}/api/v2/" - register: endpoints - - - name: Load endpoint options - uri: - method: "OPTIONS" - url: "{{ api_url }}{{ item.value }}" - loop: "{{ endpoints['json'] | dict2items }}" - loop_control: - label: "{{ item.key }}" - register: end_point_options - when: "generate_for is not defined or item.key in generate_for" - - - name: Scan POST options for different things - set_fact: - all_options: "{{ all_options | default({}) | combine(options[0]) }}" - loop: "{{ end_point_options.results }}" - vars: - options: "{{ item | json_query('json.actions.POST.[*]') }}" - loop_control: - label: "{{ item['item']['key'] }}" - when: - - item is not skipped - - options is defined - - - name: Process endpoint - template: - src: "templates/tower_module.j2" - dest: "{{ playbook_dir | dirname }}/plugins/modules/{{ file_name }}" - loop: "{{ end_point_options['results'] }}" - loop_control: - label: "{{ item['item']['key'] }}" - when: "'json' in item and 'actions' in item['json'] and 'POST' in item['json']['actions']" - vars: - item_type: "{{ item['item']['key'] }}" - human_readable: "{{ item_type | replace('_', ' ') }}" - singular_item_type: "{{ item['item']['key'] | regex_replace('ies$', 'y') | regex_replace('s$', '') }}" - file_name: "tower_{% if item['item']['key'] in ['settings'] %}{{ item['item']['key'] }}{% else %}{{ singular_item_type }}{% endif %}.py" - type_map: - bool: 'bool' - boolean: 'bool' - choice: 'str' - datetime: 'str' - id: 'str' - int: 'int' - integer: 'int' - json: 'dict' - list: 'list' - object: 'dict' - password: 'str' - string: 'str' + roles: + - generate diff --git a/awx_collection/tools/roles/generate/tasks/main.yml b/awx_collection/tools/roles/generate/tasks/main.yml new file mode 100644 index 0000000000..6a7b4cf673 --- /dev/null +++ b/awx_collection/tools/roles/generate/tasks/main.yml @@ -0,0 +1,64 @@ +--- +- name: Get date time data + setup: + gather_subset: min + +- name: Create module directory + file: + state: directory + name: "modules" + +- name: Load api/v2 + uri: + method: GET + url: "{{ api_url }}/api/v2/" + register: endpoints + +- name: Load endpoint options + uri: + method: "OPTIONS" + url: "{{ api_url }}{{ item.value }}" + loop: "{{ endpoints['json'] | dict2items }}" + loop_control: + label: "{{ item.key }}" + register: end_point_options + when: "generate_for is not defined or item.key in generate_for" + +- name: Scan POST options for different things + set_fact: + all_options: "{{ all_options | default({}) | combine(options[0]) }}" + loop: "{{ end_point_options.results }}" + vars: + options: "{{ item | json_query('json.actions.POST.[*]') }}" + loop_control: + label: "{{ item['item']['key'] }}" + when: + - item is not skipped + - options is defined + +- name: Process endpoint + template: + src: "templates/tower_module.j2" + dest: "{{ playbook_dir | dirname }}/plugins/modules/{{ file_name }}" + loop: "{{ end_point_options['results'] }}" + loop_control: + label: "{{ item['item']['key'] }}" + when: "'json' in item and 'actions' in item['json'] and 'POST' in item['json']['actions']" + vars: + item_type: "{{ item['item']['key'] }}" + human_readable: "{{ item_type | replace('_', ' ') }}" + singular_item_type: "{{ item['item']['key'] | regex_replace('ies$', 'y') | regex_replace('s$', '') }}" + file_name: "tower_{% if item['item']['key'] in ['settings'] %}{{ item['item']['key'] }}{% else %}{{ singular_item_type }}{% endif %}.py" + type_map: + bool: 'bool' + boolean: 'bool' + choice: 'str' + datetime: 'str' + id: 'str' + int: 'int' + integer: 'int' + json: 'dict' + list: 'list' + object: 'dict' + password: 'str' + string: 'str' diff --git a/awx_collection/tools/templates/tower_module.j2 b/awx_collection/tools/roles/generate/templates/tower_module.j2 similarity index 100% rename from awx_collection/tools/templates/tower_module.j2 rename to awx_collection/tools/roles/generate/templates/tower_module.j2 diff --git a/awx_collection/tools/roles/template_galaxy/tasks/main.yml b/awx_collection/tools/roles/template_galaxy/tasks/main.yml new file mode 100644 index 0000000000..c8d6f0fff3 --- /dev/null +++ b/awx_collection/tools/roles/template_galaxy/tasks/main.yml @@ -0,0 +1,40 @@ +--- +- name: Set the collection version in the tower_api.py file + replace: + path: "{{ collection_path }}/plugins/module_utils/tower_api.py" + regexp: '^ _COLLECTION_VERSION = "devel"' + replace: ' _COLLECTION_VERSION = "{{ collection_version }}"' + when: + - "awx_template_version | default(True)" + +- name: Set the collection type in the tower_api.py file + replace: + path: "{{ collection_path }}/plugins/module_utils/tower_api.py" + regexp: '^ _COLLECTION_TYPE = "awx"' + replace: ' _COLLECTION_TYPE = "{{ collection_namespace }}"' + +- name: Do file content replacements for non-default namespace or package name + block: + + - name: Change module doc_fragments to support desired namespace and package names + replace: + path: "{{ item }}" + regexp: '^extends_documentation_fragment: awx.awx.auth$' + replace: 'extends_documentation_fragment: {{ collection_namespace }}.{{ collection_package }}.auth' + with_fileglob: "{{ collection_path }}/plugins/modules/tower_*.py" + loop_control: + label: "{{ item | basename }}" + + - name: Change inventory file to support desired namespace and package names + replace: + path: "{{ collection_path }}/plugins/inventory/tower.py" + regexp: "^ NAME = 'awx.awx.tower' # REPLACE$" + replace: " NAME = '{{ collection_namespace }}.{{ collection_package }}.tower' # REPLACE" + when: + - (collection_package != 'awx') or (collection_namespace != 'awx') + +- name: Template the galaxy.yml file + template: + src: "{{ collection_path }}/tools/roles/template_galaxy/templates/galaxy.yml.j2" + dest: "{{ collection_path }}/galaxy.yml" + force: true diff --git a/awx_collection/galaxy.yml.j2 b/awx_collection/tools/roles/template_galaxy/templates/galaxy.yml.j2 similarity index 100% rename from awx_collection/galaxy.yml.j2 rename to awx_collection/tools/roles/template_galaxy/templates/galaxy.yml.j2 diff --git a/awx_collection/tools/template_galaxy.yml b/awx_collection/tools/template_galaxy.yml new file mode 100644 index 0000000000..78b69cfa5f --- /dev/null +++ b/awx_collection/tools/template_galaxy.yml @@ -0,0 +1,12 @@ +--- +- name: Template the collection galaxy.yml + hosts: localhost + gather_facts: false + connection: local + vars: + collection_package: awx + collection_namespace: awx + collection_version: 0.0.1 # not for updating, pass in extra_vars + collection_path: "{{ playbook_dir }}/../" + roles: + - template_galaxy