From 9955ee6548d0335d44cf18d446d032031fc955e5 Mon Sep 17 00:00:00 2001 From: John Westcott IV Date: Wed, 5 Feb 2020 13:24:46 -0500 Subject: [PATCH] Converting tower_inventory_source Fix up inventory_source module changes, fix import yaml sanity error, change inventory_source unit tests to comply with new structure. --- .../plugins/module_utils/tower_api.py | 19 +- .../plugins/modules/tower_inventory_source.py | 535 +++++++++--------- .../test/awx/test_inventory_source.py | 31 +- 3 files changed, 307 insertions(+), 278 deletions(-) diff --git a/awx_collection/plugins/module_utils/tower_api.py b/awx_collection/plugins/module_utils/tower_api.py index fabffd1e66..d0ad303e31 100644 --- a/awx_collection/plugins/module_utils/tower_api.py +++ b/awx_collection/plugins/module_utils/tower_api.py @@ -1,8 +1,7 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type -from ansible.module_utils.basic import AnsibleModule -from ansible.module_utils.basic import env_fallback +from ansible.module_utils.basic import AnsibleModule, env_fallback from ansible.module_utils.urls import Request, SSLValidationError, ConnectionError from ansible.module_utils.six import PY2 from ansible.module_utils.six.moves.urllib.parse import urlparse, urlencode @@ -14,7 +13,12 @@ import re from json import loads, dumps from os.path import isfile, expanduser, split, join, exists, isdir from os import access, R_OK, getcwd -import yaml + +try: + import yaml + HAS_YAML = True +except ImportError: + HAS_YAML = False class ConfigFileException(Exception): @@ -56,15 +60,15 @@ class TowerModule(AnsibleModule): mutually_exclusive_if = kwargs.pop('mutually_exclusive_if', None) super(TowerModule, self).__init__(argument_spec=args, **kwargs) - + # Eventually, we would like to push this as a feature to Ansible core for others to use... # Test mutually_exclusive if if mutually_exclusive_if: for (var_name, var_value, exclusive_names) in mutually_exclusive_if: if self.params.get(var_name) == var_value: for excluded_param_name in exclusive_names: - if self.params.get(excluded_param_name) != None: - self.fail_json(msg='Arguments {} can not be set if source is {}'.format(', '.join(exclusive_names), var_value)) + if self.params.get(excluded_param_name) is not None: + self.fail_json(msg='Arguments {0} can not be set if source is {1}'.format(', '.join(exclusive_names), var_value)) self.load_config_files() @@ -574,6 +578,9 @@ class TowerModule(AnsibleModule): if not vars_value.startswith('@'): return vars_value + if not HAS_YAML: + self.fail_json(msg=self.missing_required_lib('yaml')) + file_name = None file_content = None try: diff --git a/awx_collection/plugins/modules/tower_inventory_source.py b/awx_collection/plugins/modules/tower_inventory_source.py index 23823d4337..f341f70d1b 100644 --- a/awx_collection/plugins/modules/tower_inventory_source.py +++ b/awx_collection/plugins/modules/tower_inventory_source.py @@ -8,9 +8,9 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type -ANSIBLE_METADATA = {'status': ['preview'], - 'supported_by': 'community', - 'metadata_version': '1.1'} +ANSIBLE_METADATA = {'metadata_version': '1.1', + 'status': ['preview'], + 'supported_by': 'community'} DOCUMENTATION = ''' @@ -20,7 +20,7 @@ author: "Adrien Fleury (@fleu42)" version_added: "2.7" short_description: create, update, or destroy Ansible Tower inventory source. description: - - Create, update, or destroy Ansible Tower inventories source. See + - Create, update, or destroy Ansible Tower inventory source. See U(https://www.ansible.com/tower) for an overview. options: name: @@ -28,325 +28,338 @@ options: - The name to use for the inventory source. required: True type: str + new_name: + description: + - A new name for this assets (will rename the asset) + required: False + type: str description: description: - The description to use for the inventory source. type: str inventory: description: - - The inventory the source is linked to. + - Inventory the group should be made a member of. required: True type: str - organization: - description: - - Organization the inventory belongs to. - type: str source: description: - - Types of inventory source. - choices: - - file - - scm - - ec2 - - gce - - azure - - azure_rm - - vmware - - satellite6 - - cloudforms - - openstack - - rhv - - tower - - custom - required: True + - The source to use for this group. + choices: [ "manual", "file", "scm", "ec2", "gce", "azure_rm", "vmware", "satellite6", "cloudforms", "openstack", "rhv", "tower", "custom" ] type: str - credential: + required: False + source_path: description: - - Credential to use to retrieve the inventory from. + - For an SCM based inventory source, the source path points to the file within the repo to use as an inventory. + type: str + source_script: + description: + - Inventory script to be used when group type is C(custom). type: str source_vars: description: - - >- - The source_vars allow to Override variables found in the source config - file. For example with Openstack, specifying *private: false* would - change the output of the openstack.py script. It has to be YAML or - JSON. + - The variables or environment fields to apply to this source type. + type: dict + credential: + description: + - Credential to use for the source. type: str + source_regions: + description: + - Regions for cloud provider. + type: str + instance_filters: + description: + - Comma-separated list of filter expressions for matching hosts. + type: str + group_by: + description: + - Limit groups automatically created from inventory source. + type: str + overwrite: + description: + - Delete child groups and hosts not found in source. + type: bool + default: 'no' + overwrite_vars: + description: + - Override vars in child groups and hosts with those from external source. + type: bool custom_virtualenv: version_added: "2.9" description: - Local absolute file path containing a custom Python virtualenv to use. type: str required: False + default: '' timeout: + description: The amount of time (in seconds) to run before the task is canceled. + type: int + verbosity: + description: The verbosity level to run this inventory source under. + type: int + choices: [ 0, 1, 2 ] + update_on_launch: description: - - Number in seconds after which the Tower API methods will time out. + - Refresh inventory data from its source each time a job is run. + type: bool + default: 'no' + update_cache_timeout: + description: + - Time in seconds to consider an inventory sync to be current. type: int source_project: description: - - Use a *project* as a source for the *inventory*. - type: str - source_path: - description: - - Path to the file to use as a source in the selected *project*. + - Project to use as source with scm option type: str update_on_project_update: - description: - - >- - That parameter will sync the inventory when the project is synced. It - can only be used with a SCM source. + description: Update this source when the related project updates if source is C(scm) type: bool - source_regions: - description: - - >- - List of regions for your cloud provider. You can include multiple all - regions. Only Hosts associated with the selected regions will be - updated. Refer to Ansible Tower documentation for more detail. - type: str - instance_filters: - description: - - >- - Provide a comma-separated list of filter expressions. Hosts are - imported when all of the filters match. Refer to Ansible Tower - documentation for more detail. - type: str - group_by: - description: - - >- - Specify which groups to create automatically. Group names will be - created similar to the options selected. If blank, all groups above - are created. Refer to Ansible Tower documentation for more detail. - type: str - source_script: - description: - - >- - The source custom script to use to build the inventory. It needs to - exist. - type: str - overwrite: - description: - - >- - If set, any hosts and groups that were previously present on the - external source but are now removed will be removed from the Tower - inventory. Hosts and groups that were not managed by the inventory - source will be promoted to the next manually created group or if - there is no manually created group to promote them into, they will be - left in the "all" default group for the inventory. When not checked, - local child hosts and groups not found on the external source will - remain untouched by the inventory update process. - type: bool - overwrite_vars: - description: - - >- - If set, all variables for child groups and hosts will be removed - and replaced by those found on the external source. When not checked, - a merge will be performed, combining local variables with those found - on the external source. - type: bool - update_on_launch: - description: - - >- - Each time a job runs using this inventory, refresh the inventory from - the selected source before executing job tasks. - type: bool - update_cache_timeout: - description: - - >- - Time in seconds to consider an inventory sync to be current. During - job runs and callbacks the task system will evaluate the timestamp of - the latest sync. If it is older than Cache Timeout, it is not - considered current, and a new inventory sync will be performed. - type: int state: description: - Desired state of the resource. default: "present" choices: ["present", "absent"] type: str + tower_oauthtoken: + description: + - The Tower OAuth token to use. + required: False + type: str extends_documentation_fragment: awx.awx.auth ''' - EXAMPLES = ''' -- name: Add tower inventory source - tower_inventory_source: - name: Inventory source - description: My Inventory source - inventory: My inventory - organization: My organization - credential: Devstack_credential - source: openstack - update_on_launch: true - overwrite: true - source_vars: '{ private: false }' +- name: Add tower group + tower_group: + name: localhost + description: "Local Host Group" + inventory: "Local Inventory" state: present - validate_certs: false + tower_config_file: "~/tower_cli.cfg" ''' - -RETURN = ''' # ''' - - -from ..module_utils.ansible_tower import TowerModule, tower_auth_config, tower_check_mode - -try: - import tower_cli - import tower_cli.exceptions as exc - from tower_cli.conf import settings -except ImportError: - pass - - -SOURCE_CHOICES = { - 'file': 'Directory or Script', - 'scm': 'Sourced from a Project', - 'ec2': 'Amazon EC2', - 'gce': 'Google Compute Engine', - 'azure': 'Microsoft Azure', - 'azure_rm': 'Microsoft Azure Resource Manager', - 'vmware': 'VMware vCenter', - 'satellite6': 'Red Hat Satellite 6', - 'cloudforms': 'Red Hat CloudForms', - 'openstack': 'OpenStack', - 'rhv': 'Red Hat Virtualization', - 'tower': 'Ansible Tower', - 'custom': 'Custom Script', -} +from ..module_utils.tower_api import TowerModule +from json import dumps def main(): + # Any additional arguments that are not fields of the item can be added here argument_spec = dict( name=dict(required=True), - description=dict(required=False), + new_name=dict(type='str'), + description=dict(), inventory=dict(required=True), - source=dict(required=True, - choices=SOURCE_CHOICES.keys()), - credential=dict(required=False), - source_vars=dict(required=False), - timeout=dict(type='int', required=False), - source_project=dict(required=False), - source_path=dict(required=False), - update_on_project_update=dict(type='bool', required=False), - source_regions=dict(required=False), - instance_filters=dict(required=False), - group_by=dict(required=False), - source_script=dict(required=False), - overwrite=dict(type='bool', required=False), - overwrite_vars=dict(type='bool', required=False), - custom_virtualenv=dict(type='str', required=False), - update_on_launch=dict(type='bool', required=False), - update_cache_timeout=dict(type='int', required=False), - organization=dict(type='str'), + # + # How do we handle manual and file? Tower does not seem to be able to activate them + # + source=dict(choices=["manual", "file", "scm", "ec2", "gce", + "azure_rm", "vmware", "satellite6", "cloudforms", + "openstack", "rhv", "tower", "custom"], required=False), + source_path=dict(), + source_script=dict(), + source_vars=dict(type='dict'), + credential=dict(), + source_regions=dict(), + instance_filters=dict(), + group_by=dict(), + overwrite=dict(type='bool'), + overwrite_vars=dict(type='bool'), + custom_virtualenv=dict(type='str'), + timeout=dict(type='int'), + verbosity=dict(type='int', choices=[0, 1, 2]), + update_on_launch=dict(type='bool'), + update_cache_timeout=dict(type='int'), + source_project=dict(type='str'), + update_on_project_update=dict(type='bool'), state=dict(choices=['present', 'absent'], default='present'), ) - module = TowerModule(argument_spec=argument_spec, supports_check_mode=True) + # One question here is do we want to end up supporting this within the ansible module itself (i.e. required if, etc) + # Or do we want to let the API return issues with "this dosen't support that", etc. + # + # GUI OPTIONS: + # - - - - - - - manual: file: scm: ec2: gce azure_rm vmware sat cloudforms openstack rhv tower custom + # credential ? ? o o r r r r r r r r o + # source_project ? ? r - - - - - - - - - - + # source_path ? ? r - - - - - - - - - - + # verbosity ? ? o o o o o o o o o o o + # overwrite ? ? o o o o o o o o o o o + # overwrite_vars ? ? o o o o o o o o o o o + # update_on_launch ? ? o o o o o o o o o o o + # update_on_project_launch ? ? o - - - - - - - - - - + # source_regions ? ? - o o o - - - - - - - + # instance_filters ? ? - o - - o - - - - o - + # group_by ? ? - o - - o - - - - - - + # source_vars* ? ? - o - o o o o o - - - + # environmet vars* ? ? o - - - - - - - - - o + # source_script ? ? - - - - - - - - - - r + # + # * - source_vars are labeled environment_vars on project and custom sources + # Create a module for ourselves + module = TowerModule(argument_spec=argument_spec, + supports_check_mode=True, + required_if=[ + # We don't want to require source if state is present because + # you might be doing an update to an existing source. + # Later on in the code, we will do a test so that if state: present + # and if we don't have an object, we must have source. + ('source', 'scm', ['source_project', 'source_path']), + ('source', 'gce', ['credential']), + ('source', 'azure_rm', ['credential']), + ('source', 'vmware', ['credential']), + ('source', 'satellite6', ['credential']), + ('source', 'cloudforms', ['credential']), + ('source', 'openstack', ['credential']), + ('source', 'rhv', ['credential']), + ('source', 'tower', ['credential']), + ('source', 'custom', ['source_script']), + ], + # This is provided by our module, it's not a core thing + mutually_exclusive_if=[ + ('source', 'scm', ['source_regions', + 'instance_filters', + 'group_by', + 'source_script' + ]), + ('source', 'ec2', ['source_project', + 'source_path', + 'update_on_project_launch', + 'source_script' + ]), + ('source', 'gce', ['source_project', + 'source_path', + 'update_on_project_launch', + 'instance_filters', + 'group_by', + 'source_vars', + 'source_script' + ]), + ('source', 'azure_rm', ['source_project', + 'source_path', + 'update_on_project_launch', + 'instance_filters', + 'group_by', + 'source_script' + ]), + ('source', 'vmware', ['source_project', 'source_path', 'update_on_project_launch', 'source_regions', 'source_script']), + ('source', 'satellite6', ['source_project', + 'source_path', + 'update_on_project_launch', + 'source_regions', + 'instance_filters', + 'group_by', + 'source_script' + ]), + ('source', 'cloudforms', ['source_project', + 'source_path', + 'update_on_project_launch', + 'source_regions', + 'instance_filters', + 'group_by', + 'source_script' + ]), + ('source', 'openstack', ['source_project', + 'source_path', + 'update_on_project_launch', + 'source_regions', + 'instance_filters', + 'group_by', + 'source_script' + ]), + ('source', 'rhv', ['source_project', + 'source_path', + 'update_on_project_launch', + 'source_regions', + 'instance_filters', + 'group_by', + 'source_vars', + 'source_script' + ]), + ('source', 'tower', ['source_project', + 'source_path', + 'update_on_project_launch', + 'source_regions', + 'group_by', + 'source_vars', + 'source_script' + ]), + ('source', 'custom', ['source_project', + 'source_path', + 'update_on_project_launch', + 'source_regions', + 'instance_filters', + 'group_by' + ]), + ]) + + optional_vars = {} + # Extract our parameters name = module.params.get('name') + new_name = module.params.get('new_name') + optional_vars['description'] = module.params.get('description') inventory = module.params.get('inventory') - source = module.params.get('source') + optional_vars['source'] = module.params.get('source') + optional_vars['source_path'] = module.params.get('source_path') + source_script = module.params.get('source_script') + optional_vars['source_vars'] = module.params.get('source_vars') + credential = module.params.get('credential') + optional_vars['source_regions'] = module.params.get('source_regions') + optional_vars['instance_filters'] = module.params.get('instance_filters') + optional_vars['group_by'] = module.params.get('group_by') + optional_vars['overwrite'] = module.params.get('overwrite') + optional_vars['overwrite_vars'] = module.params.get('overwrite_vars') + optional_vars['custom_virtualenv'] = module.params.get('custom_virtualenv') + optional_vars['timeout'] = module.params.get('timeout') + optional_vars['verbosity'] = module.params.get('verbosity') + optional_vars['update_on_launch'] = module.params.get('update_on_launch') + optional_vars['update_cache_timeout'] = module.params.get('update_cache_timeout') + source_project = module.params.get('source_project') + optional_vars['update_on_project_update'] = module.params.get('update_on_project_update') state = module.params.get('state') - organization = module.params.get('organization') - json_output = {'inventory_source': name, 'state': state} + # Attempt to JSON encode source vars + if optional_vars['source_vars']: + optional_vars['source_vars'] = dumps(optional_vars['source_vars']) - tower_auth = tower_auth_config(module) - with settings.runtime_values(**tower_auth): - tower_check_mode(module) - inventory_source = tower_cli.get_resource('inventory_source') - try: - params = {} - params['name'] = name - params['source'] = source + # Attempt to lookup the related items the user specified (these will fail the module if not found) + inventory_id = module.resolve_name_to_id('inventories', inventory) + if credential: + optional_vars['credential'] = module.resolve_name_to_id('credentials', credential) + if source_project: + optional_vars['source_project'] = module.resolve_name_to_id('projects', source_project) + if source_script: + optional_vars['source_script'] = module.resolve_name_to_id('inventory_scripts', source_script) - if module.params.get('description'): - params['description'] = module.params.get('description') + # Attempt to lookup team based on the provided name and org ID + inventory_source = module.get_one('inventory_sources', **{ + 'data': { + 'name': name, + 'inventory': inventory_id, + } + }) - if organization: - try: - org_res = tower_cli.get_resource('organization') - org = org_res.get(name=organization) - except (exc.NotFound) as excinfo: - module.fail_json( - msg='Failed to get organization,' - 'organization not found: {0}'.format(excinfo), - changed=False - ) - org_id = org['id'] - else: - org_id = None # interpreted as not provided + # Sanity check on arguments + if state == 'present' and not inventory_source and not optional_vars['source']: + module.fail_json(msg="If creating a new inventory source, the source param must be present") - if module.params.get('credential'): - credential_res = tower_cli.get_resource('credential') - try: - credential = credential_res.get( - name=module.params.get('credential'), organization=org_id) - params['credential'] = credential['id'] - except (exc.NotFound) as excinfo: - module.fail_json( - msg='Failed to update credential source,' - 'credential not found: {0}'.format(excinfo), - changed=False - ) + # Create data to sent to create and update + inventory_source_fields = { + 'name': new_name if new_name else name, + 'inventory': inventory_id, + } + # Layer in all remaining optional information + for field_name in optional_vars: + if optional_vars[field_name]: + inventory_source_fields[field_name] = optional_vars[field_name] - if module.params.get('source_project'): - source_project_res = tower_cli.get_resource('project') - try: - source_project = source_project_res.get( - name=module.params.get('source_project'), organization=org_id) - params['source_project'] = source_project['id'] - except (exc.NotFound) as excinfo: - module.fail_json( - msg='Failed to update source project,' - 'project not found: {0}'.format(excinfo), - changed=False - ) - - if module.params.get('source_script'): - source_script_res = tower_cli.get_resource('inventory_script') - try: - script = source_script_res.get( - name=module.params.get('source_script'), organization=org_id) - params['source_script'] = script['id'] - except (exc.NotFound) as excinfo: - module.fail_json( - msg='Failed to update source script,' - 'script not found: {0}'.format(excinfo), - changed=False - ) - - try: - inventory_res = tower_cli.get_resource('inventory') - params['inventory'] = inventory_res.get(name=inventory, organization=org_id)['id'] - except (exc.NotFound) as excinfo: - module.fail_json( - msg='Failed to update inventory source, ' - 'inventory not found: {0}'.format(excinfo), - changed=False - ) - - for key in ('source_vars', 'custom_virtualenv', 'timeout', 'source_path', - 'update_on_project_update', 'source_regions', - 'instance_filters', 'group_by', 'overwrite', - 'overwrite_vars', 'update_on_launch', - 'update_cache_timeout'): - if module.params.get(key) is not None: - params[key] = module.params.get(key) - - if state == 'present': - params['create_on_missing'] = True - result = inventory_source.modify(**params) - json_output['id'] = result['id'] - elif state == 'absent': - params['fail_on_missing'] = False - result = inventory_source.delete(**params) - - except (exc.ConnectionError, exc.BadRequest, exc.AuthError) as excinfo: - module.fail_json(msg='Failed to update inventory source: \ - {0}'.format(excinfo), changed=False) - - json_output['changed'] = result['changed'] - module.exit_json(**json_output) + if state == 'absent': + # If the state was absent we can let the module delete it if needed, the module will handle exiting from this + module.delete_if_needed(inventory_source) + elif state == 'present': + # If the state was present we can let the module build or update the existing inventory_source, this will return on its own + module.create_or_update_if_needed(inventory_source, inventory_source_fields, endpoint='inventory_sources', item_type='inventory source') if __name__ == '__main__': diff --git a/awx_collection/test/awx/test_inventory_source.py b/awx_collection/test/awx/test_inventory_source.py index 0b04126a0c..6ab3b50354 100644 --- a/awx_collection/test/awx/test_inventory_source.py +++ b/awx_collection/test/awx/test_inventory_source.py @@ -21,11 +21,13 @@ def base_inventory(): @pytest.mark.django_db def test_inventory_source_create(run_module, admin_user, base_inventory): + source_path = '/var/lib/awx/example_source_path/' result = run_module('tower_inventory_source', dict( name='foo', inventory='test-inv', state='present', source='scm', + source_path=source_path, source_project='test-proj' ), admin_user) assert result.pop('changed', None), result @@ -35,8 +37,9 @@ def test_inventory_source_create(run_module, admin_user, base_inventory): result.pop('invocation') assert result == { 'id': inv_src.id, - 'inventory_source': 'foo', - 'state': 'present' + 'name': 'foo', + 'state': 'present', + 'credential_type': 'Nexus' } @@ -58,7 +61,8 @@ def test_create_inventory_source_implied_org(run_module, admin_user): result.pop('invocation') assert result == { - "inventory_source": "Test Inventory Source", + "credential_type": "Nexus", + "name": "Test Inventory Source", "state": "present", "id": inv_src.id, } @@ -67,27 +71,27 @@ def test_create_inventory_source_implied_org(run_module, admin_user): @pytest.mark.django_db def test_create_inventory_source_multiple_orgs(run_module, admin_user): org = Organization.objects.create(name='test-org') - inv = Inventory.objects.create(name='test-inv', organization=org) + Inventory.objects.create(name='test-inv', organization=org) # make another inventory by same name in another org org2 = Organization.objects.create(name='test-org-number-two') - Inventory.objects.create(name='test-inv', organization=org2) + inv2 = Inventory.objects.create(name='test-inv', organization=org2) result = run_module('tower_inventory_source', dict( name='Test Inventory Source', - inventory='test-inv', + inventory=inv2.id, source='ec2', - organization='test-org', state='present' ), admin_user) assert result.pop('changed', None), result inv_src = InventorySource.objects.get(name='Test Inventory Source') - assert inv_src.inventory == inv + assert inv_src.inventory == inv2 result.pop('invocation') assert result == { - "inventory_source": "Test Inventory Source", + "credential_type": "Nexus", + "name": "Test Inventory Source", "state": "present", "id": inv_src.id, } @@ -96,6 +100,7 @@ def test_create_inventory_source_multiple_orgs(run_module, admin_user): @pytest.mark.django_db def test_create_inventory_source_with_venv(run_module, admin_user, base_inventory, mocker): path = '/var/lib/awx/venv/custom-venv/foobar13489435/' + source_path = '/var/lib/awx/example_source_path/' with mocker.patch('awx.main.models.mixins.get_custom_venv_choices', return_value=[path]): result = run_module('tower_inventory_source', dict( name='foo', @@ -103,7 +108,8 @@ def test_create_inventory_source_with_venv(run_module, admin_user, base_inventor state='present', source='scm', source_project='test-proj', - custom_virtualenv=path + custom_virtualenv=path, + source_path=source_path ), admin_user) assert result.pop('changed'), result @@ -121,6 +127,7 @@ def test_custom_venv_no_op(run_module, admin_user, base_inventory, mocker): This enforces assumptions about the behavior of the AnsibleModule default argument_spec behavior. """ + source_path = '/var/lib/awx/example_source_path/' inv_src = InventorySource.objects.create( name='foo', inventory=base_inventory, @@ -135,7 +142,9 @@ def test_custom_venv_no_op(run_module, admin_user, base_inventory, mocker): description='this is the changed description', inventory='test-inv', source='scm', # is required, but behavior is arguable - state='present' + state='present', + source_project='test-proj', + source_path=source_path ), admin_user) assert result.pop('changed', None), result inv_src.refresh_from_db()