From e11040f4216171d548f7b15a4e932bfcfa8a1b98 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Wed, 15 Jul 2020 12:39:23 -0400 Subject: [PATCH 01/27] migrate to new style inv plugin --- .../migrations/0118_v380_inventory_plugins.py | 59 ++ awx/main/models/inventory.py | 708 +----------------- 2 files changed, 83 insertions(+), 684 deletions(-) create mode 100644 awx/main/migrations/0118_v380_inventory_plugins.py diff --git a/awx/main/migrations/0118_v380_inventory_plugins.py b/awx/main/migrations/0118_v380_inventory_plugins.py new file mode 100644 index 0000000000..e2c8cd2d82 --- /dev/null +++ b/awx/main/migrations/0118_v380_inventory_plugins.py @@ -0,0 +1,59 @@ +# Generated by Django 2.2.11 on 2020-07-20 19:56 + +import logging +import json + +from django.db import migrations, models + +from awx.main.models.inventory import InventorySource +from ._inventory_source_vars import FrozenInjectors + + +logger = logging.getLogger('awx.main.migrations') +BACKUP_FILENAME = '/tmp/tower_migration_inventory_source_vars.json' + + +def _get_inventory_sources(): + # TODO: Maybe pull this list from an import + return InventorySource.objects.filter(source__in=['ec2', 'gce', 'azure_rm', 'vmware', 'satellite6', 'openstack', 'rhv', 'tower']) + + +def inventory_source_vars_forward(apps, schema_editor): + source_vars_backup = dict() + + for inv_source_obj in _get_inventory_sources(): + # TODO: Log error if this is false, it shouldn't be false + if inv_source_obj.source in FrozenInjectors: + source_vars_backup[inv_source_obj.id] = dict(inv_source_obj.source_vars_dict) + with open(BACKUP_FILENAME, 'w') as fh: + json.dump(source_vars_backup, fh) + + injector = FrozenInjectors[inv_source_obj.source]() + new_inv_source_vars = injector.inventory_as_dict(inv_source_obj, None) + inv_source_obj.source_vars = new_inv_source_vars + inv_source_obj.save() + + +def inventory_source_vars_backward(apps, schema_editor): + try: + with open(BACKUP_FILENAME, 'r') as fh: + source_vars_backup = json.load(fh) + except FileNotFoundError: + print(f"Rollback file not found {BACKUP_FILENAME}") + return + + for inv_source_obj in _get_inventory_sources(): + if inv_source_obj.id in source_vars_backup: + inv_source_obj.source_vars = source_vars_backup[inv_source_obj.id] + inv_source_obj.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0117_v400_remove_cloudforms_inventory'), + ] + + operations = [ + migrations.RunPython(inventory_source_vars_forward, inventory_source_vars_backward,), + ] diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 137c056111..b56f3bd03e 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -983,6 +983,24 @@ class InventorySourceOptions(BaseModel): default=1, ) + @classmethod + def get_ec2_group_by_choices(cls): + return [ + ('ami_id', _('Image ID')), + ('availability_zone', _('Availability Zone')), + ('aws_account', _('Account')), + ('instance_id', _('Instance ID')), + ('instance_state', _('Instance State')), + ('platform', _('Platform')), + ('instance_type', _('Instance Type')), + ('key_pair', _('Key Name')), + ('region', _('Region')), + ('security_group', _('Security Group')), + ('tag_keys', _('Tags')), + ('tag_none', _('Tag None')), + ('vpc_id', _('VPC ID')), + ] + @classmethod def get_ec2_region_choices(cls): ec2_region_names = getattr(settings, 'EC2_REGION_NAMES', {}) @@ -1005,24 +1023,6 @@ class InventorySourceOptions(BaseModel): regions.append((region.name, label)) return sorted(regions, key=region_sorting) - @classmethod - def get_ec2_group_by_choices(cls): - return [ - ('ami_id', _('Image ID')), - ('availability_zone', _('Availability Zone')), - ('aws_account', _('Account')), - ('instance_id', _('Instance ID')), - ('instance_state', _('Instance State')), - ('platform', _('Platform')), - ('instance_type', _('Instance Type')), - ('key_pair', _('Key Name')), - ('region', _('Region')), - ('security_group', _('Security Group')), - ('tag_keys', _('Tags')), - ('tag_none', _('Tag None')), - ('vpc_id', _('VPC ID')), - ] - @classmethod def get_gce_region_choices(self): """Return a complete list of regions in GCE, as a list of @@ -1613,26 +1613,18 @@ class PluginFileInjector(object): """ return '{0}.yml'.format(self.plugin_name) - def inventory_as_dict(self, inventory_update, private_data_dir): - """Default implementation of inventory plugin file contents. - There are some valid cases when all parameters can be obtained from - the environment variables, example "plugin: linode" is valid - ideally, however, some options should be filled from the inventory source data - """ - if self.plugin_name is None: - raise NotImplementedError('At minimum the plugin name is needed for inventory plugin use.') - proper_name = f'{self.namespace}.{self.collection}.{self.plugin_name}' - return {'plugin': proper_name} - def inventory_contents(self, inventory_update, private_data_dir): """Returns a string that is the content for the inventory file for the inventory plugin """ return yaml.safe_dump( - self.inventory_as_dict(inventory_update, private_data_dir), + inventory_update.source_vars_dict, default_flow_style=False, width=1000 ) + def inventory_as_dict(self, inventory_update, private_data_dir): + return inventory_update.source_vars_dict + def build_env(self, inventory_update, env, private_data_dir, private_data_files): injector_env = self.get_plugin_env(inventory_update, private_data_dir, private_data_files) env.update(injector_env) @@ -1690,106 +1682,6 @@ class azure_rm(PluginFileInjector): ret['ANSIBLE_JINJA2_NATIVE'] = str(True) return ret - def inventory_as_dict(self, inventory_update, private_data_dir): - ret = super(azure_rm, self).inventory_as_dict(inventory_update, private_data_dir) - - source_vars = inventory_update.source_vars_dict - - ret['fail_on_template_errors'] = False - - group_by_hostvar = { - 'location': {'prefix': '', 'separator': '', 'key': 'location'}, - 'tag': {'prefix': '', 'separator': '', 'key': 'tags.keys() | list if tags else []'}, - # Introduced with https://github.com/ansible/ansible/pull/53046 - 'security_group': {'prefix': '', 'separator': '', 'key': 'security_group'}, - 'resource_group': {'prefix': '', 'separator': '', 'key': 'resource_group'}, - # Note, os_family was not documented correctly in script, but defaulted to grouping by it - 'os_family': {'prefix': '', 'separator': '', 'key': 'os_disk.operating_system_type'} - } - # by default group by everything - # always respect user setting, if they gave it - group_by = [ - grouping_name for grouping_name in group_by_hostvar - if source_vars.get('group_by_{}'.format(grouping_name), True) - ] - ret['keyed_groups'] = [group_by_hostvar[grouping_name] for grouping_name in group_by] - if 'tag' in group_by: - # Nasty syntax to reproduce "key_value" group names in addition to "key" - ret['keyed_groups'].append({ - 'prefix': '', 'separator': '', - 'key': r'dict(tags.keys() | map("regex_replace", "^(.*)$", "\1_") | list | zip(tags.values() | list)) if tags else []' - }) - - # Compatibility content - # TODO: add proper support for instance_filters non-specific to compatibility - # TODO: add proper support for group_by non-specific to compatibility - # Dashes were not configurable in azure_rm.py script, we do not want unicode, so always use this - ret['use_contrib_script_compatible_sanitization'] = True - # use same host names as script - ret['plain_host_names'] = True - # By default the script did not filter hosts - ret['default_host_filters'] = [] - # User-given host filters - user_filters = [] - old_filterables = [ - ('resource_groups', 'resource_group'), - ('tags', 'tags') - # locations / location would be an entry - # but this would conflict with source_regions - ] - for key, loc in old_filterables: - value = source_vars.get(key, None) - if value and isinstance(value, str): - # tags can be list of key:value pairs - # e.g. 'Creator:jmarshall, peanutbutter:jelly' - # or tags can be a list of keys - # e.g. 'Creator, peanutbutter' - if key == "tags": - # grab each key value pair - for kvpair in value.split(','): - # split into key and value - kv = kvpair.split(':') - # filter out any host that does not have key - # in their tags.keys() variable - user_filters.append('"{}" not in tags.keys()'.format(kv[0].strip())) - # if a value is provided, check that the key:value pair matches - if len(kv) > 1: - user_filters.append('tags["{}"] != "{}"'.format(kv[0].strip(), kv[1].strip())) - else: - user_filters.append('{} not in {}'.format( - loc, value.split(',') - )) - if user_filters: - ret.setdefault('exclude_host_filters', []) - ret['exclude_host_filters'].extend(user_filters) - - ret['conditional_groups'] = {'azure': True} - ret['hostvar_expressions'] = { - 'provisioning_state': 'provisioning_state | title', - 'computer_name': 'name', - 'type': 'resource_type', - 'private_ip': 'private_ipv4_addresses[0] if private_ipv4_addresses else None', - 'public_ip': 'public_ipv4_addresses[0] if public_ipv4_addresses else None', - 'public_ip_name': 'public_ip_name if public_ip_name is defined else None', - 'public_ip_id': 'public_ip_id if public_ip_id is defined else None', - 'tags': 'tags if tags else None' - } - # Special functionality from script - if source_vars.get('use_private_ip', False): - ret['hostvar_expressions']['ansible_host'] = 'private_ipv4_addresses[0]' - # end compatibility content - - if inventory_update.source_regions and 'all' not in inventory_update.source_regions: - # initialize a list for this section in inventory file - ret.setdefault('exclude_host_filters', []) - # make a python list of the regions we will use - python_regions = [x.strip() for x in inventory_update.source_regions.split(',')] - # convert that list in memory to python syntax in a string - # now put that in jinja2 syntax operating on hostvar key "location" - # and put that as an entry in the exclusions list - ret['exclude_host_filters'].append("location not in {}".format(repr(python_regions))) - return ret - class ec2(PluginFileInjector): plugin_name = 'aws_ec2' @@ -1803,218 +1695,6 @@ class ec2(PluginFileInjector): ret['ANSIBLE_JINJA2_NATIVE'] = str(True) return ret - def _compat_compose_vars(self): - return { - # vars that change - 'ec2_block_devices': ( - "dict(block_device_mappings | map(attribute='device_name') | list | zip(block_device_mappings " - "| map(attribute='ebs.volume_id') | list))" - ), - 'ec2_dns_name': 'public_dns_name', - 'ec2_group_name': 'placement.group_name', - 'ec2_instance_profile': 'iam_instance_profile | default("")', - 'ec2_ip_address': 'public_ip_address', - 'ec2_kernel': 'kernel_id | default("")', - 'ec2_monitored': "monitoring.state in ['enabled', 'pending']", - 'ec2_monitoring_state': 'monitoring.state', - 'ec2_placement': 'placement.availability_zone', - 'ec2_ramdisk': 'ramdisk_id | default("")', - 'ec2_reason': 'state_transition_reason', - 'ec2_security_group_ids': "security_groups | map(attribute='group_id') | list | join(',')", - 'ec2_security_group_names': "security_groups | map(attribute='group_name') | list | join(',')", - 'ec2_tag_Name': 'tags.Name', - 'ec2_state': 'state.name', - 'ec2_state_code': 'state.code', - 'ec2_state_reason': 'state_reason.message if state_reason is defined else ""', - 'ec2_sourceDestCheck': 'source_dest_check | default(false) | lower | string', # snake_case syntax intended - 'ec2_account_id': 'owner_id', - # vars that just need ec2_ prefix - 'ec2_ami_launch_index': 'ami_launch_index | string', - 'ec2_architecture': 'architecture', - 'ec2_client_token': 'client_token', - 'ec2_ebs_optimized': 'ebs_optimized', - 'ec2_hypervisor': 'hypervisor', - 'ec2_image_id': 'image_id', - 'ec2_instance_type': 'instance_type', - 'ec2_key_name': 'key_name', - 'ec2_launch_time': r'launch_time | regex_replace(" ", "T") | regex_replace("(\+)(\d\d):(\d)(\d)$", ".\g<2>\g<3>Z")', - 'ec2_platform': 'platform | default("")', - 'ec2_private_dns_name': 'private_dns_name', - 'ec2_private_ip_address': 'private_ip_address', - 'ec2_public_dns_name': 'public_dns_name', - 'ec2_region': 'placement.region', - 'ec2_root_device_name': 'root_device_name', - 'ec2_root_device_type': 'root_device_type', - # many items need blank defaults because the script tended to keep a common schema - 'ec2_spot_instance_request_id': 'spot_instance_request_id | default("")', - 'ec2_subnet_id': 'subnet_id | default("")', - 'ec2_virtualization_type': 'virtualization_type', - 'ec2_vpc_id': 'vpc_id | default("")', - # same as ec2_ip_address, the script provided this - 'ansible_host': 'public_ip_address', - # new with https://github.com/ansible/ansible/pull/53645 - 'ec2_eventsSet': 'events | default("")', - 'ec2_persistent': 'persistent | default(false)', - 'ec2_requester_id': 'requester_id | default("")' - } - - def inventory_as_dict(self, inventory_update, private_data_dir): - ret = super(ec2, self).inventory_as_dict(inventory_update, private_data_dir) - - keyed_groups = [] - group_by_hostvar = { - 'ami_id': {'prefix': '', 'separator': '', 'key': 'image_id', 'parent_group': 'images'}, - # 2 entries for zones for same groups to establish 2 parentage trees - 'availability_zone': {'prefix': '', 'separator': '', 'key': 'placement.availability_zone', 'parent_group': 'zones'}, - 'aws_account': {'prefix': '', 'separator': '', 'key': 'ec2_account_id', 'parent_group': 'accounts'}, # composed var - 'instance_id': {'prefix': '', 'separator': '', 'key': 'instance_id', 'parent_group': 'instances'}, # normally turned off - 'instance_state': {'prefix': 'instance_state', 'key': 'ec2_state', 'parent_group': 'instance_states'}, # composed var - # ec2_platform is a composed var, but group names do not match up to hostvar exactly - 'platform': {'prefix': 'platform', 'key': 'platform | default("undefined")', 'parent_group': 'platforms'}, - 'instance_type': {'prefix': 'type', 'key': 'instance_type', 'parent_group': 'types'}, - 'key_pair': {'prefix': 'key', 'key': 'key_name', 'parent_group': 'keys'}, - 'region': {'prefix': '', 'separator': '', 'key': 'placement.region', 'parent_group': 'regions'}, - # Security requires some ninja jinja2 syntax, credit to s-hertel - 'security_group': {'prefix': 'security_group', 'key': 'security_groups | map(attribute="group_name")', 'parent_group': 'security_groups'}, - # tags cannot be parented in exactly the same way as the script due to - # https://github.com/ansible/ansible/pull/53812 - 'tag_keys': [ - {'prefix': 'tag', 'key': 'tags', 'parent_group': 'tags'}, - {'prefix': 'tag', 'key': 'tags.keys()', 'parent_group': 'tags'} - ], - # 'tag_none': None, # grouping by no tags isn't a different thing with plugin - # naming is redundant, like vpc_id_vpc_8c412cea, but intended - 'vpc_id': {'prefix': 'vpc_id', 'key': 'vpc_id', 'parent_group': 'vpcs'}, - } - # -- same-ish as script here -- - group_by = [x.strip().lower() for x in inventory_update.group_by.split(',') if x.strip()] - for choice in inventory_update.get_ec2_group_by_choices(): - value = bool((group_by and choice[0] in group_by) or (not group_by and choice[0] != 'instance_id')) - # -- end sameness to script -- - if value: - this_keyed_group = group_by_hostvar.get(choice[0], None) - # If a keyed group syntax does not exist, there is nothing we can do to get this group - if this_keyed_group is not None: - if isinstance(this_keyed_group, list): - keyed_groups.extend(this_keyed_group) - else: - keyed_groups.append(this_keyed_group) - # special case, this parentage is only added if both zones and regions are present - if not group_by or ('region' in group_by and 'availability_zone' in group_by): - keyed_groups.append({'prefix': '', 'separator': '', 'key': 'placement.availability_zone', 'parent_group': '{{ placement.region }}'}) - - source_vars = inventory_update.source_vars_dict - # This is a setting from the script, hopefully no one used it - # if true, it replaces dashes, but not in region / loc names - replace_dash = bool(source_vars.get('replace_dash_in_groups', True)) - # Compatibility content - legacy_regex = { - True: r"[^A-Za-z0-9\_]", - False: r"[^A-Za-z0-9\_\-]" # do not replace dash, dash is allowed - }[replace_dash] - list_replacer = 'map("regex_replace", "{rx}", "_") | list'.format(rx=legacy_regex) - # this option, a plugin option, will allow dashes, but not unicode - # when set to False, unicode will be allowed, but it was not allowed by script - # thus, we always have to use this option, and always use our custom regex - ret['use_contrib_script_compatible_sanitization'] = True - for grouping_data in keyed_groups: - if grouping_data['key'] in ('placement.region', 'placement.availability_zone'): - # us-east-2 is always us-east-2 according to ec2.py - # no sanitization in region-ish groups for the script standards, ever ever - continue - if grouping_data['key'] == 'tags': - # dict jinja2 transformation - grouping_data['key'] = 'dict(tags.keys() | {replacer} | zip(tags.values() | {replacer}))'.format( - replacer=list_replacer - ) - elif grouping_data['key'] == 'tags.keys()' or grouping_data['prefix'] == 'security_group': - # list jinja2 transformation - grouping_data['key'] += ' | {replacer}'.format(replacer=list_replacer) - else: - # string transformation - grouping_data['key'] += ' | regex_replace("{rx}", "_")'.format(rx=legacy_regex) - # end compatibility content - - if source_vars.get('iam_role_arn', None): - ret['iam_role_arn'] = source_vars['iam_role_arn'] - - # This was an allowed ec2.ini option, also plugin option, so pass through - if source_vars.get('boto_profile', None): - ret['boto_profile'] = source_vars['boto_profile'] - - elif not replace_dash: - # Using the plugin, but still want dashes allowed - ret['use_contrib_script_compatible_sanitization'] = True - - if source_vars.get('nested_groups') is False: - for this_keyed_group in keyed_groups: - this_keyed_group.pop('parent_group', None) - - if keyed_groups: - ret['keyed_groups'] = keyed_groups - - # Instance ID not part of compat vars, because of settings.EC2_INSTANCE_ID_VAR - compose_dict = {'ec2_id': 'instance_id'} - inst_filters = {} - - # Compatibility content - compose_dict.update(self._compat_compose_vars()) - # plugin provides "aws_ec2", but not this which the script gave - ret['groups'] = {'ec2': True} - if source_vars.get('hostname_variable') is not None: - hnames = [] - for expr in source_vars.get('hostname_variable').split(','): - if expr == 'public_dns_name': - hnames.append('dns-name') - elif not expr.startswith('tag:') and '_' in expr: - hnames.append(expr.replace('_', '-')) - else: - hnames.append(expr) - ret['hostnames'] = hnames - else: - # public_ip as hostname is non-default plugin behavior, script behavior - ret['hostnames'] = [ - 'network-interface.addresses.association.public-ip', - 'dns-name', - 'private-dns-name' - ] - # The script returned only running state by default, the plugin does not - # https://docs.aws.amazon.com/cli/latest/reference/ec2/describe-instances.html#options - # options: pending | running | shutting-down | terminated | stopping | stopped - inst_filters['instance-state-name'] = ['running'] - # end compatibility content - - if source_vars.get('destination_variable') or source_vars.get('vpc_destination_variable'): - for fd in ('destination_variable', 'vpc_destination_variable'): - if source_vars.get(fd): - compose_dict['ansible_host'] = source_vars.get(fd) - break - - if compose_dict: - ret['compose'] = compose_dict - - if inventory_update.instance_filters: - # logic used to live in ec2.py, now it belongs to us. Yay more code? - filter_sets = [f for f in inventory_update.instance_filters.split(',') if f] - - for instance_filter in filter_sets: - # AND logic not supported, unclear how to... - instance_filter = instance_filter.strip() - if not instance_filter or '=' not in instance_filter: - continue - filter_key, filter_value = [x.strip() for x in instance_filter.split('=', 1)] - if not filter_key: - continue - inst_filters[filter_key] = filter_value - - if inst_filters: - ret['filters'] = inst_filters - - if inventory_update.source_regions and 'all' not in inventory_update.source_regions: - ret['regions'] = inventory_update.source_regions.split(',') - - return ret - class gce(PluginFileInjector): plugin_name = 'gcp_compute' @@ -2028,76 +1708,10 @@ class gce(PluginFileInjector): ret['ANSIBLE_JINJA2_NATIVE'] = str(True) return ret - def _compat_compose_vars(self): - # missing: gce_image, gce_uuid - # https://github.com/ansible/ansible/issues/51884 - return { - 'gce_description': 'description if description else None', - 'gce_machine_type': 'machineType', - 'gce_name': 'name', - 'gce_network': 'networkInterfaces[0].network.name', - 'gce_private_ip': 'networkInterfaces[0].networkIP', - 'gce_public_ip': 'networkInterfaces[0].accessConfigs[0].natIP | default(None)', - 'gce_status': 'status', - 'gce_subnetwork': 'networkInterfaces[0].subnetwork.name', - 'gce_tags': 'tags.get("items", [])', - 'gce_zone': 'zone', - 'gce_metadata': 'metadata.get("items", []) | items2dict(key_name="key", value_name="value")', - # NOTE: image hostvar is enabled via retrieve_image_info option - 'gce_image': 'image', - # We need this as long as hostnames is non-default, otherwise hosts - # will not be addressed correctly, was returned in script - 'ansible_ssh_host': 'networkInterfaces[0].accessConfigs[0].natIP | default(networkInterfaces[0].networkIP)' - } - def inventory_as_dict(self, inventory_update, private_data_dir): - ret = super(gce, self).inventory_as_dict(inventory_update, private_data_dir) - credential = inventory_update.get_cloud_credential() - - # auth related items + ret = super().inventory_as_dict(inventory_update, private_data_dir) + credential = inventory_source.get_cloud_credential() ret['projects'] = [credential.get_input('project', default='')] - ret['auth_kind'] = "serviceaccount" - - filters = [] - # TODO: implement gce group_by options - # gce never processed the group_by field, if it had, we would selectively - # apply those options here, but it did not, so all groups are added here - keyed_groups = [ - # the jinja2 syntax is duplicated with compose - # https://github.com/ansible/ansible/issues/51883 - {'prefix': 'network', 'key': 'gce_subnetwork'}, # composed var - {'prefix': '', 'separator': '', 'key': 'gce_private_ip'}, # composed var - {'prefix': '', 'separator': '', 'key': 'gce_public_ip'}, # composed var - {'prefix': '', 'separator': '', 'key': 'machineType'}, - {'prefix': '', 'separator': '', 'key': 'zone'}, - {'prefix': 'tag', 'key': 'gce_tags'}, # composed var - {'prefix': 'status', 'key': 'status | lower'}, - # NOTE: image hostvar is enabled via retrieve_image_info option - {'prefix': '', 'separator': '', 'key': 'image'}, - ] - # This will be used as the gce instance_id, must be universal, non-compat - compose_dict = {'gce_id': 'id'} - - # Compatibility content - # TODO: proper group_by and instance_filters support, irrelevant of compat mode - # The gce.py script never sanitized any names in any way - ret['use_contrib_script_compatible_sanitization'] = True - # Perform extra API query to get the image hostvar - ret['retrieve_image_info'] = True - # Add in old hostvars aliases - compose_dict.update(self._compat_compose_vars()) - # Non-default names to match script - ret['hostnames'] = ['name', 'public_ip', 'private_ip'] - # end compatibility content - - if keyed_groups: - ret['keyed_groups'] = keyed_groups - if filters: - ret['filters'] = filters - if compose_dict: - ret['compose'] = compose_dict - if inventory_update.source_regions and 'all' not in inventory_update.source_regions: - ret['zones'] = inventory_update.source_regions.split(',') return ret @@ -2107,106 +1721,6 @@ class vmware(PluginFileInjector): namespace = 'community' collection = 'vmware' - def inventory_as_dict(self, inventory_update, private_data_dir): - ret = super(vmware, self).inventory_as_dict(inventory_update, private_data_dir) - ret['strict'] = False - # Documentation of props, see - # https://github.com/ansible/ansible/blob/devel/docs/docsite/rst/scenario_guides/vmware_scenarios/vmware_inventory_vm_attributes.rst - UPPERCASE_PROPS = [ - "availableField", - "configIssue", - "configStatus", - "customValue", # optional - "datastore", - "effectiveRole", - "guestHeartbeatStatus", # optional - "layout", # optional - "layoutEx", # optional - "name", - "network", - "overallStatus", - "parentVApp", # optional - "permission", - "recentTask", - "resourcePool", - "rootSnapshot", - "snapshot", # optional - "triggeredAlarmState", - "value" - ] - NESTED_PROPS = [ - "capability", - "config", - "guest", - "runtime", - "storage", - "summary", # repeat of other properties - ] - ret['properties'] = UPPERCASE_PROPS + NESTED_PROPS - ret['compose'] = {'ansible_host': 'guest.ipAddress'} # default value - ret['compose']['ansible_ssh_host'] = ret['compose']['ansible_host'] - # the ansible_uuid was unique every host, every import, from the script - ret['compose']['ansible_uuid'] = '99999999 | random | to_uuid' - for prop in UPPERCASE_PROPS: - if prop == prop.lower(): - continue - ret['compose'][prop.lower()] = prop - ret['with_nested_properties'] = True - # ret['property_name_format'] = 'lower_case' # only dacrystal/topic/vmware-inventory-plugin-property-format - - # process custom options - vmware_opts = dict(inventory_update.source_vars_dict.items()) - if inventory_update.instance_filters: - vmware_opts.setdefault('host_filters', inventory_update.instance_filters) - if inventory_update.group_by: - vmware_opts.setdefault('groupby_patterns', inventory_update.group_by) - - alias_pattern = vmware_opts.get('alias_pattern') - if alias_pattern: - ret.setdefault('hostnames', []) - for alias in alias_pattern.split(','): # make best effort - striped_alias = alias.replace('{', '').replace('}', '').strip() # make best effort - if not striped_alias: - continue - ret['hostnames'].append(striped_alias) - - host_pattern = vmware_opts.get('host_pattern') # not working in script - if host_pattern: - stripped_hp = host_pattern.replace('{', '').replace('}', '').strip() # make best effort - ret['compose']['ansible_host'] = stripped_hp - ret['compose']['ansible_ssh_host'] = stripped_hp - - host_filters = vmware_opts.get('host_filters') - if host_filters: - ret.setdefault('filters', []) - for hf in host_filters.split(','): - striped_hf = hf.replace('{', '').replace('}', '').strip() # make best effort - if not striped_hf: - continue - ret['filters'].append(striped_hf) - else: - # default behavior filters by power state - ret['filters'] = ['runtime.powerState == "poweredOn"'] - - groupby_patterns = vmware_opts.get('groupby_patterns') - ret.setdefault('keyed_groups', []) - if groupby_patterns: - for pattern in groupby_patterns.split(','): - stripped_pattern = pattern.replace('{', '').replace('}', '').strip() # make best effort - ret['keyed_groups'].append({ - 'prefix': '', 'separator': '', - 'key': stripped_pattern - }) - else: - # default groups from script - for entry in ('config.guestId', '"templates" if config.template else "guests"'): - ret['keyed_groups'].append({ - 'prefix': '', 'separator': '', - 'key': entry - }) - - return ret - class openstack(PluginFileInjector): plugin_name = 'openstack' @@ -2243,40 +1757,6 @@ class openstack(PluginFileInjector): ) return private_data - def inventory_as_dict(self, inventory_update, private_data_dir): - def use_host_name_for_name(a_bool_maybe): - if not isinstance(a_bool_maybe, bool): - # Could be specified by user via "host" or "uuid" - return a_bool_maybe - elif a_bool_maybe: - return 'name' # plugin default - else: - return 'uuid' - - ret = super(openstack, self).inventory_as_dict(inventory_update, private_data_dir) - ret['fail_on_errors'] = True - ret['expand_hostvars'] = True - ret['inventory_hostname'] = use_host_name_for_name(False) - # Note: mucking with defaults will break import integrity - # For the plugin, we need to use the same defaults as the old script - # or else imports will conflict. To find script defaults you have - # to read source code of the script. - # - # Script Defaults Plugin Defaults - # 'use_hostnames': False, 'name' (True) - # 'expand_hostvars': True, 'no' (False) - # 'fail_on_errors': True, 'no' (False) - # - # These are, yet again, different from ansible_variables in script logic - # but those are applied inconsistently - source_vars = inventory_update.source_vars_dict - for var_name in ['expand_hostvars', 'fail_on_errors']: - if var_name in source_vars: - ret[var_name] = source_vars[var_name] - if 'use_hostnames' in source_vars: - ret['inventory_hostname'] = use_host_name_for_name(source_vars['use_hostnames']) - return ret - def get_plugin_env(self, inventory_update, private_data_dir, private_data_files): env = super(openstack, self).get_plugin_env(inventory_update, private_data_dir, private_data_files) credential = inventory_update.get_cloud_credential() @@ -2294,25 +1774,6 @@ class rhv(PluginFileInjector): namespace = 'ovirt' collection = 'ovirt' - def inventory_as_dict(self, inventory_update, private_data_dir): - ret = super(rhv, self).inventory_as_dict(inventory_update, private_data_dir) - ret['ovirt_insecure'] = False # Default changed from script - # TODO: process strict option upstream - ret['compose'] = { - 'ansible_host': '(devices.values() | list)[0][0] if devices else None' - } - ret['keyed_groups'] = [] - for key in ('cluster', 'status'): - ret['keyed_groups'].append({'prefix': key, 'separator': '_', 'key': key}) - ret['keyed_groups'].append({'prefix': 'tag', 'separator': '_', 'key': 'tags'}) - ret['ovirt_hostname_preference'] = ['name', 'fqdn'] - source_vars = inventory_update.source_vars_dict - for key, value in source_vars.items(): - if key == 'plugin': - continue - ret[key] = value - return ret - class satellite6(PluginFileInjector): plugin_name = 'foreman' @@ -2330,114 +1791,6 @@ class satellite6(PluginFileInjector): ret['FOREMAN_PASSWORD'] = credential.get_input('password', default='') return ret - def inventory_as_dict(self, inventory_update, private_data_dir): - ret = super(satellite6, self).inventory_as_dict(inventory_update, private_data_dir) - ret['validate_certs'] = False - - group_patterns = '[]' - group_prefix = 'foreman_' - want_hostcollections = False - want_ansible_ssh_host = False - want_facts = True - - foreman_opts = inventory_update.source_vars_dict.copy() - for k, v in foreman_opts.items(): - if k == 'satellite6_group_patterns' and isinstance(v, str): - group_patterns = v - elif k == 'satellite6_group_prefix' and isinstance(v, str): - group_prefix = v - elif k == 'satellite6_want_hostcollections' and isinstance(v, bool): - want_hostcollections = v - elif k == 'satellite6_want_ansible_ssh_host' and isinstance(v, bool): - want_ansible_ssh_host = v - elif k == 'satellite6_want_facts' and isinstance(v, bool): - want_facts = v - # add backwards support for ssl_verify - # plugin uses new option, validate_certs, instead - elif k == 'ssl_verify' and isinstance(v, bool): - ret['validate_certs'] = v - else: - ret[k] = str(v) - - # Compatibility content - group_by_hostvar = { - "environment": {"prefix": "{}environment_".format(group_prefix), - "separator": "", - "key": "foreman['environment_name'] | lower | regex_replace(' ', '') | " - "regex_replace('[^A-Za-z0-9_]', '_') | regex_replace('none', '')"}, - "location": {"prefix": "{}location_".format(group_prefix), - "separator": "", - "key": "foreman['location_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_')"}, - "organization": {"prefix": "{}organization_".format(group_prefix), - "separator": "", - "key": "foreman['organization_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_')"}, - "lifecycle_environment": {"prefix": "{}lifecycle_environment_".format(group_prefix), - "separator": "", - "key": "foreman['content_facet_attributes']['lifecycle_environment_name'] | " - "lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_')"}, - "content_view": {"prefix": "{}content_view_".format(group_prefix), - "separator": "", - "key": "foreman['content_facet_attributes']['content_view_name'] | " - "lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_')"} - } - - ret['legacy_hostvars'] = True # convert hostvar structure to the form used by the script - ret['want_params'] = True - ret['group_prefix'] = group_prefix - ret['want_hostcollections'] = want_hostcollections - ret['want_facts'] = want_facts - - if want_ansible_ssh_host: - ret['compose'] = {'ansible_ssh_host': "foreman['ip6'] | default(foreman['ip'], true)"} - ret['keyed_groups'] = [group_by_hostvar[grouping_name] for grouping_name in group_by_hostvar] - - def form_keyed_group(group_pattern): - """ - Converts foreman group_pattern to - inventory plugin keyed_group - - e.g. {app_param}-{tier_param}-{dc_param} - becomes - "%s-%s-%s" | format(app_param, tier_param, dc_param) - """ - if type(group_pattern) is not str: - return None - params = re.findall('{[^}]*}', group_pattern) - if len(params) == 0: - return None - - param_names = [] - for p in params: - param_names.append(p[1:-1].strip()) # strip braces and space - - # form keyed_group key by - # replacing curly braces with '%s' - # (for use with jinja's format filter) - key = group_pattern - for p in params: - key = key.replace(p, '%s', 1) - - # apply jinja filter to key - key = '"{}" | format({})'.format(key, ', '.join(param_names)) - - keyed_group = {'key': key, - 'separator': ''} - return keyed_group - - try: - group_patterns = json.loads(group_patterns) - - if type(group_patterns) is list: - for group_pattern in group_patterns: - keyed_group = form_keyed_group(group_pattern) - if keyed_group: - ret['keyed_groups'].append(keyed_group) - except json.JSONDecodeError: - logger.warning('Could not parse group_patterns. Expected JSON-formatted string, found: {}' - .format(group_patterns)) - - return ret - class tower(PluginFileInjector): plugin_name = 'tower' @@ -2445,19 +1798,6 @@ class tower(PluginFileInjector): namespace = 'awx' collection = 'awx' - def inventory_as_dict(self, inventory_update, private_data_dir): - ret = super(tower, self).inventory_as_dict(inventory_update, private_data_dir) - # Credentials injected as env vars, same as script - try: - # plugin can take an actual int type - identifier = int(inventory_update.instance_filters) - except ValueError: - # inventory_id could be a named URL - identifier = iri_to_uri(inventory_update.instance_filters) - ret['inventory_id'] = identifier - ret['include_metadata'] = True # used for license check - return ret - for cls in PluginFileInjector.__subclasses__(): InventorySourceOptions.injectors[cls.__name__] = cls From 7278e7c02502eb2069617b856dcbc19bfe613b6f Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 27 Jul 2020 10:46:42 -0400 Subject: [PATCH 02/27] remove group_by from inventory source * Does not remove group_by testing --- awx/api/metadata.py | 6 --- awx/api/serializers.py | 2 +- .../migrations/0118_v380_inventory_plugins.py | 8 ++++ awx/main/models/inventory.py | 45 ------------------- .../plugins/modules/tower_inventory_source.py | 7 +-- .../test/awx/test_inventory_source.py | 1 - awxkit/awxkit/api/pages/inventory.py | 1 - 7 files changed, 10 insertions(+), 60 deletions(-) diff --git a/awx/api/metadata.py b/awx/api/metadata.py index 8bbfb906ef..0820902cd9 100644 --- a/awx/api/metadata.py +++ b/awx/api/metadata.py @@ -122,12 +122,6 @@ class Metadata(metadata.SimpleMetadata): get_regions = getattr(InventorySource, 'get_%s_region_choices' % cp) field_info['%s_region_choices' % cp] = get_regions() - # Special handling of group_by choices for EC2. - if field.field_name == 'group_by': - for cp in ('ec2',): - get_group_by_choices = getattr(InventorySource, 'get_%s_group_by_choices' % cp) - field_info['%s_group_by_choices' % cp] = get_group_by_choices() - # Special handling of notification configuration where the required properties # are conditional on the type selected. if field.field_name == 'notification_configuration': diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 6ce32b3f51..cdd03a4e93 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1937,7 +1937,7 @@ class InventorySourceOptionsSerializer(BaseSerializer): class Meta: fields = ('*', 'source', 'source_path', 'source_script', 'source_vars', 'credential', - 'source_regions', 'instance_filters', 'group_by', 'overwrite', 'overwrite_vars', + 'source_regions', 'instance_filters', 'overwrite', 'overwrite_vars', 'custom_virtualenv', 'timeout', 'verbosity') def get_related(self, obj): diff --git a/awx/main/migrations/0118_v380_inventory_plugins.py b/awx/main/migrations/0118_v380_inventory_plugins.py index e2c8cd2d82..939922b206 100644 --- a/awx/main/migrations/0118_v380_inventory_plugins.py +++ b/awx/main/migrations/0118_v380_inventory_plugins.py @@ -56,4 +56,12 @@ class Migration(migrations.Migration): operations = [ migrations.RunPython(inventory_source_vars_forward, inventory_source_vars_backward,), + migrations.RemoveField( + model_name='inventorysource', + name='group_by', + ), + migrations.RemoveField( + model_name='inventoryupdate', + name='group_by', + ), ] diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index b56f3bd03e..59e3b2d644 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -958,12 +958,6 @@ class InventorySourceOptions(BaseModel): default='', help_text=_('Comma-separated list of filter expressions (EC2 only). Hosts are imported when ANY of the filters match.'), ) - group_by = models.CharField( - max_length=1024, - blank=True, - default='', - help_text=_('Limit groups automatically created from inventory source (EC2 only).'), - ) overwrite = models.BooleanField( default=False, help_text=_('Overwrite local groups and hosts from remote inventory source.'), @@ -983,24 +977,6 @@ class InventorySourceOptions(BaseModel): default=1, ) - @classmethod - def get_ec2_group_by_choices(cls): - return [ - ('ami_id', _('Image ID')), - ('availability_zone', _('Availability Zone')), - ('aws_account', _('Account')), - ('instance_id', _('Instance ID')), - ('instance_state', _('Instance State')), - ('platform', _('Platform')), - ('instance_type', _('Instance Type')), - ('key_pair', _('Key Name')), - ('region', _('Region')), - ('security_group', _('Security Group')), - ('tag_keys', _('Tags')), - ('tag_none', _('Tag None')), - ('vpc_id', _('VPC ID')), - ] - @classmethod def get_ec2_region_choices(cls): ec2_region_names = getattr(settings, 'EC2_REGION_NAMES', {}) @@ -1189,27 +1165,6 @@ class InventorySourceOptions(BaseModel): else: return '' - def clean_group_by(self): - group_by = str(self.group_by or '') - if self.source == 'ec2': - get_choices = getattr(self, 'get_%s_group_by_choices' % self.source) - valid_choices = [x[0] for x in get_choices()] - choice_transform = lambda x: x.strip().lower() - valid_choices = [choice_transform(x) for x in valid_choices] - choices = [choice_transform(x) for x in group_by.split(',') if x.strip()] - invalid_choices = [] - for c in choices: - if c not in valid_choices and c not in invalid_choices: - invalid_choices.append(c) - if invalid_choices: - raise ValidationError(_('Invalid group by choice: %(choice)s') % - {'choice': ', '.join(invalid_choices)}) - return ','.join(choices) - elif self.source == 'vmware': - return group_by - else: - return '' - class InventorySource(UnifiedJobTemplate, InventorySourceOptions, CustomVirtualEnvMixin, RelatedJobsMixin): diff --git a/awx_collection/plugins/modules/tower_inventory_source.py b/awx_collection/plugins/modules/tower_inventory_source.py index 5b0e2961df..3110c873a3 100644 --- a/awx_collection/plugins/modules/tower_inventory_source.py +++ b/awx_collection/plugins/modules/tower_inventory_source.py @@ -69,10 +69,6 @@ options: description: - Comma-separated list of filter expressions for matching hosts. type: str - group_by: - description: - - Limit groups automatically created from inventory source. - type: str overwrite: description: - Delete child groups and hosts not found in source. @@ -167,7 +163,6 @@ def main(): credential=dict(), source_regions=dict(), instance_filters=dict(), - group_by=dict(), overwrite=dict(type='bool'), overwrite_vars=dict(type='bool'), custom_virtualenv=dict(), @@ -245,7 +240,7 @@ def main(): OPTIONAL_VARS = ( 'description', 'source', 'source_path', 'source_vars', - 'source_regions', 'instance_filters', 'group_by', + 'source_regions', 'instance_filters', 'overwrite', 'overwrite_vars', 'custom_virtualenv', 'timeout', 'verbosity', 'update_on_launch', 'update_cache_timeout', 'update_on_project_update' diff --git a/awx_collection/test/awx/test_inventory_source.py b/awx_collection/test/awx/test_inventory_source.py index ab0296689b..35267bc4b3 100644 --- a/awx_collection/test/awx/test_inventory_source.py +++ b/awx_collection/test/awx/test_inventory_source.py @@ -192,7 +192,6 @@ def test_falsy_value(run_module, admin_user, base_inventory): # UoPL ? ? o - - - - - - - - - - # source_regions ? ? - o o o - - - - - - - # instance_filters ? ? - o - - o - - - - o - -# group_by ? ? - o - - o - - - - - - # source_vars* ? ? - o - o o o o o - - - # environmet vars* ? ? o - - - - - - - - - o # source_script ? ? - - - - - - - - - - r diff --git a/awxkit/awxkit/api/pages/inventory.py b/awxkit/awxkit/api/pages/inventory.py index e00f0d329a..179082e5a8 100644 --- a/awxkit/awxkit/api/pages/inventory.py +++ b/awxkit/awxkit/api/pages/inventory.py @@ -499,7 +499,6 @@ class InventorySource(HasCreate, HasNotifications, UnifiedJobTemplate): payload.source_project = project.id optional_fields = ( - 'group_by', 'instance_filters', 'source_path', 'source_regions', From f32716a0f14d1c63e9a668ed7bfba59488f5a50f Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 27 Jul 2020 10:55:36 -0400 Subject: [PATCH 03/27] remove instance_filter --- awx/api/serializers.py | 2 +- .../migrations/0118_v380_inventory_plugins.py | 8 ++ awx/main/models/inventory.py | 115 ------------------ awx/main/tests/unit/test_tasks.py | 2 - .../plugins/modules/tower_inventory_source.py | 7 +- .../test/awx/test_inventory_source.py | 1 - awxkit/awxkit/api/pages/inventory.py | 1 - tools/scripts/get_ec2_filter_names.py | 21 ---- 8 files changed, 10 insertions(+), 147 deletions(-) delete mode 100755 tools/scripts/get_ec2_filter_names.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index cdd03a4e93..3d6adc6a78 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1937,7 +1937,7 @@ class InventorySourceOptionsSerializer(BaseSerializer): class Meta: fields = ('*', 'source', 'source_path', 'source_script', 'source_vars', 'credential', - 'source_regions', 'instance_filters', 'overwrite', 'overwrite_vars', + 'source_regions', 'overwrite', 'overwrite_vars', 'custom_virtualenv', 'timeout', 'verbosity') def get_related(self, obj): diff --git a/awx/main/migrations/0118_v380_inventory_plugins.py b/awx/main/migrations/0118_v380_inventory_plugins.py index 939922b206..3fddf729e0 100644 --- a/awx/main/migrations/0118_v380_inventory_plugins.py +++ b/awx/main/migrations/0118_v380_inventory_plugins.py @@ -64,4 +64,12 @@ class Migration(migrations.Migration): model_name='inventoryupdate', name='group_by', ), + migrations.RemoveField( + model_name='inventorysource', + name='instance_filter', + ), + migrations.RemoveField( + model_name='inventoryupdate', + name='instance_filter', + ), ] diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 59e3b2d644..f34ce3d236 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -838,89 +838,6 @@ class InventorySourceOptions(BaseModel): (2, '2 (DEBUG)'), ] - # Use tools/scripts/get_ec2_filter_names.py to build this list. - INSTANCE_FILTER_NAMES = [ - "architecture", - "association.allocation-id", - "association.association-id", - "association.ip-owner-id", - "association.public-ip", - "availability-zone", - "block-device-mapping.attach-time", - "block-device-mapping.delete-on-termination", - "block-device-mapping.device-name", - "block-device-mapping.status", - "block-device-mapping.volume-id", - "client-token", - "dns-name", - "group-id", - "group-name", - "hypervisor", - "iam-instance-profile.arn", - "image-id", - "instance-id", - "instance-lifecycle", - "instance-state-code", - "instance-state-name", - "instance-type", - "instance.group-id", - "instance.group-name", - "ip-address", - "kernel-id", - "key-name", - "launch-index", - "launch-time", - "monitoring-state", - "network-interface-private-dns-name", - "network-interface.addresses.association.ip-owner-id", - "network-interface.addresses.association.public-ip", - "network-interface.addresses.primary", - "network-interface.addresses.private-ip-address", - "network-interface.attachment.attach-time", - "network-interface.attachment.attachment-id", - "network-interface.attachment.delete-on-termination", - "network-interface.attachment.device-index", - "network-interface.attachment.instance-id", - "network-interface.attachment.instance-owner-id", - "network-interface.attachment.status", - "network-interface.availability-zone", - "network-interface.description", - "network-interface.group-id", - "network-interface.group-name", - "network-interface.mac-address", - "network-interface.network-interface.id", - "network-interface.owner-id", - "network-interface.requester-id", - "network-interface.requester-managed", - "network-interface.source-destination-check", - "network-interface.status", - "network-interface.subnet-id", - "network-interface.vpc-id", - "owner-id", - "placement-group-name", - "platform", - "private-dns-name", - "private-ip-address", - "product-code", - "product-code.type", - "ramdisk-id", - "reason", - "requester-id", - "reservation-id", - "root-device-name", - "root-device-type", - "source-dest-check", - "spot-instance-request-id", - "state-reason-code", - "state-reason-message", - "subnet-id", - "tag-key", - "tag-value", - "tenancy", - "virtualization-type", - "vpc-id" - ] - class Meta: abstract = True @@ -952,12 +869,6 @@ class InventorySourceOptions(BaseModel): blank=True, default='', ) - instance_filters = models.CharField( - max_length=1024, - blank=True, - default='', - help_text=_('Comma-separated list of filter expressions (EC2 only). Hosts are imported when ANY of the filters match.'), - ) overwrite = models.BooleanField( default=False, help_text=_('Overwrite local groups and hosts from remote inventory source.'), @@ -1139,32 +1050,6 @@ class InventorySourceOptions(BaseModel): source_vars_dict = VarsDictProperty('source_vars') - def clean_instance_filters(self): - instance_filters = str(self.instance_filters or '') - if self.source == 'ec2': - invalid_filters = [] - instance_filter_re = re.compile(r'^((tag:.+)|([a-z][a-z\.-]*[a-z]))=.*$') - for instance_filter in instance_filters.split(','): - instance_filter = instance_filter.strip() - if not instance_filter: - continue - if not instance_filter_re.match(instance_filter): - invalid_filters.append(instance_filter) - continue - instance_filter_name = instance_filter.split('=', 1)[0] - if instance_filter_name.startswith('tag:'): - continue - if instance_filter_name not in self.INSTANCE_FILTER_NAMES: - invalid_filters.append(instance_filter) - if invalid_filters: - raise ValidationError(_('Invalid filter expression: %(filter)s') % - {'filter': ', '.join(invalid_filters)}) - return instance_filters - elif self.source in ('vmware', 'tower'): - return instance_filters - else: - return '' - class InventorySource(UnifiedJobTemplate, InventorySourceOptions, CustomVirtualEnvMixin, RelatedJobsMixin): diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 7dcea33ccd..341fcdc82d 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -2216,7 +2216,6 @@ class TestInventoryUpdateCredentials(TestJobExecution): task = tasks.RunInventoryUpdate() tower = CredentialType.defaults['tower']() inventory_update.source = 'tower' - inventory_update.instance_filters = '12345' inputs = { 'host': 'https://tower.example.org', 'username': 'bob', @@ -2248,7 +2247,6 @@ class TestInventoryUpdateCredentials(TestJobExecution): task = tasks.RunInventoryUpdate() tower = CredentialType.defaults['tower']() inventory_update.source = 'tower' - inventory_update.instance_filters = '12345' inputs = { 'host': 'https://tower.example.org', 'username': 'bob', diff --git a/awx_collection/plugins/modules/tower_inventory_source.py b/awx_collection/plugins/modules/tower_inventory_source.py index 3110c873a3..3840806f8f 100644 --- a/awx_collection/plugins/modules/tower_inventory_source.py +++ b/awx_collection/plugins/modules/tower_inventory_source.py @@ -65,10 +65,6 @@ options: description: - Regions for cloud provider. type: str - instance_filters: - description: - - Comma-separated list of filter expressions for matching hosts. - type: str overwrite: description: - Delete child groups and hosts not found in source. @@ -162,7 +158,6 @@ def main(): source_vars=dict(type='dict'), credential=dict(), source_regions=dict(), - instance_filters=dict(), overwrite=dict(type='bool'), overwrite_vars=dict(type='bool'), custom_virtualenv=dict(), @@ -240,7 +235,7 @@ def main(): OPTIONAL_VARS = ( 'description', 'source', 'source_path', 'source_vars', - 'source_regions', 'instance_filters', + 'source_regions', 'overwrite', 'overwrite_vars', 'custom_virtualenv', 'timeout', 'verbosity', 'update_on_launch', 'update_cache_timeout', 'update_on_project_update' diff --git a/awx_collection/test/awx/test_inventory_source.py b/awx_collection/test/awx/test_inventory_source.py index 35267bc4b3..fff6cdd3c3 100644 --- a/awx_collection/test/awx/test_inventory_source.py +++ b/awx_collection/test/awx/test_inventory_source.py @@ -191,7 +191,6 @@ def test_falsy_value(run_module, admin_user, base_inventory): # update_on_launch ? ? o o o o o o o o o o o # UoPL ? ? o - - - - - - - - - - # source_regions ? ? - o o o - - - - - - - -# instance_filters ? ? - o - - o - - - - o - # source_vars* ? ? - o - o o o o o - - - # environmet vars* ? ? o - - - - - - - - - o # source_script ? ? - - - - - - - - - - r diff --git a/awxkit/awxkit/api/pages/inventory.py b/awxkit/awxkit/api/pages/inventory.py index 179082e5a8..0976b09afe 100644 --- a/awxkit/awxkit/api/pages/inventory.py +++ b/awxkit/awxkit/api/pages/inventory.py @@ -499,7 +499,6 @@ class InventorySource(HasCreate, HasNotifications, UnifiedJobTemplate): payload.source_project = project.id optional_fields = ( - 'instance_filters', 'source_path', 'source_regions', 'source_vars', diff --git a/tools/scripts/get_ec2_filter_names.py b/tools/scripts/get_ec2_filter_names.py deleted file mode 100755 index d001b1f8e7..0000000000 --- a/tools/scripts/get_ec2_filter_names.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python - -import json -import sys -import requests -from bs4 import BeautifulSoup - -response = requests.get('http://docs.aws.amazon.com/AWSEC2/latest/APIReference/ApiReference-query-DescribeInstances.html') -soup = BeautifulSoup(response.text) - -section_h3 = soup.find(id='query-DescribeInstances-filters') -section_div = section_h3.find_parent('div', attrs={'class': 'section'}) - -filter_names = [] -for term in section_div.select('div.variablelist dt span.term'): - filter_name = term.get_text() - if not filter_name.startswith('tag:'): - filter_names.append(filter_name) -filter_names.sort() - -json.dump(filter_names, sys.stdout, indent=4) From a8a47f314e52db0672555741dd678fb7542db63f Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 27 Jul 2020 11:04:51 -0400 Subject: [PATCH 04/27] remove source_regions --- awx/api/metadata.py | 9 +- awx/api/serializers.py | 4 +- .../migrations/0118_v380_inventory_plugins.py | 22 ++-- awx/main/models/inventory.py | 106 +----------------- awx/main/tests/unit/test_tasks.py | 3 - awx/main/tests/unit/utils/test_common.py | 8 -- awx/main/utils/common.py | 11 +- awx/settings/defaults.py | 86 -------------- .../plugins/modules/tower_inventory_source.py | 6 - .../test/awx/test_inventory_source.py | 1 - awxkit/awxkit/api/pages/inventory.py | 1 - 11 files changed, 21 insertions(+), 236 deletions(-) diff --git a/awx/api/metadata.py b/awx/api/metadata.py index 0820902cd9..847e353890 100644 --- a/awx/api/metadata.py +++ b/awx/api/metadata.py @@ -23,7 +23,7 @@ from rest_framework.request import clone_request # AWX from awx.api.fields import ChoiceNullField from awx.main.fields import JSONField, ImplicitRoleField -from awx.main.models import InventorySource, NotificationTemplate +from awx.main.models import NotificationTemplate from awx.main.scheduler.kubernetes import PodManager @@ -115,13 +115,6 @@ class Metadata(metadata.SimpleMetadata): if getattr(field, 'write_only', False): field_info['write_only'] = True - # Special handling of inventory source_region choices that vary based on - # selected inventory source. - if field.field_name == 'source_regions': - for cp in ('azure_rm', 'ec2', 'gce'): - get_regions = getattr(InventorySource, 'get_%s_region_choices' % cp) - field_info['%s_region_choices' % cp] = get_regions() - # Special handling of notification configuration where the required properties # are conditional on the type selected. if field.field_name == 'notification_configuration': diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 3d6adc6a78..9cf953262b 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1937,7 +1937,7 @@ class InventorySourceOptionsSerializer(BaseSerializer): class Meta: fields = ('*', 'source', 'source_path', 'source_script', 'source_vars', 'credential', - 'source_regions', 'overwrite', 'overwrite_vars', + 'overwrite', 'overwrite_vars', 'custom_virtualenv', 'timeout', 'verbosity') def get_related(self, obj): @@ -1957,7 +1957,7 @@ class InventorySourceOptionsSerializer(BaseSerializer): return ret def validate(self, attrs): - # TODO: Validate source, validate source_regions + # TODO: Validate source errors = {} source = attrs.get('source', self.instance and self.instance.source or '') diff --git a/awx/main/migrations/0118_v380_inventory_plugins.py b/awx/main/migrations/0118_v380_inventory_plugins.py index 3fddf729e0..d2c9a06398 100644 --- a/awx/main/migrations/0118_v380_inventory_plugins.py +++ b/awx/main/migrations/0118_v380_inventory_plugins.py @@ -3,9 +3,8 @@ import logging import json -from django.db import migrations, models +from django.db import migrations -from awx.main.models.inventory import InventorySource from ._inventory_source_vars import FrozenInjectors @@ -13,15 +12,16 @@ logger = logging.getLogger('awx.main.migrations') BACKUP_FILENAME = '/tmp/tower_migration_inventory_source_vars.json' -def _get_inventory_sources(): - # TODO: Maybe pull this list from an import +def _get_inventory_sources(InventorySource): + # TODO: Maybe pull the list of cloud sources from code return InventorySource.objects.filter(source__in=['ec2', 'gce', 'azure_rm', 'vmware', 'satellite6', 'openstack', 'rhv', 'tower']) def inventory_source_vars_forward(apps, schema_editor): + InventorySource = apps.get_model("main", "InventorySource") source_vars_backup = dict() - for inv_source_obj in _get_inventory_sources(): + for inv_source_obj in _get_inventory_sources(InventorySource): # TODO: Log error if this is false, it shouldn't be false if inv_source_obj.source in FrozenInjectors: source_vars_backup[inv_source_obj.id] = dict(inv_source_obj.source_vars_dict) @@ -66,10 +66,18 @@ class Migration(migrations.Migration): ), migrations.RemoveField( model_name='inventorysource', - name='instance_filter', + name='instance_filters', ), migrations.RemoveField( model_name='inventoryupdate', - name='instance_filter', + name='instance_filters', + ), + migrations.RemoveField( + model_name='inventorysource', + name='source_regions', + ), + migrations.RemoveField( + model_name='inventoryupdate', + name='source_regions', ), ] diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index f34ce3d236..36b358a1bd 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -4,7 +4,6 @@ # Python import datetime import time -import json import logging import re import copy @@ -19,7 +18,6 @@ from django.utils.translation import ugettext_lazy as _ from django.db import transaction from django.core.exceptions import ValidationError from django.utils.timezone import now -from django.utils.encoding import iri_to_uri from django.db.models import Q # REST Framework @@ -56,7 +54,7 @@ from awx.main.models.notifications import ( JobNotificationMixin, ) from awx.main.models.credential.injectors import _openstack_data -from awx.main.utils import _inventory_updates, region_sorting +from awx.main.utils import _inventory_updates from awx.main.utils.safe_yaml import sanitize_jinja @@ -864,11 +862,6 @@ class InventorySourceOptions(BaseModel): default='', help_text=_('Inventory source variables in YAML or JSON format.'), ) - source_regions = models.CharField( - max_length=1024, - blank=True, - default='', - ) overwrite = models.BooleanField( default=False, help_text=_('Overwrite local groups and hosts from remote inventory source.'), @@ -888,79 +881,6 @@ class InventorySourceOptions(BaseModel): default=1, ) - @classmethod - def get_ec2_region_choices(cls): - ec2_region_names = getattr(settings, 'EC2_REGION_NAMES', {}) - ec2_name_replacements = { - 'us': 'US', - 'ap': 'Asia Pacific', - 'eu': 'Europe', - 'sa': 'South America', - } - import boto.ec2 - regions = [('all', 'All')] - for region in boto.ec2.regions(): - label = ec2_region_names.get(region.name, '') - if not label: - label_parts = [] - for part in region.name.split('-'): - part = ec2_name_replacements.get(part.lower(), part.title()) - label_parts.append(part) - label = ' '.join(label_parts) - regions.append((region.name, label)) - return sorted(regions, key=region_sorting) - - @classmethod - def get_gce_region_choices(self): - """Return a complete list of regions in GCE, as a list of - two-tuples. - """ - # It's not possible to get a list of regions from GCE without - # authenticating first. Therefore, use a list from settings. - regions = list(getattr(settings, 'GCE_REGION_CHOICES', [])) - regions.insert(0, ('all', 'All')) - return sorted(regions, key=region_sorting) - - @classmethod - def get_azure_rm_region_choices(self): - """Return a complete list of regions in Microsoft Azure, as a list of - two-tuples. - """ - # It's not possible to get a list of regions from Azure without - # authenticating first (someone reading these might think there's - # a pattern here!). Therefore, you guessed it, use a list from - # settings. - regions = list(getattr(settings, 'AZURE_RM_REGION_CHOICES', [])) - regions.insert(0, ('all', 'All')) - return sorted(regions, key=region_sorting) - - @classmethod - def get_vmware_region_choices(self): - """Return a complete list of regions in VMware, as a list of two-tuples - (but note that VMware doesn't actually have regions!). - """ - return [('all', 'All')] - - @classmethod - def get_openstack_region_choices(self): - """I don't think openstack has regions""" - return [('all', 'All')] - - @classmethod - def get_satellite6_region_choices(self): - """Red Hat Satellite 6 region choices (not implemented)""" - return [('all', 'All')] - - @classmethod - def get_rhv_region_choices(self): - """No region supprt""" - return [('all', 'All')] - - @classmethod - def get_tower_region_choices(self): - """No region supprt""" - return [('all', 'All')] - @staticmethod def cloud_credential_validation(source, cred): if not source: @@ -1025,28 +945,6 @@ class InventorySourceOptions(BaseModel): if cred is not None: return cred.pk - def clean_source_regions(self): - regions = self.source_regions - - if self.source in CLOUD_PROVIDERS: - get_regions = getattr(self, 'get_%s_region_choices' % self.source) - valid_regions = [x[0] for x in get_regions()] - region_transform = lambda x: x.strip().lower() - else: - return '' - all_region = region_transform('all') - valid_regions = [region_transform(x) for x in valid_regions] - regions = [region_transform(x) for x in regions.split(',') if x.strip()] - if all_region in regions: - return all_region - invalid_regions = [] - for r in regions: - if r not in valid_regions and r not in invalid_regions: - invalid_regions.append(r) - if invalid_regions: - raise ValidationError(_('Invalid %(source)s region: %(region)s') % { - 'source': self.source, 'region': ', '.join(invalid_regions)}) - return ','.join(regions) source_vars_dict = VarsDictProperty('source_vars') @@ -1550,7 +1448,7 @@ class gce(PluginFileInjector): def inventory_as_dict(self, inventory_update, private_data_dir): ret = super().inventory_as_dict(inventory_update, private_data_dir) - credential = inventory_source.get_cloud_credential() + credential = inventory_update.get_cloud_credential() ret['projects'] = [credential.get_input('project', default='')] return ret diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 341fcdc82d..cda720b6ab 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -2020,7 +2020,6 @@ class TestInventoryUpdateCredentials(TestJobExecution): task = tasks.RunInventoryUpdate() azure_rm = CredentialType.defaults['azure_rm']() inventory_update.source = 'azure_rm' - inventory_update.source_regions = 'north, south, east, west' def get_cred(): cred = Credential( @@ -2059,7 +2058,6 @@ class TestInventoryUpdateCredentials(TestJobExecution): task = tasks.RunInventoryUpdate() azure_rm = CredentialType.defaults['azure_rm']() inventory_update.source = 'azure_rm' - inventory_update.source_regions = 'all' def get_cred(): cred = Credential( @@ -2097,7 +2095,6 @@ class TestInventoryUpdateCredentials(TestJobExecution): task = tasks.RunInventoryUpdate() gce = CredentialType.defaults['gce']() inventory_update.source = 'gce' - inventory_update.source_regions = 'all' def get_cred(): cred = Credential( diff --git a/awx/main/tests/unit/utils/test_common.py b/awx/main/tests/unit/utils/test_common.py index edacfc1423..8c07020c53 100644 --- a/awx/main/tests/unit/utils/test_common.py +++ b/awx/main/tests/unit/utils/test_common.py @@ -215,11 +215,3 @@ def test_get_custom_venv_choices(): os.path.join(temp_dir, ''), os.path.join(custom_venv_1, '') ] - - -def test_region_sorting(): - s = [('Huey', 'China1'), - ('Dewey', 'UK1'), - ('Lewie', 'US1'), - ('All', 'All')] - assert [x[1] for x in sorted(s, key=common.region_sorting)] == ['All', 'US1', 'China1', 'UK1'] diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index f34cf4e4d8..a017dba61b 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -45,7 +45,7 @@ __all__ = [ 'get_object_or_400', 'camelcase_to_underscore', 'underscore_to_camelcase', 'memoize', 'memoize_delete', 'get_ansible_version', 'get_licenser', 'get_awx_http_client_headers', 'get_awx_version', 'update_scm_url', 'get_type_for_model', 'get_model_for_type', - 'copy_model_by_class', 'region_sorting', 'copy_m2m_relationships', + 'copy_model_by_class', 'copy_m2m_relationships', 'prefetch_page_capabilities', 'to_python_boolean', 'ignore_inventory_computed_fields', 'ignore_inventory_group_removal', '_inventory_updates', 'get_pk_from_dict', 'getattrd', 'getattr_dne', 'NoDefaultProvided', 'get_current_apps', 'set_current_apps', @@ -87,15 +87,6 @@ def to_python_boolean(value, allow_none=False): raise ValueError(_(u'Unable to convert "%s" to boolean') % value) -def region_sorting(region): - # python3's removal of sorted(cmp=...) is _stupid_ - if region[1].lower() == 'all': - return '' - elif region[1].lower().startswith('us'): - return region[1] - return 'ZZZ' + str(region[1]) - - def camelcase_to_underscore(s): ''' Convert CamelCase names to lowercase_with_underscore. diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 716aea3aa7..cb7dc49e46 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -672,28 +672,6 @@ INV_ENV_VARIABLE_BLOCKED = ("HOME", "USER", "_", "TERM") # -- Amazon EC2 -- # ---------------- -# AWS does not appear to provide pretty region names via any API, so store the -# list of names here. The available region IDs will be pulled from boto. -# http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region -EC2_REGION_NAMES = { - 'us-east-1': _('US East (Northern Virginia)'), - 'us-east-2': _('US East (Ohio)'), - 'us-west-2': _('US West (Oregon)'), - 'us-west-1': _('US West (Northern California)'), - 'ca-central-1': _('Canada (Central)'), - 'eu-central-1': _('EU (Frankfurt)'), - 'eu-west-1': _('EU (Ireland)'), - 'eu-west-2': _('EU (London)'), - 'ap-southeast-1': _('Asia Pacific (Singapore)'), - 'ap-southeast-2': _('Asia Pacific (Sydney)'), - 'ap-northeast-1': _('Asia Pacific (Tokyo)'), - 'ap-northeast-2': _('Asia Pacific (Seoul)'), - 'ap-south-1': _('Asia Pacific (Mumbai)'), - 'sa-east-1': _('South America (Sao Paulo)'), - 'us-gov-west-1': _('US West (GovCloud)'), - 'cn-north-1': _('China (Beijing)'), -} - # Inventory variable name/values for determining if host is active/enabled. EC2_ENABLED_VAR = 'ec2_state' EC2_ENABLED_VALUE = 'running' @@ -729,41 +707,6 @@ VMWARE_VALIDATE_CERTS = False # -- Google Compute Engine -- # --------------------------- -# It's not possible to get zones in GCE without authenticating, so we -# provide a list here. -# Source: https://developers.google.com/compute/docs/zones -GCE_REGION_CHOICES = [ - ('us-east1-b', _('US East 1 (B)')), - ('us-east1-c', _('US East 1 (C)')), - ('us-east1-d', _('US East 1 (D)')), - ('us-east4-a', _('US East 4 (A)')), - ('us-east4-b', _('US East 4 (B)')), - ('us-east4-c', _('US East 4 (C)')), - ('us-central1-a', _('US Central (A)')), - ('us-central1-b', _('US Central (B)')), - ('us-central1-c', _('US Central (C)')), - ('us-central1-f', _('US Central (F)')), - ('us-west1-a', _('US West (A)')), - ('us-west1-b', _('US West (B)')), - ('us-west1-c', _('US West (C)')), - ('europe-west1-b', _('Europe West 1 (B)')), - ('europe-west1-c', _('Europe West 1 (C)')), - ('europe-west1-d', _('Europe West 1 (D)')), - ('europe-west2-a', _('Europe West 2 (A)')), - ('europe-west2-b', _('Europe West 2 (B)')), - ('europe-west2-c', _('Europe West 2 (C)')), - ('asia-east1-a', _('Asia East (A)')), - ('asia-east1-b', _('Asia East (B)')), - ('asia-east1-c', _('Asia East (C)')), - ('asia-southeast1-a', _('Asia Southeast (A)')), - ('asia-southeast1-b', _('Asia Southeast (B)')), - ('asia-northeast1-a', _('Asia Northeast (A)')), - ('asia-northeast1-b', _('Asia Northeast (B)')), - ('asia-northeast1-c', _('Asia Northeast (C)')), - ('australia-southeast1-a', _('Australia Southeast (A)')), - ('australia-southeast1-b', _('Australia Southeast (B)')), - ('australia-southeast1-c', _('Australia Southeast (C)')), -] # Inventory variable name/value for determining whether a host is active # in Google Compute Engine. GCE_ENABLED_VAR = 'status' @@ -779,35 +722,6 @@ GCE_INSTANCE_ID_VAR = 'gce_id' # -------------------------------------- # -- Microsoft Azure Resource Manager -- # -------------------------------------- -# It's not possible to get zones in Azure without authenticating, so we -# provide a list here. -AZURE_RM_REGION_CHOICES = [ - ('eastus', _('US East')), - ('eastus2', _('US East 2')), - ('centralus', _('US Central')), - ('northcentralus', _('US North Central')), - ('southcentralus', _('US South Central')), - ('westcentralus', _('US West Central')), - ('westus', _('US West')), - ('westus2', _('US West 2')), - ('canadaeast', _('Canada East')), - ('canadacentral', _('Canada Central')), - ('brazilsouth', _('Brazil South')), - ('northeurope', _('Europe North')), - ('westeurope', _('Europe West')), - ('ukwest', _('UK West')), - ('uksouth', _('UK South')), - ('eastasia', _('Asia East')), - ('southestasia', _('Asia Southeast')), - ('australiaeast', _('Australia East')), - ('australiasoutheast', _('Australia Southeast')), - ('westindia', _('India West')), - ('southindia', _('India South')), - ('japaneast', _('Japan East')), - ('japanwest', _('Japan West')), - ('koreacentral', _('Korea Central')), - ('koreasouth', _('Korea South')), -] AZURE_RM_GROUP_FILTER = r'^.+$' AZURE_RM_HOST_FILTER = r'^.+$' AZURE_RM_ENABLED_VAR = 'powerstate' diff --git a/awx_collection/plugins/modules/tower_inventory_source.py b/awx_collection/plugins/modules/tower_inventory_source.py index 3840806f8f..d3517b3f5a 100644 --- a/awx_collection/plugins/modules/tower_inventory_source.py +++ b/awx_collection/plugins/modules/tower_inventory_source.py @@ -61,10 +61,6 @@ options: description: - Credential to use for the source. type: str - source_regions: - description: - - Regions for cloud provider. - type: str overwrite: description: - Delete child groups and hosts not found in source. @@ -157,7 +153,6 @@ def main(): source_script=dict(), source_vars=dict(type='dict'), credential=dict(), - source_regions=dict(), overwrite=dict(type='bool'), overwrite_vars=dict(type='bool'), custom_virtualenv=dict(), @@ -235,7 +230,6 @@ def main(): OPTIONAL_VARS = ( 'description', 'source', 'source_path', 'source_vars', - 'source_regions', 'overwrite', 'overwrite_vars', 'custom_virtualenv', 'timeout', 'verbosity', 'update_on_launch', 'update_cache_timeout', 'update_on_project_update' diff --git a/awx_collection/test/awx/test_inventory_source.py b/awx_collection/test/awx/test_inventory_source.py index fff6cdd3c3..b27653fb94 100644 --- a/awx_collection/test/awx/test_inventory_source.py +++ b/awx_collection/test/awx/test_inventory_source.py @@ -190,7 +190,6 @@ def test_falsy_value(run_module, admin_user, base_inventory): # overwrite_vars ? ? o o o o o o o o o o o # update_on_launch ? ? o o o o o o o o o o o # UoPL ? ? o - - - - - - - - - - -# source_regions ? ? - o o o - - - - - - - # source_vars* ? ? - o - o o o o o - - - # environmet vars* ? ? o - - - - - - - - - o # source_script ? ? - - - - - - - - - - r diff --git a/awxkit/awxkit/api/pages/inventory.py b/awxkit/awxkit/api/pages/inventory.py index 0976b09afe..ea7e70b3b8 100644 --- a/awxkit/awxkit/api/pages/inventory.py +++ b/awxkit/awxkit/api/pages/inventory.py @@ -500,7 +500,6 @@ class InventorySource(HasCreate, HasNotifications, UnifiedJobTemplate): optional_fields = ( 'source_path', - 'source_regions', 'source_vars', 'timeout', 'overwrite', From 34adbe60284832195575a5378c643b0a1b080bb5 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Tue, 28 Jul 2020 13:14:43 -0400 Subject: [PATCH 05/27] fix tests for new inv plugin behavior * Enforce plugin: --- awx/main/models/inventory.py | 21 +++++ .../plugins/azure_rm/files/azure_rm.yml | 43 --------- .../inventory/plugins/ec2/files/aws_ec2.yml | 81 ----------------- .../plugins/gce/files/gcp_compute.yml | 50 ----------- .../plugins/openstack/files/file_reference | 6 +- .../plugins/openstack/files/openstack.yml | 4 - .../inventory/plugins/rhv/files/ovirt.yml | 20 ----- .../plugins/satellite6/files/foreman.yml | 30 ------- .../inventory/plugins/tower/files/tower.yml | 3 - .../vmware/files/vmware_vm_inventory.yml | 55 ------------ .../tests/functional/api/test_inventory.py | 70 +++++++++++++-- .../tests/functional/models/test_inventory.py | 38 +++----- .../test_inventory_source_injectors.py | 88 +++---------------- awx/main/tests/unit/models/test_inventory.py | 17 ---- awx/settings/defaults.py | 2 - 15 files changed, 111 insertions(+), 417 deletions(-) delete mode 100644 awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml delete mode 100644 awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml delete mode 100644 awx/main/tests/data/inventory/plugins/gce/files/gcp_compute.yml delete mode 100644 awx/main/tests/data/inventory/plugins/openstack/files/openstack.yml delete mode 100644 awx/main/tests/data/inventory/plugins/rhv/files/ovirt.yml delete mode 100644 awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml delete mode 100644 awx/main/tests/data/inventory/plugins/tower/files/tower.yml delete mode 100644 awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 36b358a1bd..7d6c350274 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -7,6 +7,7 @@ import time import logging import re import copy +import json import os.path from urllib.parse import urljoin import yaml @@ -1157,6 +1158,20 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions, CustomVirtualE raise ValidationError(_("Cannot set source_path if not SCM type.")) return self.source_path + def clean_source_vars(self): + injector = self.injectors.get(self.source) + source_vars = dict(self.source_vars_dict) # make a copy + if injector and self.source_vars_dict.get('plugin', '') != injector.get_proper_name(): + source_vars['plugin'] = injector.get_proper_name() + elif not injector: + source_vars = dict(self.source_vars_dict) # make a copy + collection_pattern = re.compile("^(.+)\.(.+)\.(.+)$") # noqa + if 'plugin' not in source_vars: + raise ValidationError(_("plugin: must be present and of the form namespace.collection.inv_plugin")) + elif not bool(collection_pattern.match(source_vars['plugin'])): + raise ValidationError(_("plugin: must be of the form namespace.collection.inv_plugin")) + return json.dumps(source_vars) + ''' RelatedJobsMixin ''' @@ -1344,6 +1359,12 @@ class PluginFileInjector(object): # This is InventoryOptions instance, could be source or inventory update self.ansible_version = ansible_version + @classmethod + def get_proper_name(cls): + if cls.plugin_name is None: + return None + return f'{cls.namespace}.{cls.collection}.{cls.plugin_name}' + @property def filename(self): """Inventory filename for using the inventory plugin diff --git a/awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml b/awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml deleted file mode 100644 index 8d6c1dbfa7..0000000000 --- a/awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml +++ /dev/null @@ -1,43 +0,0 @@ -conditional_groups: - azure: true -default_host_filters: [] -exclude_host_filters: -- resource_group not in ['foo_resources', 'bar_resources'] -- '"Creator" not in tags.keys()' -- tags["Creator"] != "jmarshall" -- '"peanutbutter" not in tags.keys()' -- tags["peanutbutter"] != "jelly" -- location not in ['southcentralus', 'westus'] -fail_on_template_errors: false -hostvar_expressions: - ansible_host: private_ipv4_addresses[0] - computer_name: name - private_ip: private_ipv4_addresses[0] if private_ipv4_addresses else None - provisioning_state: provisioning_state | title - public_ip: public_ipv4_addresses[0] if public_ipv4_addresses else None - public_ip_id: public_ip_id if public_ip_id is defined else None - public_ip_name: public_ip_name if public_ip_name is defined else None - tags: tags if tags else None - type: resource_type -keyed_groups: -- key: location - prefix: '' - separator: '' -- key: tags.keys() | list if tags else [] - prefix: '' - separator: '' -- key: security_group - prefix: '' - separator: '' -- key: resource_group - prefix: '' - separator: '' -- key: os_disk.operating_system_type - prefix: '' - separator: '' -- key: dict(tags.keys() | map("regex_replace", "^(.*)$", "\1_") | list | zip(tags.values() | list)) if tags else [] - prefix: '' - separator: '' -plain_host_names: true -plugin: azure.azcollection.azure_rm -use_contrib_script_compatible_sanitization: true diff --git a/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml b/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml deleted file mode 100644 index 8984d4cb56..0000000000 --- a/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml +++ /dev/null @@ -1,81 +0,0 @@ -boto_profile: /tmp/my_boto_stuff -compose: - ansible_host: public_dns_name - ec2_account_id: owner_id - ec2_ami_launch_index: ami_launch_index | string - ec2_architecture: architecture - ec2_block_devices: dict(block_device_mappings | map(attribute='device_name') | list | zip(block_device_mappings | map(attribute='ebs.volume_id') | list)) - ec2_client_token: client_token - ec2_dns_name: public_dns_name - ec2_ebs_optimized: ebs_optimized - ec2_eventsSet: events | default("") - ec2_group_name: placement.group_name - ec2_hypervisor: hypervisor - ec2_id: instance_id - ec2_image_id: image_id - ec2_instance_profile: iam_instance_profile | default("") - ec2_instance_type: instance_type - ec2_ip_address: public_ip_address - ec2_kernel: kernel_id | default("") - ec2_key_name: key_name - ec2_launch_time: launch_time | regex_replace(" ", "T") | regex_replace("(\+)(\d\d):(\d)(\d)$", ".\g<2>\g<3>Z") - ec2_monitored: monitoring.state in ['enabled', 'pending'] - ec2_monitoring_state: monitoring.state - ec2_persistent: persistent | default(false) - ec2_placement: placement.availability_zone - ec2_platform: platform | default("") - ec2_private_dns_name: private_dns_name - ec2_private_ip_address: private_ip_address - ec2_public_dns_name: public_dns_name - ec2_ramdisk: ramdisk_id | default("") - ec2_reason: state_transition_reason - ec2_region: placement.region - ec2_requester_id: requester_id | default("") - ec2_root_device_name: root_device_name - ec2_root_device_type: root_device_type - ec2_security_group_ids: security_groups | map(attribute='group_id') | list | join(',') - ec2_security_group_names: security_groups | map(attribute='group_name') | list | join(',') - ec2_sourceDestCheck: source_dest_check | default(false) | lower | string - ec2_spot_instance_request_id: spot_instance_request_id | default("") - ec2_state: state.name - ec2_state_code: state.code - ec2_state_reason: state_reason.message if state_reason is defined else "" - ec2_subnet_id: subnet_id | default("") - ec2_tag_Name: tags.Name - ec2_virtualization_type: virtualization_type - ec2_vpc_id: vpc_id | default("") -filters: - instance-state-name: - - running -groups: - ec2: true -hostnames: -- dns-name -iam_role_arn: arn:aws:iam::123456789012:role/test-role -keyed_groups: -- key: placement.availability_zone - parent_group: zones - prefix: '' - separator: '' -- key: instance_type | regex_replace("[^A-Za-z0-9\_]", "_") - parent_group: types - prefix: type -- key: placement.region - parent_group: regions - prefix: '' - separator: '' -- key: dict(tags.keys() | map("regex_replace", "[^A-Za-z0-9\_]", "_") | list | zip(tags.values() | map("regex_replace", "[^A-Za-z0-9\_]", "_") | list)) - parent_group: tags - prefix: tag -- key: tags.keys() | map("regex_replace", "[^A-Za-z0-9\_]", "_") | list - parent_group: tags - prefix: tag -- key: placement.availability_zone - parent_group: '{{ placement.region }}' - prefix: '' - separator: '' -plugin: amazon.aws.aws_ec2 -regions: -- us-east-2 -- ap-south-1 -use_contrib_script_compatible_sanitization: true diff --git a/awx/main/tests/data/inventory/plugins/gce/files/gcp_compute.yml b/awx/main/tests/data/inventory/plugins/gce/files/gcp_compute.yml deleted file mode 100644 index 63f8a44f64..0000000000 --- a/awx/main/tests/data/inventory/plugins/gce/files/gcp_compute.yml +++ /dev/null @@ -1,50 +0,0 @@ -auth_kind: serviceaccount -compose: - ansible_ssh_host: networkInterfaces[0].accessConfigs[0].natIP | default(networkInterfaces[0].networkIP) - gce_description: description if description else None - gce_id: id - gce_image: image - gce_machine_type: machineType - gce_metadata: metadata.get("items", []) | items2dict(key_name="key", value_name="value") - gce_name: name - gce_network: networkInterfaces[0].network.name - gce_private_ip: networkInterfaces[0].networkIP - gce_public_ip: networkInterfaces[0].accessConfigs[0].natIP | default(None) - gce_status: status - gce_subnetwork: networkInterfaces[0].subnetwork.name - gce_tags: tags.get("items", []) - gce_zone: zone -hostnames: -- name -- public_ip -- private_ip -keyed_groups: -- key: gce_subnetwork - prefix: network -- key: gce_private_ip - prefix: '' - separator: '' -- key: gce_public_ip - prefix: '' - separator: '' -- key: machineType - prefix: '' - separator: '' -- key: zone - prefix: '' - separator: '' -- key: gce_tags - prefix: tag -- key: status | lower - prefix: status -- key: image - prefix: '' - separator: '' -plugin: google.cloud.gcp_compute -projects: -- fooo -retrieve_image_info: true -use_contrib_script_compatible_sanitization: true -zones: -- us-east4-a -- us-west1-b diff --git a/awx/main/tests/data/inventory/plugins/openstack/files/file_reference b/awx/main/tests/data/inventory/plugins/openstack/files/file_reference index 895a1eb8a8..c578942ca1 100644 --- a/awx/main/tests/data/inventory/plugins/openstack/files/file_reference +++ b/awx/main/tests/data/inventory/plugins/openstack/files/file_reference @@ -1,7 +1,3 @@ -ansible: - expand_hostvars: true - fail_on_errors: true - use_hostnames: false clouds: devstack: auth: @@ -11,5 +7,5 @@ clouds: project_domain_name: fooo project_name: fooo username: fooo - private: false + private: true verify: false diff --git a/awx/main/tests/data/inventory/plugins/openstack/files/openstack.yml b/awx/main/tests/data/inventory/plugins/openstack/files/openstack.yml deleted file mode 100644 index 36e9024b54..0000000000 --- a/awx/main/tests/data/inventory/plugins/openstack/files/openstack.yml +++ /dev/null @@ -1,4 +0,0 @@ -expand_hostvars: true -fail_on_errors: true -inventory_hostname: uuid -plugin: openstack.cloud.openstack diff --git a/awx/main/tests/data/inventory/plugins/rhv/files/ovirt.yml b/awx/main/tests/data/inventory/plugins/rhv/files/ovirt.yml deleted file mode 100644 index 67a94ae6de..0000000000 --- a/awx/main/tests/data/inventory/plugins/rhv/files/ovirt.yml +++ /dev/null @@ -1,20 +0,0 @@ -base_source_var: value_of_var -compose: - ansible_host: (devices.values() | list)[0][0] if devices else None -groups: - dev: '"dev" in tags' -keyed_groups: -- key: cluster - prefix: cluster - separator: _ -- key: status - prefix: status - separator: _ -- key: tags - prefix: tag - separator: _ -ovirt_hostname_preference: -- name -- fqdn -ovirt_insecure: false -plugin: ovirt.ovirt.ovirt diff --git a/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml b/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml deleted file mode 100644 index fcad2586f6..0000000000 --- a/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml +++ /dev/null @@ -1,30 +0,0 @@ -base_source_var: value_of_var -compose: - ansible_ssh_host: foreman['ip6'] | default(foreman['ip'], true) -group_prefix: foo_group_prefix -keyed_groups: -- key: foreman['environment_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_') | regex_replace('none', '') - prefix: foo_group_prefixenvironment_ - separator: '' -- key: foreman['location_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_') - prefix: foo_group_prefixlocation_ - separator: '' -- key: foreman['organization_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_') - prefix: foo_group_prefixorganization_ - separator: '' -- key: foreman['content_facet_attributes']['lifecycle_environment_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_') - prefix: foo_group_prefixlifecycle_environment_ - separator: '' -- key: foreman['content_facet_attributes']['content_view_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_') - prefix: foo_group_prefixcontent_view_ - separator: '' -- key: '"%s-%s-%s" | format(app, tier, color)' - separator: '' -- key: '"%s-%s" | format(app, color)' - separator: '' -legacy_hostvars: true -plugin: theforeman.foreman.foreman -validate_certs: false -want_facts: true -want_hostcollections: true -want_params: true diff --git a/awx/main/tests/data/inventory/plugins/tower/files/tower.yml b/awx/main/tests/data/inventory/plugins/tower/files/tower.yml deleted file mode 100644 index 2c41f1b55d..0000000000 --- a/awx/main/tests/data/inventory/plugins/tower/files/tower.yml +++ /dev/null @@ -1,3 +0,0 @@ -include_metadata: true -inventory_id: 42 -plugin: awx.awx.tower diff --git a/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml b/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml deleted file mode 100644 index ac1db9f4cf..0000000000 --- a/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml +++ /dev/null @@ -1,55 +0,0 @@ -compose: - ansible_host: guest.ipAddress - ansible_ssh_host: guest.ipAddress - ansible_uuid: 99999999 | random | to_uuid - availablefield: availableField - configissue: configIssue - configstatus: configStatus - customvalue: customValue - effectiverole: effectiveRole - guestheartbeatstatus: guestHeartbeatStatus - layoutex: layoutEx - overallstatus: overallStatus - parentvapp: parentVApp - recenttask: recentTask - resourcepool: resourcePool - rootsnapshot: rootSnapshot - triggeredalarmstate: triggeredAlarmState -filters: -- config.zoo == "DC0_H0_VM0" -hostnames: -- config.foo -keyed_groups: -- key: config.asdf - prefix: '' - separator: '' -plugin: community.vmware.vmware_vm_inventory -properties: -- availableField -- configIssue -- configStatus -- customValue -- datastore -- effectiveRole -- guestHeartbeatStatus -- layout -- layoutEx -- name -- network -- overallStatus -- parentVApp -- permission -- recentTask -- resourcePool -- rootSnapshot -- snapshot -- triggeredAlarmState -- value -- capability -- config -- guest -- runtime -- storage -- summary -strict: false -with_nested_properties: true diff --git a/awx/main/tests/functional/api/test_inventory.py b/awx/main/tests/functional/api/test_inventory.py index e63286f7e8..8eda9cddd0 100644 --- a/awx/main/tests/functional/api/test_inventory.py +++ b/awx/main/tests/functional/api/test_inventory.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import pytest +import json from unittest import mock from django.core.exceptions import ValidationError @@ -8,8 +9,6 @@ from awx.api.versioning import reverse from awx.main.models import InventorySource, Inventory, ActivityStream -import json - @pytest.fixture def scm_inventory(inventory, project): @@ -457,6 +456,56 @@ def test_inventory_source_vars_prohibition(post, inventory, admin_user): assert 'FOOBAR' in r.data['source_vars'][0] +@pytest.mark.django_db +@pytest.mark.parametrize('source,source_var_actual,source_var_expected,description', [ + ('ec2', {'plugin': 'blah'}, {'plugin': 'amazon.aws.aws_ec2'}, 'source plugin mismatch'), + ('ec2', {'plugin': 'amazon.aws.aws_ec2'}, {'plugin': 'amazon.aws.aws_ec2'}, 'valid plugin'), +]) +def test_inventory_source_vars_source_plugin_ok(post, inventory, admin_user, source, source_var_actual, source_var_expected, description): + r = post(reverse('api:inventory_source_list'), + {'name': 'new inv src', 'source_vars': json.dumps(source_var_actual), 'inventory': inventory.pk, 'source': source}, + admin_user, expect=201) + + assert r.data['source_vars'] == json.dumps(source_var_expected) + + +@pytest.mark.django_db +@pytest.mark.parametrize('source_var_actual,description', [ + ({'plugin': 'namespace.collection.script'}, 'valid scm user plugin'), +]) +def test_inventory_source_vars_source_plugin_scm_ok(post, inventory, admin_user, project, source_var_actual, description): + r = post(reverse('api:inventory_source_list'), + {'name': 'new inv src', + 'source_vars': json.dumps(source_var_actual), + 'inventory': inventory.pk, + 'source': 'scm', + 'source_project': project.id,}, + admin_user, expect=201) + + assert r.data['source_vars'] == json.dumps(source_var_actual) + + +@pytest.mark.django_db +@pytest.mark.parametrize('source_var_actual,err_msg,description', [ + ({'foo': 'bar'}, 'plugin: must be present and of the form namespace.collection.inv_plugin', 'no plugin line'), + ({'plugin': ''}, 'plugin: must be of the form namespace.collection.inv_plugin', 'blank plugin line'), + ({'plugin': '.'}, 'plugin: must be of the form namespace.collection.inv_plugin', 'missing namespace, collection name, and inventory plugin'), + ({'plugin': 'a.'}, 'plugin: must be of the form namespace.collection.inv_plugin', 'missing collection name and inventory plugin'), + ({'plugin': 'a.b'}, 'plugin: must be of the form namespace.collection.inv_plugin', 'missing inventory plugin'), + ({'plugin': 'a.b.'}, 'plugin: must be of the form namespace.collection.inv_plugin', 'missing inventory plugin'), +]) +def test_inventory_source_vars_source_plugin_scm_invalid(post, inventory, admin_user, project, source_var_actual, err_msg, description): + r = post(reverse('api:inventory_source_list'), + {'name': 'new inv src', + 'source_vars': json.dumps(source_var_actual), + 'inventory': inventory.pk, + 'source': 'scm', + 'source_project': project.id,}, + admin_user, expect=400) + + assert err_msg in r.data['source_vars'][0] + + @pytest.mark.django_db @pytest.mark.parametrize('role,expect', [ ('admin_role', 200), @@ -522,7 +571,8 @@ class TestInventorySourceCredential: data={ 'inventory': inventory.pk, 'name': 'fobar', 'source': 'scm', 'source_project': project.pk, 'source_path': '', - 'credential': vault_credential.pk + 'credential': vault_credential.pk, + 'source_vars': 'plugin: a.b.c', }, expect=400, user=admin_user @@ -561,7 +611,7 @@ class TestInventorySourceCredential: data={ 'inventory': inventory.pk, 'name': 'fobar', 'source': 'scm', 'source_project': project.pk, 'source_path': '', - 'credential': os_cred.pk + 'credential': os_cred.pk, 'source_vars': 'plugin: a.b.c', }, expect=201, user=admin_user @@ -636,8 +686,14 @@ class TestControlledBySCM: assert scm_inventory.inventory_sources.count() == 0 def test_adding_inv_src_ok(self, post, scm_inventory, project, admin_user): - post(reverse('api:inventory_inventory_sources_list', kwargs={'pk': scm_inventory.id}), - {'name': 'new inv src', 'source_project': project.pk, 'update_on_project_update': False, 'source': 'scm', 'overwrite_vars': True}, + post(reverse('api:inventory_inventory_sources_list', + kwargs={'pk': scm_inventory.id}), + {'name': 'new inv src', + 'source_project': project.pk, + 'update_on_project_update': False, + 'source': 'scm', + 'overwrite_vars': True, + 'source_vars': 'plugin: a.b.c'}, admin_user, expect=201) def test_adding_inv_src_prohibited(self, post, scm_inventory, project, admin_user): @@ -657,7 +713,7 @@ class TestControlledBySCM: def test_adding_inv_src_without_proj_access_prohibited(self, post, project, inventory, rando): inventory.admin_role.members.add(rando) post(reverse('api:inventory_inventory_sources_list', kwargs={'pk': inventory.id}), - {'name': 'new inv src', 'source_project': project.pk, 'source': 'scm', 'overwrite_vars': True}, + {'name': 'new inv src', 'source_project': project.pk, 'source': 'scm', 'overwrite_vars': True, 'source_vars': 'plugin: a.b.c'}, rando, expect=403) diff --git a/awx/main/tests/functional/models/test_inventory.py b/awx/main/tests/functional/models/test_inventory.py index 6765f0e73b..2b3c747868 100644 --- a/awx/main/tests/functional/models/test_inventory.py +++ b/awx/main/tests/functional/models/test_inventory.py @@ -2,7 +2,6 @@ import pytest from unittest import mock -import json from django.core.exceptions import ValidationError @@ -259,30 +258,19 @@ class TestInventorySourceInjectors: injector = InventorySource.injectors[source]('2.7.7') assert injector.filename == filename - def test_group_by_azure(self): - injector = InventorySource.injectors['azure_rm']('2.9') - inv_src = InventorySource( - name='azure source', source='azure_rm', - source_vars={'group_by_os_family': True} - ) - group_by_on = injector.inventory_as_dict(inv_src, '/tmp/foo') - # suspicious, yes, that is just what the script did - expected_groups = 6 - assert len(group_by_on['keyed_groups']) == expected_groups - inv_src.source_vars = json.dumps({'group_by_os_family': False}) - group_by_off = injector.inventory_as_dict(inv_src, '/tmp/foo') - # much better, everyone should turn off the flag and live in the future - assert len(group_by_off['keyed_groups']) == expected_groups - 1 - - def test_tower_plugin_named_url(self): - injector = InventorySource.injectors['tower']('2.9') - inv_src = InventorySource( - name='my tower source', source='tower', - # named URL pattern "inventory++organization" - instance_filters='Designer hair 읰++Cosmetic_products䵆' - ) - result = injector.inventory_as_dict(inv_src, '/tmp/foo') - assert result['inventory_id'] == 'Designer%20hair%20%EC%9D%B0++Cosmetic_products%E4%B5%86' + @pytest.mark.parametrize('source,proper_name', [ + ('ec2', 'amazon.aws.aws_ec2'), + ('openstack', 'openstack.cloud.openstack'), + ('gce', 'google.cloud.gcp_compute'), + ('azure_rm', 'azure.azcollection.azure_rm'), + ('vmware', 'community.vmware.vmware_vm_inventory'), + ('rhv', 'ovirt.ovirt.ovirt'), + ('satellite6', 'theforeman.foreman.foreman'), + ('tower', 'awx.awx.tower'), + ]) + def test_plugin_proper_names(self, source, proper_name): + injector = InventorySource.injectors[source]('2.9') + assert injector.get_proper_name() == proper_name @pytest.mark.django_db diff --git a/awx/main/tests/functional/test_inventory_source_injectors.py b/awx/main/tests/functional/test_inventory_source_injectors.py index 392abf8535..994c2b4c11 100644 --- a/awx/main/tests/functional/test_inventory_source_injectors.py +++ b/awx/main/tests/functional/test_inventory_source_injectors.py @@ -14,69 +14,6 @@ from django.conf import settings DATA = os.path.join(os.path.dirname(data.__file__), 'inventory') -TEST_SOURCE_FIELDS = { - 'vmware': { - 'instance_filters': '{{ config.name == "only_my_server" }},{{ somevar == "bar"}}', - 'group_by': 'fouo' - }, - 'ec2': { - 'instance_filters': 'foobaa', - # group_by selected to capture some non-trivial cross-interactions - 'group_by': 'availability_zone,instance_type,tag_keys,region', - 'source_regions': 'us-east-2,ap-south-1' - }, - 'gce': { - 'source_regions': 'us-east4-a,us-west1-b' # surfaced as env var - }, - 'azure_rm': { - 'source_regions': 'southcentralus,westus' - }, - 'tower': { - 'instance_filters': '42' - } -} - -INI_TEST_VARS = { - 'ec2': { - 'boto_profile': '/tmp/my_boto_stuff', - 'iam_role_arn': 'arn:aws:iam::123456789012:role/test-role', - 'hostname_variable': 'public_dns_name', - 'destination_variable': 'public_dns_name' - }, - 'gce': {}, - 'openstack': { - 'private': False, - 'use_hostnames': False, - 'expand_hostvars': True, - 'fail_on_errors': True - }, - 'tower': {}, # there are none - 'vmware': { - 'alias_pattern': "{{ config.foo }}", - 'host_filters': '{{ config.zoo == "DC0_H0_VM0" }}', - 'groupby_patterns': "{{ config.asdf }}", - # setting VMWARE_VALIDATE_CERTS is duplicated with env var - }, - 'azure_rm': { - 'use_private_ip': True, - 'resource_groups': 'foo_resources,bar_resources', - 'tags': 'Creator:jmarshall, peanutbutter:jelly' - }, - 'satellite6': { - 'satellite6_group_patterns': '["{app}-{tier}-{color}", "{app}-{color}"]', - 'satellite6_group_prefix': 'foo_group_prefix', - 'satellite6_want_hostcollections': True, - 'satellite6_want_ansible_ssh_host': True, - 'satellite6_want_facts': True - }, - 'rhv': { # options specific to the plugin - 'ovirt_insecure': False, - 'groups': { - 'dev': '"dev" in tags' - } - } -} - def generate_fake_var(element): """Given a credential type field element, makes up something acceptable. @@ -245,25 +182,21 @@ def create_reference_data(source_dir, env, content): @pytest.mark.django_db @pytest.mark.parametrize('this_kind', CLOUD_PROVIDERS) def test_inventory_update_injected_content(this_kind, inventory, fake_credential_factory): + injector = InventorySource.injectors[this_kind] + if injector.plugin_name is None: + pytest.skip('Use of inventory plugin is not enabled for this source') + src_vars = dict(base_source_var='value_of_var') - if this_kind in INI_TEST_VARS: - src_vars.update(INI_TEST_VARS[this_kind]) - extra_kwargs = {} - if this_kind in TEST_SOURCE_FIELDS: - extra_kwargs.update(TEST_SOURCE_FIELDS[this_kind]) + src_vars['plugin'] = injector.get_proper_name() inventory_source = InventorySource.objects.create( inventory=inventory, source=this_kind, source_vars=src_vars, - **extra_kwargs ) inventory_source.credentials.add(fake_credential_factory(this_kind)) inventory_update = inventory_source.create_unified_job() task = RunInventoryUpdate() - if InventorySource.injectors[this_kind].plugin_name is None: - pytest.skip('Use of inventory plugin is not enabled for this source') - def substitute_run(envvars=None, **_kw): """This method will replace run_pexpect instead of running, it will read the private data directory contents @@ -274,6 +207,12 @@ def test_inventory_update_injected_content(this_kind, inventory, fake_credential assert envvars.pop('ANSIBLE_INVENTORY_ENABLED') == 'auto' set_files = bool(os.getenv("MAKE_INVENTORY_REFERENCE_FILES", 'false').lower()[0] not in ['f', '0']) env, content = read_content(private_data_dir, envvars, inventory_update) + + # Assert inventory plugin inventory file is in private_data_dir + inventory_filename = InventorySource.injectors[inventory_update.source]('2.9').filename + assert len([True for k in content.keys() if k.endswith(inventory_filename)]) > 0, \ + f"'{inventory_filename}' file not found in inventory update runtime files {content.keys()}" + env.pop('ANSIBLE_COLLECTIONS_PATHS', None) # collection paths not relevant to this test base_dir = os.path.join(DATA, 'plugins') if not os.path.exists(base_dir): @@ -283,6 +222,8 @@ def test_inventory_update_injected_content(this_kind, inventory, fake_credential create_reference_data(source_dir, env, content) pytest.skip('You set MAKE_INVENTORY_REFERENCE_FILES, so this created files, unset to run actual test.') else: + source_dir = os.path.join(base_dir, this_kind) # this_kind is a global + if not os.path.exists(source_dir): raise FileNotFoundError( 'Maybe you never made reference files? ' @@ -292,9 +233,6 @@ def test_inventory_update_injected_content(this_kind, inventory, fake_credential expected_file_list = os.listdir(files_dir) except FileNotFoundError: expected_file_list = [] - assert set(expected_file_list) == set(content.keys()), ( - 'Inventory update runtime environment does not have expected files' - ) for f_name in expected_file_list: with open(os.path.join(files_dir, f_name), 'r') as f: ref_content = f.read() diff --git a/awx/main/tests/unit/models/test_inventory.py b/awx/main/tests/unit/models/test_inventory.py index dc6af0e828..26ef5e1fa9 100644 --- a/awx/main/tests/unit/models/test_inventory.py +++ b/awx/main/tests/unit/models/test_inventory.py @@ -72,23 +72,6 @@ def test_invalid_kind_clean_insights_credential(): assert json.dumps(str(e.value)) == json.dumps(str([u'Assignment not allowed for Smart Inventory'])) -@pytest.mark.parametrize('source_vars,validate_certs', [ - ({'ssl_verify': True}, True), - ({'ssl_verify': False}, False), - ({'validate_certs': True}, True), - ({'validate_certs': False}, False)]) -def test_satellite_plugin_backwards_support_for_ssl_verify(source_vars, validate_certs): - injector = InventorySource.injectors['satellite6']('2.9') - inv_src = InventorySource( - name='satellite source', source='satellite6', - source_vars=source_vars - ) - - ret = injector.inventory_as_dict(inv_src, '/tmp/foo') - assert 'validate_certs' in ret - assert ret['validate_certs'] in (validate_certs, str(validate_certs)) - - class TestControlledBySCM(): def test_clean_source_path_valid(self): inv_src = InventorySource(source_path='/not_real/', diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index cb7dc49e46..93e373e401 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -8,8 +8,6 @@ from datetime import timedelta # global settings from django.conf import global_settings -# ugettext lazy -from django.utils.translation import ugettext_lazy as _ # Update this module's local settings from the global settings module. this_module = sys.modules[__name__] From 35d264d7f8217114f60c921a5801a4ba48de0e44 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Wed, 29 Jul 2020 14:29:38 -0400 Subject: [PATCH 06/27] forgot to add migration helper * move models/inventory.py plugin injector logic to a place frozen in time useable by the migration code. --- awx/main/migrations/_inventory_source_vars.py | 749 ++++++++++++++++++ 1 file changed, 749 insertions(+) create mode 100644 awx/main/migrations/_inventory_source_vars.py diff --git a/awx/main/migrations/_inventory_source_vars.py b/awx/main/migrations/_inventory_source_vars.py new file mode 100644 index 0000000000..857bb1c07e --- /dev/null +++ b/awx/main/migrations/_inventory_source_vars.py @@ -0,0 +1,749 @@ +from django.utils.translation import ugettext_lazy as _ + + +FrozenInjectors = dict() + + +class PluginFileInjector(object): + plugin_name = None # Ansible core name used to reference plugin + # every source should have collection, these are for the collection name + namespace = None + collection = None + + def inventory_as_dict(self, inventory_source, private_data_dir): + """Default implementation of inventory plugin file contents. + There are some valid cases when all parameters can be obtained from + the environment variables, example "plugin: linode" is valid + ideally, however, some options should be filled from the inventory source data + """ + if self.plugin_name is None: + raise NotImplementedError('At minimum the plugin name is needed for inventory plugin use.') + proper_name = f'{self.namespace}.{self.collection}.{self.plugin_name}' + return {'plugin': proper_name} + + +class azure_rm(PluginFileInjector): + plugin_name = 'azure_rm' + namespace = 'azure' + collection = 'azcollection' + + def inventory_as_dict(self, inventory_source, private_data_dir): + ret = super(azure_rm, self).inventory_as_dict(inventory_source, private_data_dir) + + source_vars = inventory_source.source_vars_dict + + ret['fail_on_template_errors'] = False + + group_by_hostvar = { + 'location': {'prefix': '', 'separator': '', 'key': 'location'}, + 'tag': {'prefix': '', 'separator': '', 'key': 'tags.keys() | list if tags else []'}, + # Introduced with https://github.com/ansible/ansible/pull/53046 + 'security_group': {'prefix': '', 'separator': '', 'key': 'security_group'}, + 'resource_group': {'prefix': '', 'separator': '', 'key': 'resource_group'}, + # Note, os_family was not documented correctly in script, but defaulted to grouping by it + 'os_family': {'prefix': '', 'separator': '', 'key': 'os_disk.operating_system_type'} + } + # by default group by everything + # always respect user setting, if they gave it + group_by = [ + grouping_name for grouping_name in group_by_hostvar + if source_vars.get('group_by_{}'.format(grouping_name), True) + ] + ret['keyed_groups'] = [group_by_hostvar[grouping_name] for grouping_name in group_by] + if 'tag' in group_by: + # Nasty syntax to reproduce "key_value" group names in addition to "key" + ret['keyed_groups'].append({ + 'prefix': '', 'separator': '', + 'key': r'dict(tags.keys() | map("regex_replace", "^(.*)$", "\1_") | list | zip(tags.values() | list)) if tags else []' + }) + + # Compatibility content + # TODO: add proper support for instance_filters non-specific to compatibility + # TODO: add proper support for group_by non-specific to compatibility + # Dashes were not configurable in azure_rm.py script, we do not want unicode, so always use this + ret['use_contrib_script_compatible_sanitization'] = True + # use same host names as script + ret['plain_host_names'] = True + # By default the script did not filter hosts + ret['default_host_filters'] = [] + # User-given host filters + user_filters = [] + old_filterables = [ + ('resource_groups', 'resource_group'), + ('tags', 'tags') + # locations / location would be an entry + # but this would conflict with source_regions + ] + for key, loc in old_filterables: + value = source_vars.get(key, None) + if value and isinstance(value, str): + # tags can be list of key:value pairs + # e.g. 'Creator:jmarshall, peanutbutter:jelly' + # or tags can be a list of keys + # e.g. 'Creator, peanutbutter' + if key == "tags": + # grab each key value pair + for kvpair in value.split(','): + # split into key and value + kv = kvpair.split(':') + # filter out any host that does not have key + # in their tags.keys() variable + user_filters.append('"{}" not in tags.keys()'.format(kv[0].strip())) + # if a value is provided, check that the key:value pair matches + if len(kv) > 1: + user_filters.append('tags["{}"] != "{}"'.format(kv[0].strip(), kv[1].strip())) + else: + user_filters.append('{} not in {}'.format( + loc, value.split(',') + )) + if user_filters: + ret.setdefault('exclude_host_filters', []) + ret['exclude_host_filters'].extend(user_filters) + + ret['conditional_groups'] = {'azure': True} + ret['hostvar_expressions'] = { + 'provisioning_state': 'provisioning_state | title', + 'computer_name': 'name', + 'type': 'resource_type', + 'private_ip': 'private_ipv4_addresses[0] if private_ipv4_addresses else None', + 'public_ip': 'public_ipv4_addresses[0] if public_ipv4_addresses else None', + 'public_ip_name': 'public_ip_name if public_ip_name is defined else None', + 'public_ip_id': 'public_ip_id if public_ip_id is defined else None', + 'tags': 'tags if tags else None' + } + # Special functionality from script + if source_vars.get('use_private_ip', False): + ret['hostvar_expressions']['ansible_host'] = 'private_ipv4_addresses[0]' + # end compatibility content + + if inventory_source.source_regions and 'all' not in inventory_source.source_regions: + # initialize a list for this section in inventory file + ret.setdefault('exclude_host_filters', []) + # make a python list of the regions we will use + python_regions = [x.strip() for x in inventory_source.source_regions.split(',')] + # convert that list in memory to python syntax in a string + # now put that in jinja2 syntax operating on hostvar key "location" + # and put that as an entry in the exclusions list + ret['exclude_host_filters'].append("location not in {}".format(repr(python_regions))) + return ret + +class ec2(PluginFileInjector): + plugin_name = 'aws_ec2' + namespace = 'amazon' + collection = 'aws' + + + def _get_ec2_group_by_choices(self): + return [ + ('ami_id', _('Image ID')), + ('availability_zone', _('Availability Zone')), + ('aws_account', _('Account')), + ('instance_id', _('Instance ID')), + ('instance_state', _('Instance State')), + ('platform', _('Platform')), + ('instance_type', _('Instance Type')), + ('key_pair', _('Key Name')), + ('region', _('Region')), + ('security_group', _('Security Group')), + ('tag_keys', _('Tags')), + ('tag_none', _('Tag None')), + ('vpc_id', _('VPC ID')), + ] + + def _compat_compose_vars(self): + return { + # vars that change + 'ec2_block_devices': ( + "dict(block_device_mappings | map(attribute='device_name') | list | zip(block_device_mappings " + "| map(attribute='ebs.volume_id') | list))" + ), + 'ec2_dns_name': 'public_dns_name', + 'ec2_group_name': 'placement.group_name', + 'ec2_instance_profile': 'iam_instance_profile | default("")', + 'ec2_ip_address': 'public_ip_address', + 'ec2_kernel': 'kernel_id | default("")', + 'ec2_monitored': "monitoring.state in ['enabled', 'pending']", + 'ec2_monitoring_state': 'monitoring.state', + 'ec2_placement': 'placement.availability_zone', + 'ec2_ramdisk': 'ramdisk_id | default("")', + 'ec2_reason': 'state_transition_reason', + 'ec2_security_group_ids': "security_groups | map(attribute='group_id') | list | join(',')", + 'ec2_security_group_names': "security_groups | map(attribute='group_name') | list | join(',')", + 'ec2_tag_Name': 'tags.Name', + 'ec2_state': 'state.name', + 'ec2_state_code': 'state.code', + 'ec2_state_reason': 'state_reason.message if state_reason is defined else ""', + 'ec2_sourceDestCheck': 'source_dest_check | default(false) | lower | string', # snake_case syntax intended + 'ec2_account_id': 'owner_id', + # vars that just need ec2_ prefix + 'ec2_ami_launch_index': 'ami_launch_index | string', + 'ec2_architecture': 'architecture', + 'ec2_client_token': 'client_token', + 'ec2_ebs_optimized': 'ebs_optimized', + 'ec2_hypervisor': 'hypervisor', + 'ec2_image_id': 'image_id', + 'ec2_instance_type': 'instance_type', + 'ec2_key_name': 'key_name', + 'ec2_launch_time': r'launch_time | regex_replace(" ", "T") | regex_replace("(\+)(\d\d):(\d)(\d)$", ".\g<2>\g<3>Z")', + 'ec2_platform': 'platform | default("")', + 'ec2_private_dns_name': 'private_dns_name', + 'ec2_private_ip_address': 'private_ip_address', + 'ec2_public_dns_name': 'public_dns_name', + 'ec2_region': 'placement.region', + 'ec2_root_device_name': 'root_device_name', + 'ec2_root_device_type': 'root_device_type', + # many items need blank defaults because the script tended to keep a common schema + 'ec2_spot_instance_request_id': 'spot_instance_request_id | default("")', + 'ec2_subnet_id': 'subnet_id | default("")', + 'ec2_virtualization_type': 'virtualization_type', + 'ec2_vpc_id': 'vpc_id | default("")', + # same as ec2_ip_address, the script provided this + 'ansible_host': 'public_ip_address', + # new with https://github.com/ansible/ansible/pull/53645 + 'ec2_eventsSet': 'events | default("")', + 'ec2_persistent': 'persistent | default(false)', + 'ec2_requester_id': 'requester_id | default("")' + } + + def inventory_as_dict(self, inventory_source, private_data_dir): + ret = super(ec2, self).inventory_as_dict(inventory_source, private_data_dir) + + keyed_groups = [] + group_by_hostvar = { + 'ami_id': {'prefix': '', 'separator': '', 'key': 'image_id', 'parent_group': 'images'}, + # 2 entries for zones for same groups to establish 2 parentage trees + 'availability_zone': {'prefix': '', 'separator': '', 'key': 'placement.availability_zone', 'parent_group': 'zones'}, + 'aws_account': {'prefix': '', 'separator': '', 'key': 'ec2_account_id', 'parent_group': 'accounts'}, # composed var + 'instance_id': {'prefix': '', 'separator': '', 'key': 'instance_id', 'parent_group': 'instances'}, # normally turned off + 'instance_state': {'prefix': 'instance_state', 'key': 'ec2_state', 'parent_group': 'instance_states'}, # composed var + # ec2_platform is a composed var, but group names do not match up to hostvar exactly + 'platform': {'prefix': 'platform', 'key': 'platform | default("undefined")', 'parent_group': 'platforms'}, + 'instance_type': {'prefix': 'type', 'key': 'instance_type', 'parent_group': 'types'}, + 'key_pair': {'prefix': 'key', 'key': 'key_name', 'parent_group': 'keys'}, + 'region': {'prefix': '', 'separator': '', 'key': 'placement.region', 'parent_group': 'regions'}, + # Security requires some ninja jinja2 syntax, credit to s-hertel + 'security_group': {'prefix': 'security_group', 'key': 'security_groups | map(attribute="group_name")', 'parent_group': 'security_groups'}, + # tags cannot be parented in exactly the same way as the script due to + # https://github.com/ansible/ansible/pull/53812 + 'tag_keys': [ + {'prefix': 'tag', 'key': 'tags', 'parent_group': 'tags'}, + {'prefix': 'tag', 'key': 'tags.keys()', 'parent_group': 'tags'} + ], + # 'tag_none': None, # grouping by no tags isn't a different thing with plugin + # naming is redundant, like vpc_id_vpc_8c412cea, but intended + 'vpc_id': {'prefix': 'vpc_id', 'key': 'vpc_id', 'parent_group': 'vpcs'}, + } + # -- same-ish as script here -- + group_by = [x.strip().lower() for x in inventory_source.group_by.split(',') if x.strip()] + for choice in self._get_ec2_group_by_choices(): + value = bool((group_by and choice[0] in group_by) or (not group_by and choice[0] != 'instance_id')) + # -- end sameness to script -- + if value: + this_keyed_group = group_by_hostvar.get(choice[0], None) + # If a keyed group syntax does not exist, there is nothing we can do to get this group + if this_keyed_group is not None: + if isinstance(this_keyed_group, list): + keyed_groups.extend(this_keyed_group) + else: + keyed_groups.append(this_keyed_group) + # special case, this parentage is only added if both zones and regions are present + if not group_by or ('region' in group_by and 'availability_zone' in group_by): + keyed_groups.append({'prefix': '', 'separator': '', 'key': 'placement.availability_zone', 'parent_group': '{{ placement.region }}'}) + + source_vars = inventory_source.source_vars_dict + # This is a setting from the script, hopefully no one used it + # if true, it replaces dashes, but not in region / loc names + replace_dash = bool(source_vars.get('replace_dash_in_groups', True)) + # Compatibility content + legacy_regex = { + True: r"[^A-Za-z0-9\_]", + False: r"[^A-Za-z0-9\_\-]" # do not replace dash, dash is allowed + }[replace_dash] + list_replacer = 'map("regex_replace", "{rx}", "_") | list'.format(rx=legacy_regex) + # this option, a plugin option, will allow dashes, but not unicode + # when set to False, unicode will be allowed, but it was not allowed by script + # thus, we always have to use this option, and always use our custom regex + ret['use_contrib_script_compatible_sanitization'] = True + for grouping_data in keyed_groups: + if grouping_data['key'] in ('placement.region', 'placement.availability_zone'): + # us-east-2 is always us-east-2 according to ec2.py + # no sanitization in region-ish groups for the script standards, ever ever + continue + if grouping_data['key'] == 'tags': + # dict jinja2 transformation + grouping_data['key'] = 'dict(tags.keys() | {replacer} | zip(tags.values() | {replacer}))'.format( + replacer=list_replacer + ) + elif grouping_data['key'] == 'tags.keys()' or grouping_data['prefix'] == 'security_group': + # list jinja2 transformation + grouping_data['key'] += ' | {replacer}'.format(replacer=list_replacer) + else: + # string transformation + grouping_data['key'] += ' | regex_replace("{rx}", "_")'.format(rx=legacy_regex) + # end compatibility content + + if source_vars.get('iam_role_arn', None): + ret['iam_role_arn'] = source_vars['iam_role_arn'] + + # This was an allowed ec2.ini option, also plugin option, so pass through + if source_vars.get('boto_profile', None): + ret['boto_profile'] = source_vars['boto_profile'] + + elif not replace_dash: + # Using the plugin, but still want dashes allowed + ret['use_contrib_script_compatible_sanitization'] = True + + if source_vars.get('nested_groups') is False: + for this_keyed_group in keyed_groups: + this_keyed_group.pop('parent_group', None) + + if keyed_groups: + ret['keyed_groups'] = keyed_groups + + # Instance ID not part of compat vars, because of settings.EC2_INSTANCE_ID_VAR + compose_dict = {'ec2_id': 'instance_id'} + inst_filters = {} + + # Compatibility content + compose_dict.update(self._compat_compose_vars()) + # plugin provides "aws_ec2", but not this which the script gave + ret['groups'] = {'ec2': True} + if source_vars.get('hostname_variable') is not None: + hnames = [] + for expr in source_vars.get('hostname_variable').split(','): + if expr == 'public_dns_name': + hnames.append('dns-name') + elif not expr.startswith('tag:') and '_' in expr: + hnames.append(expr.replace('_', '-')) + else: + hnames.append(expr) + ret['hostnames'] = hnames + else: + # public_ip as hostname is non-default plugin behavior, script behavior + ret['hostnames'] = [ + 'network-interface.addresses.association.public-ip', + 'dns-name', + 'private-dns-name' + ] + # The script returned only running state by default, the plugin does not + # https://docs.aws.amazon.com/cli/latest/reference/ec2/describe-instances.html#options + # options: pending | running | shutting-down | terminated | stopping | stopped + inst_filters['instance-state-name'] = ['running'] + # end compatibility content + + if source_vars.get('destination_variable') or source_vars.get('vpc_destination_variable'): + for fd in ('destination_variable', 'vpc_destination_variable'): + if source_vars.get(fd): + compose_dict['ansible_host'] = source_vars.get(fd) + break + + if compose_dict: + ret['compose'] = compose_dict + + if inventory_source.instance_filters: + # logic used to live in ec2.py, now it belongs to us. Yay more code? + filter_sets = [f for f in inventory_source.instance_filters.split(',') if f] + + for instance_filter in filter_sets: + # AND logic not supported, unclear how to... + instance_filter = instance_filter.strip() + if not instance_filter or '=' not in instance_filter: + continue + filter_key, filter_value = [x.strip() for x in instance_filter.split('=', 1)] + if not filter_key: + continue + inst_filters[filter_key] = filter_value + + if inst_filters: + ret['filters'] = inst_filters + + if inventory_source.source_regions and 'all' not in inventory_source.source_regions: + ret['regions'] = inventory_source.source_regions.split(',') + + return ret + + +class gce(PluginFileInjector): + plugin_name = 'gcp_compute' + namespace = 'google' + collection = 'cloud' + + def _compat_compose_vars(self): + # missing: gce_image, gce_uuid + # https://github.com/ansible/ansible/issues/51884 + return { + 'gce_description': 'description if description else None', + 'gce_machine_type': 'machineType', + 'gce_name': 'name', + 'gce_network': 'networkInterfaces[0].network.name', + 'gce_private_ip': 'networkInterfaces[0].networkIP', + 'gce_public_ip': 'networkInterfaces[0].accessConfigs[0].natIP | default(None)', + 'gce_status': 'status', + 'gce_subnetwork': 'networkInterfaces[0].subnetwork.name', + 'gce_tags': 'tags.get("items", [])', + 'gce_zone': 'zone', + 'gce_metadata': 'metadata.get("items", []) | items2dict(key_name="key", value_name="value")', + # NOTE: image hostvar is enabled via retrieve_image_info option + 'gce_image': 'image', + # We need this as long as hostnames is non-default, otherwise hosts + # will not be addressed correctly, was returned in script + 'ansible_ssh_host': 'networkInterfaces[0].accessConfigs[0].natIP | default(networkInterfaces[0].networkIP)' + } + + def inventory_as_dict(self, inventory_source, private_data_dir): + ret = super(gce, self).inventory_as_dict(inventory_source, private_data_dir) + + # auth related items + ret['auth_kind'] = "serviceaccount" + + filters = [] + # TODO: implement gce group_by options + # gce never processed the group_by field, if it had, we would selectively + # apply those options here, but it did not, so all groups are added here + keyed_groups = [ + # the jinja2 syntax is duplicated with compose + # https://github.com/ansible/ansible/issues/51883 + {'prefix': 'network', 'key': 'gce_subnetwork'}, # composed var + {'prefix': '', 'separator': '', 'key': 'gce_private_ip'}, # composed var + {'prefix': '', 'separator': '', 'key': 'gce_public_ip'}, # composed var + {'prefix': '', 'separator': '', 'key': 'machineType'}, + {'prefix': '', 'separator': '', 'key': 'zone'}, + {'prefix': 'tag', 'key': 'gce_tags'}, # composed var + {'prefix': 'status', 'key': 'status | lower'}, + # NOTE: image hostvar is enabled via retrieve_image_info option + {'prefix': '', 'separator': '', 'key': 'image'}, + ] + # This will be used as the gce instance_id, must be universal, non-compat + compose_dict = {'gce_id': 'id'} + + # Compatibility content + # TODO: proper group_by and instance_filters support, irrelevant of compat mode + # The gce.py script never sanitized any names in any way + ret['use_contrib_script_compatible_sanitization'] = True + # Perform extra API query to get the image hostvar + ret['retrieve_image_info'] = True + # Add in old hostvars aliases + compose_dict.update(self._compat_compose_vars()) + # Non-default names to match script + ret['hostnames'] = ['name', 'public_ip', 'private_ip'] + # end compatibility content + + if keyed_groups: + ret['keyed_groups'] = keyed_groups + if filters: + ret['filters'] = filters + if compose_dict: + ret['compose'] = compose_dict + if inventory_source.source_regions and 'all' not in inventory_source.source_regions: + ret['zones'] = inventory_source.source_regions.split(',') + return ret + + +class vmware(PluginFileInjector): + plugin_name = 'vmware_vm_inventory' + namespace = 'community' + collection = 'vmware' + + def inventory_as_dict(self, inventory_source, private_data_dir): + ret = super(vmware, self).inventory_as_dict(inventory_source, private_data_dir) + ret['strict'] = False + # Documentation of props, see + # https://github.com/ansible/ansible/blob/devel/docs/docsite/rst/scenario_guides/vmware_scenarios/vmware_inventory_vm_attributes.rst + UPPERCASE_PROPS = [ + "availableField", + "configIssue", + "configStatus", + "customValue", # optional + "datastore", + "effectiveRole", + "guestHeartbeatStatus", # optional + "layout", # optional + "layoutEx", # optional + "name", + "network", + "overallStatus", + "parentVApp", # optional + "permission", + "recentTask", + "resourcePool", + "rootSnapshot", + "snapshot", # optional + "triggeredAlarmState", + "value" + ] + NESTED_PROPS = [ + "capability", + "config", + "guest", + "runtime", + "storage", + "summary", # repeat of other properties + ] + ret['properties'] = UPPERCASE_PROPS + NESTED_PROPS + ret['compose'] = {'ansible_host': 'guest.ipAddress'} # default value + ret['compose']['ansible_ssh_host'] = ret['compose']['ansible_host'] + # the ansible_uuid was unique every host, every import, from the script + ret['compose']['ansible_uuid'] = '99999999 | random | to_uuid' + for prop in UPPERCASE_PROPS: + if prop == prop.lower(): + continue + ret['compose'][prop.lower()] = prop + ret['with_nested_properties'] = True + # ret['property_name_format'] = 'lower_case' # only dacrystal/topic/vmware-inventory-plugin-property-format + + # process custom options + vmware_opts = dict(inventory_source.source_vars_dict.items()) + if inventory_source.instance_filters: + vmware_opts.setdefault('host_filters', inventory_source.instance_filters) + if inventory_source.group_by: + vmware_opts.setdefault('groupby_patterns', inventory_source.group_by) + + alias_pattern = vmware_opts.get('alias_pattern') + if alias_pattern: + ret.setdefault('hostnames', []) + for alias in alias_pattern.split(','): # make best effort + striped_alias = alias.replace('{', '').replace('}', '').strip() # make best effort + if not striped_alias: + continue + ret['hostnames'].append(striped_alias) + + host_pattern = vmware_opts.get('host_pattern') # not working in script + if host_pattern: + stripped_hp = host_pattern.replace('{', '').replace('}', '').strip() # make best effort + ret['compose']['ansible_host'] = stripped_hp + ret['compose']['ansible_ssh_host'] = stripped_hp + + host_filters = vmware_opts.get('host_filters') + if host_filters: + ret.setdefault('filters', []) + for hf in host_filters.split(','): + striped_hf = hf.replace('{', '').replace('}', '').strip() # make best effort + if not striped_hf: + continue + ret['filters'].append(striped_hf) + else: + # default behavior filters by power state + ret['filters'] = ['runtime.powerState == "poweredOn"'] + + groupby_patterns = vmware_opts.get('groupby_patterns') + ret.setdefault('keyed_groups', []) + if groupby_patterns: + for pattern in groupby_patterns.split(','): + stripped_pattern = pattern.replace('{', '').replace('}', '').strip() # make best effort + ret['keyed_groups'].append({ + 'prefix': '', 'separator': '', + 'key': stripped_pattern + }) + else: + # default groups from script + for entry in ('config.guestId', '"templates" if config.template else "guests"'): + ret['keyed_groups'].append({ + 'prefix': '', 'separator': '', + 'key': entry + }) + + return ret + + +class openstack(PluginFileInjector): + plugin_name = 'openstack' + namespace = 'openstack' + collection = 'cloud' + + def inventory_as_dict(self, inventory_source, private_data_dir): + def use_host_name_for_name(a_bool_maybe): + if not isinstance(a_bool_maybe, bool): + # Could be specified by user via "host" or "uuid" + return a_bool_maybe + elif a_bool_maybe: + return 'name' # plugin default + else: + return 'uuid' + + ret = super(openstack, self).inventory_as_dict(inventory_source, private_data_dir) + ret['fail_on_errors'] = True + ret['expand_hostvars'] = True + ret['inventory_hostname'] = use_host_name_for_name(False) + # Note: mucking with defaults will break import integrity + # For the plugin, we need to use the same defaults as the old script + # or else imports will conflict. To find script defaults you have + # to read source code of the script. + # + # Script Defaults Plugin Defaults + # 'use_hostnames': False, 'name' (True) + # 'expand_hostvars': True, 'no' (False) + # 'fail_on_errors': True, 'no' (False) + # + # These are, yet again, different from ansible_variables in script logic + # but those are applied inconsistently + source_vars = inventory_source.source_vars_dict + for var_name in ['expand_hostvars', 'fail_on_errors']: + if var_name in source_vars: + ret[var_name] = source_vars[var_name] + if 'use_hostnames' in source_vars: + ret['inventory_hostname'] = use_host_name_for_name(source_vars['use_hostnames']) + return ret + +class rhv(PluginFileInjector): + """ovirt uses the custom credential templating, and that is all + """ + plugin_name = 'ovirt' + initial_version = '2.9' + namespace = 'ovirt' + collection = 'ovirt' + + def inventory_as_dict(self, inventory_source, private_data_dir): + ret = super(rhv, self).inventory_as_dict(inventory_source, private_data_dir) + ret['ovirt_insecure'] = False # Default changed from script + # TODO: process strict option upstream + ret['compose'] = { + 'ansible_host': '(devices.values() | list)[0][0] if devices else None' + } + ret['keyed_groups'] = [] + for key in ('cluster', 'status'): + ret['keyed_groups'].append({'prefix': key, 'separator': '_', 'key': key}) + ret['keyed_groups'].append({'prefix': 'tag', 'separator': '_', 'key': 'tags'}) + ret['ovirt_hostname_preference'] = ['name', 'fqdn'] + source_vars = inventory_source.source_vars_dict + for key, value in source_vars.items(): + if key == 'plugin': + continue + ret[key] = value + return ret + + +class satellite6(PluginFileInjector): + plugin_name = 'foreman' + namespace = 'theforeman' + collection = 'foreman' + + def inventory_as_dict(self, inventory_source, private_data_dir): + ret = super(satellite6, self).inventory_as_dict(inventory_source, private_data_dir) + ret['validate_certs'] = False + + group_patterns = '[]' + group_prefix = 'foreman_' + want_hostcollections = False + want_ansible_ssh_host = False + want_facts = True + + foreman_opts = inventory_source.source_vars_dict.copy() + for k, v in foreman_opts.items(): + if k == 'satellite6_group_patterns' and isinstance(v, str): + group_patterns = v + elif k == 'satellite6_group_prefix' and isinstance(v, str): + group_prefix = v + elif k == 'satellite6_want_hostcollections' and isinstance(v, bool): + want_hostcollections = v + elif k == 'satellite6_want_ansible_ssh_host' and isinstance(v, bool): + want_ansible_ssh_host = v + elif k == 'satellite6_want_facts' and isinstance(v, bool): + want_facts = v + # add backwards support for ssl_verify + # plugin uses new option, validate_certs, instead + elif k == 'ssl_verify' and isinstance(v, bool): + ret['validate_certs'] = v + else: + ret[k] = str(v) + + # Compatibility content + group_by_hostvar = { + "environment": {"prefix": "{}environment_".format(group_prefix), + "separator": "", + "key": "foreman['environment_name'] | lower | regex_replace(' ', '') | " + "regex_replace('[^A-Za-z0-9_]', '_') | regex_replace('none', '')"}, + "location": {"prefix": "{}location_".format(group_prefix), + "separator": "", + "key": "foreman['location_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_')"}, + "organization": {"prefix": "{}organization_".format(group_prefix), + "separator": "", + "key": "foreman['organization_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_')"}, + "lifecycle_environment": {"prefix": "{}lifecycle_environment_".format(group_prefix), + "separator": "", + "key": "foreman['content_facet_attributes']['lifecycle_environment_name'] | " + "lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_')"}, + "content_view": {"prefix": "{}content_view_".format(group_prefix), + "separator": "", + "key": "foreman['content_facet_attributes']['content_view_name'] | " + "lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_')"} + } + + ret['legacy_hostvars'] = True # convert hostvar structure to the form used by the script + ret['want_params'] = True + ret['group_prefix'] = group_prefix + ret['want_hostcollections'] = want_hostcollections + ret['want_facts'] = want_facts + + if want_ansible_ssh_host: + ret['compose'] = {'ansible_ssh_host': "foreman['ip6'] | default(foreman['ip'], true)"} + ret['keyed_groups'] = [group_by_hostvar[grouping_name] for grouping_name in group_by_hostvar] + + def form_keyed_group(group_pattern): + """ + Converts foreman group_pattern to + inventory plugin keyed_group + + e.g. {app_param}-{tier_param}-{dc_param} + becomes + "%s-%s-%s" | format(app_param, tier_param, dc_param) + """ + if type(group_pattern) is not str: + return None + params = re.findall('{[^}]*}', group_pattern) + if len(params) == 0: + return None + + param_names = [] + for p in params: + param_names.append(p[1:-1].strip()) # strip braces and space + + # form keyed_group key by + # replacing curly braces with '%s' + # (for use with jinja's format filter) + key = group_pattern + for p in params: + key = key.replace(p, '%s', 1) + + # apply jinja filter to key + key = '"{}" | format({})'.format(key, ', '.join(param_names)) + + keyed_group = {'key': key, + 'separator': ''} + return keyed_group + + try: + group_patterns = json.loads(group_patterns) + + if type(group_patterns) is list: + for group_pattern in group_patterns: + keyed_group = form_keyed_group(group_pattern) + if keyed_group: + ret['keyed_groups'].append(keyed_group) + except json.JSONDecodeError: + logger.warning('Could not parse group_patterns. Expected JSON-formatted string, found: {}' + .format(group_patterns)) + + return ret + + +class tower(PluginFileInjector): + plugin_name = 'tower' + namespace = 'awx' + collection = 'awx' + + def inventory_as_dict(self, inventory_source, private_data_dir): + ret = super(tower, self).inventory_as_dict(inventory_source, private_data_dir) + # Credentials injected as env vars, same as script + try: + # plugin can take an actual int type + identifier = int(inventory_source.instance_filters) + except ValueError: + # inventory_id could be a named URL + identifier = iri_to_uri(inventory_source.instance_filters) + ret['inventory_id'] = identifier + ret['include_metadata'] = True # used for license check + return ret + + +for cls in PluginFileInjector.__subclasses__(): + FrozenInjectors[cls.__name__] = cls From b7efad564047e16759aca2cd1449c5117adb820d Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Wed, 29 Jul 2020 15:26:17 -0400 Subject: [PATCH 07/27] do not enforce plugin: for source=scm * InventorySource w/ source type scm point to an inventory file via source_file. source_vars are ignored. --- awx/main/models/inventory.py | 7 ----- .../tests/functional/api/test_inventory.py | 26 +++---------------- 2 files changed, 4 insertions(+), 29 deletions(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 7d6c350274..7d9069e4f0 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -1163,13 +1163,6 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions, CustomVirtualE source_vars = dict(self.source_vars_dict) # make a copy if injector and self.source_vars_dict.get('plugin', '') != injector.get_proper_name(): source_vars['plugin'] = injector.get_proper_name() - elif not injector: - source_vars = dict(self.source_vars_dict) # make a copy - collection_pattern = re.compile("^(.+)\.(.+)\.(.+)$") # noqa - if 'plugin' not in source_vars: - raise ValidationError(_("plugin: must be present and of the form namespace.collection.inv_plugin")) - elif not bool(collection_pattern.match(source_vars['plugin'])): - raise ValidationError(_("plugin: must be of the form namespace.collection.inv_plugin")) return json.dumps(source_vars) ''' diff --git a/awx/main/tests/functional/api/test_inventory.py b/awx/main/tests/functional/api/test_inventory.py index 8eda9cddd0..a04a68c9bb 100644 --- a/awx/main/tests/functional/api/test_inventory.py +++ b/awx/main/tests/functional/api/test_inventory.py @@ -471,7 +471,10 @@ def test_inventory_source_vars_source_plugin_ok(post, inventory, admin_user, sou @pytest.mark.django_db @pytest.mark.parametrize('source_var_actual,description', [ - ({'plugin': 'namespace.collection.script'}, 'valid scm user plugin'), + ({'plugin': 'namespace.collection.script'}, 'scm source type source_vars are ignored valid'), + ({'plugin': 'namespace.collection.script'}, 'scm source type source_vars are ignored invalid'), + ({'plugin': ''}, 'scm source type source_vars are ignored blank'), + ({}, 'scm source type source_vars are ignored non-existent'), ]) def test_inventory_source_vars_source_plugin_scm_ok(post, inventory, admin_user, project, source_var_actual, description): r = post(reverse('api:inventory_source_list'), @@ -485,27 +488,6 @@ def test_inventory_source_vars_source_plugin_scm_ok(post, inventory, admin_user, assert r.data['source_vars'] == json.dumps(source_var_actual) -@pytest.mark.django_db -@pytest.mark.parametrize('source_var_actual,err_msg,description', [ - ({'foo': 'bar'}, 'plugin: must be present and of the form namespace.collection.inv_plugin', 'no plugin line'), - ({'plugin': ''}, 'plugin: must be of the form namespace.collection.inv_plugin', 'blank plugin line'), - ({'plugin': '.'}, 'plugin: must be of the form namespace.collection.inv_plugin', 'missing namespace, collection name, and inventory plugin'), - ({'plugin': 'a.'}, 'plugin: must be of the form namespace.collection.inv_plugin', 'missing collection name and inventory plugin'), - ({'plugin': 'a.b'}, 'plugin: must be of the form namespace.collection.inv_plugin', 'missing inventory plugin'), - ({'plugin': 'a.b.'}, 'plugin: must be of the form namespace.collection.inv_plugin', 'missing inventory plugin'), -]) -def test_inventory_source_vars_source_plugin_scm_invalid(post, inventory, admin_user, project, source_var_actual, err_msg, description): - r = post(reverse('api:inventory_source_list'), - {'name': 'new inv src', - 'source_vars': json.dumps(source_var_actual), - 'inventory': inventory.pk, - 'source': 'scm', - 'source_project': project.id,}, - admin_user, expect=400) - - assert err_msg in r.data['source_vars'][0] - - @pytest.mark.django_db @pytest.mark.parametrize('role,expect', [ ('admin_role', 200), From 2eec1317bded2da851c3fb7f0f8e5081415c9917 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 3 Aug 2020 09:50:02 -0400 Subject: [PATCH 08/27] safer migrations --- .../migrations/0118_v380_inventory_plugins.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/awx/main/migrations/0118_v380_inventory_plugins.py b/awx/main/migrations/0118_v380_inventory_plugins.py index d2c9a06398..5ac36b6c1f 100644 --- a/awx/main/migrations/0118_v380_inventory_plugins.py +++ b/awx/main/migrations/0118_v380_inventory_plugins.py @@ -5,6 +5,8 @@ import json from django.db import migrations +from awx.main.models.inventory import InventorySourceOptions + from ._inventory_source_vars import FrozenInjectors @@ -12,8 +14,20 @@ logger = logging.getLogger('awx.main.migrations') BACKUP_FILENAME = '/tmp/tower_migration_inventory_source_vars.json' +class InventorySourceOptionsWrapper(InventorySourceOptions): + ''' + InventorySource inherits from InventorySourceOptions but that is not + "recorded" by Django's app registry model tracking. This will, effectively, + reintroduce the inheritance. + ''' + def __init__(self, *args, **kw): + self.target = kw.pop('target') + super().__init__(self, *args, **kw) + def __getattr__(self, attr): + return getattr(self.target, attr) + + def _get_inventory_sources(InventorySource): - # TODO: Maybe pull the list of cloud sources from code return InventorySource.objects.filter(source__in=['ec2', 'gce', 'azure_rm', 'vmware', 'satellite6', 'openstack', 'rhv', 'tower']) @@ -22,7 +36,7 @@ def inventory_source_vars_forward(apps, schema_editor): source_vars_backup = dict() for inv_source_obj in _get_inventory_sources(InventorySource): - # TODO: Log error if this is false, it shouldn't be false + inv_source_obj = InventorySourceOptionsWrapper(target=inv_source_obj) if inv_source_obj.source in FrozenInjectors: source_vars_backup[inv_source_obj.id] = dict(inv_source_obj.source_vars_dict) with open(BACKUP_FILENAME, 'w') as fh: From dce946e93fe55f3e280f6cfa667d18ab4371c341 Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Wed, 29 Jul 2020 16:39:05 -0400 Subject: [PATCH 09/27] Delete inventory source fields --- .../sources/add/sources-add.controller.js | 107 --------------- .../sources/edit/sources-edit.controller.js | 122 ------------------ .../related/sources/sources.form.js | 40 ------ .../related/sources/sources.service.js | 18 --- 4 files changed, 287 deletions(-) diff --git a/awx/ui/client/src/inventories-hosts/inventories/related/sources/add/sources-add.controller.js b/awx/ui/client/src/inventories-hosts/inventories/related/sources/add/sources-add.controller.js index 9e254fd7e5..0ee7b8392b 100644 --- a/awx/ui/client/src/inventories-hosts/inventories/related/sources/add/sources-add.controller.js +++ b/awx/ui/client/src/inventories-hosts/inventories/related/sources/add/sources-add.controller.js @@ -24,49 +24,6 @@ export default ['$state', 'ConfigData', '$scope', 'SourcesFormDefinition', 'Pars const virtualEnvs = ConfigData.custom_virtualenvs || []; $scope.custom_virtualenvs_options = virtualEnvs; - GetChoices({ - scope: $scope, - field: 'source_regions', - variable: 'rax_regions', - choice_name: 'rax_region_choices', - options: inventorySourcesOptions - }); - - GetChoices({ - scope: $scope, - field: 'source_regions', - variable: 'ec2_regions', - choice_name: 'ec2_region_choices', - options: inventorySourcesOptions - }); - - GetChoices({ - scope: $scope, - field: 'source_regions', - variable: 'gce_regions', - choice_name: 'gce_region_choices', - options: inventorySourcesOptions - }); - - GetChoices({ - scope: $scope, - field: 'source_regions', - variable: 'azure_regions', - choice_name: 'azure_rm_region_choices', - options: inventorySourcesOptions - }); - - // Load options for group_by - GetChoices({ - scope: $scope, - field: 'group_by', - variable: 'ec2_group_by', - choice_name: 'ec2_group_by_choices', - options: inventorySourcesOptions - }); - - initRegionSelect(); - GetChoices({ scope: $scope, field: 'verbosity', @@ -205,20 +162,11 @@ export default ['$state', 'ConfigData', '$scope', 'SourcesFormDefinition', 'Pars $scope.projectBasePath = GetBasePath('projects') + '?not__status=never updated'; } - // reset fields - $scope.group_by_choices = source === 'ec2' ? $scope.ec2_group_by : null; - // azure_rm regions choices are keyed as "azure" in an OPTIONS request to the inventory_sources endpoint - $scope.source_region_choices = source === 'azure_rm' ? $scope.azure_regions : $scope[source + '_regions']; $scope.cloudCredentialRequired = source !== '' && source !== 'scm' && source !== 'custom' && source !== 'ec2' ? true : false; - $scope.source_regions = null; $scope.credential = null; $scope.credential_name = null; - $scope.group_by = null; - $scope.group_by_choices = []; $scope.overwrite_vars = false; - initRegionSelect(); }; - // region / source options callback $scope.$on('sourceTypeOptionsReady', function() { CreateSelect2({ @@ -227,57 +175,6 @@ export default ['$state', 'ConfigData', '$scope', 'SourcesFormDefinition', 'Pars }); }); - function initRegionSelect(){ - CreateSelect2({ - element: '#inventory_source_source_regions', - multiple: true - }); - - let add_new = false; - if( _.get($scope, 'source') === 'ec2' || _.get($scope.source, 'value') === 'ec2') { - $scope.group_by_choices = $scope.ec2_group_by; - $scope.groupByPopOver = "

" + i18n._("Select which groups to create automatically. ") + - $rootScope.BRAND_NAME + i18n._(" will create group names similar to the following examples based on the options selected:") + "

    " + - "
  • " + i18n._("Availability Zone:") + "zones » us-east-1b
    • " + - "
  • " + i18n._("Image ID:") + "images » ami-b007ab1e
    • " + - "
  • " + i18n._("Instance ID:") + "instances » i-ca11ab1e
    • " + - "
  • " + i18n._("Instance Type:") + "types » type_m1_medium
    • " + - "
  • " + i18n._("Key Name:") + "keys » key_testing
    • " + - "
  • " + i18n._("Region:") + "regions » us-east-1
    • " + - "
  • " + i18n._("Security Group:") + "security_groups » security_group_default
    • " + - "
  • " + i18n._("Tags:") + "tags » tag_Name » tag_Name_host1
    • " + - "
  • " + i18n._("VPC ID:") + "vpcs » vpc-5ca1ab1e
    • " + - "
  • " + i18n._("Tag None:") + "tags » tag_none
    • " + - "

" + i18n._("If blank, all groups above are created except") + "" + i18n._("Instance ID") + ".

"; - - $scope.instanceFilterPopOver = "

" + i18n._("Provide a comma-separated list of filter expressions. ") + - i18n._("Hosts are imported to ") + $rootScope.BRAND_NAME + i18n._(" when ") + "" + i18n._("ANY") + "" + i18n._(" of the filters match.") + "

" + - i18n._("Limit to hosts having a tag:") + "
\n" + - "
tag-key=TowerManaged
\n" + - i18n._("Limit to hosts using either key pair:") + "
\n" + - "
key-name=staging, key-name=production
\n" + - i18n._("Limit to hosts where the Name tag begins with ") + "" + i18n._("test") + ":
\n" + - "
tag:Name=test*
\n" + - "

" + i18n._("View the ") + "" + i18n._("Describe Instances documentation") + " " + - i18n._("for a complete list of supported filters.") + "

"; - } - if( _.get($scope, 'source') === 'vmware' || _.get($scope.source, 'value') === 'vmware') { - add_new = true; - $scope.group_by_choices = []; - $scope.group_by = $scope.group_by_choices; - $scope.groupByPopOver = i18n._("Specify which groups to create automatically. Group names will be created similar to the options selected. If blank, all groups above are created. Refer to Ansible Tower documentation for more detail."); - $scope.instanceFilterPopOver = i18n._("Provide a comma-separated list of filter expressions. Hosts are imported when all of the filters match. Refer to Ansible Tower documentation for more detail."); - } - if( _.get($scope, 'source') === 'tower' || _.get($scope.source, 'value') === 'tower') { - $scope.instanceFilterPopOver = i18n._("Provide the named URL encoded name or id of the remote Tower inventory to be imported."); - } - CreateSelect2({ - element: '#inventory_source_group_by', - multiple: true, - addNew: add_new - }); - } - $scope.formCancel = function() { $state.go('^'); }; @@ -289,7 +186,6 @@ export default ['$state', 'ConfigData', '$scope', 'SourcesFormDefinition', 'Pars name: $scope.name, description: $scope.description, inventory: inventoryData.id, - instance_filters: $scope.instance_filters, source_script: $scope.inventory_script, credential: $scope.credential, overwrite: $scope.overwrite, @@ -298,9 +194,6 @@ export default ['$state', 'ConfigData', '$scope', 'SourcesFormDefinition', 'Pars verbosity: $scope.verbosity.value, update_cache_timeout: $scope.update_cache_timeout || 0, custom_virtualenv: $scope.custom_virtualenv || null, - // comma-delimited strings - group_by: SourcesService.encodeGroupBy($scope.source, $scope.group_by), - source_regions: _.map($scope.source_regions, 'value').join(','), }; if ($scope.source) { diff --git a/awx/ui/client/src/inventories-hosts/inventories/related/sources/edit/sources-edit.controller.js b/awx/ui/client/src/inventories-hosts/inventories/related/sources/edit/sources-edit.controller.js index 40dc4fc970..1bbaecaf4b 100644 --- a/awx/ui/client/src/inventories-hosts/inventories/related/sources/edit/sources-edit.controller.js +++ b/awx/ui/client/src/inventories-hosts/inventories/related/sources/edit/sources-edit.controller.js @@ -34,7 +34,6 @@ export default ['$state', '$scope', 'ParseVariableString', 'ParseTypeChange', {overwrite_vars: inventorySourceData.overwrite_vars}, {update_on_launch: inventorySourceData.update_on_launch}, {update_cache_timeout: inventorySourceData.update_cache_timeout}, - {instance_filters: inventorySourceData.instance_filters}, {inventory_script: inventorySourceData.source_script}, {verbosity: inventorySourceData.verbosity}); @@ -100,56 +99,6 @@ export default ['$state', '$scope', 'ParseVariableString', 'ParseTypeChange', scope: $scope, variable: 'source_type_options' }); - GetChoices({ - scope: $scope, - field: 'source_regions', - variable: 'rax_regions', - choice_name: 'rax_region_choices', - options: inventorySourcesOptions - }); - GetChoices({ - scope: $scope, - field: 'source_regions', - variable: 'ec2_regions', - choice_name: 'ec2_region_choices', - options: inventorySourcesOptions - }); - GetChoices({ - scope: $scope, - field: 'source_regions', - variable: 'gce_regions', - choice_name: 'gce_region_choices', - options: inventorySourcesOptions - }); - GetChoices({ - scope: $scope, - field: 'source_regions', - variable: 'azure_regions', - choice_name: 'azure_rm_region_choices', - options: inventorySourcesOptions - }); - GetChoices({ - scope: $scope, - field: 'group_by', - variable: 'ec2_group_by', - choice_name: 'ec2_group_by_choices', - options: inventorySourcesOptions - }); - - var source = $scope.source === 'azure_rm' ? 'azure' : $scope.source; - var regions = inventorySourceData.source_regions.split(','); - // azure_rm regions choices are keyed as "azure" in an OPTIONS request to the inventory_sources endpoint - $scope.source_region_choices = $scope[source + '_regions']; - - // the API stores azure regions as all-lowercase strings - but the azure regions received from OPTIONS are Snake_Cased - if (source === 'azure') { - $scope.source_regions = _.map(regions, (region) => _.find($scope[source + '_regions'], (o) => o.value.toLowerCase() === region)); - } - // all other regions are 1-1 - else { - $scope.source_regions = _.map(regions, (region) => _.find($scope[source + '_regions'], (o) => o.value === region)); - } - initRegionSelect(); GetChoices({ scope: $scope, @@ -236,63 +185,6 @@ export default ['$state', '$scope', 'ParseVariableString', 'ParseTypeChange', } } - function initRegionSelect() { - CreateSelect2({ - element: '#inventory_source_source_regions', - multiple: true - }); - - let add_new = false; - if( _.get($scope, 'source') === 'ec2' || _.get($scope.source, 'value') === 'ec2') { - $scope.group_by_choices = $scope.ec2_group_by; - let group_by = inventorySourceData.group_by.split(','); - $scope.group_by = _.map(group_by, (item) => _.find($scope.ec2_group_by, { value: item })); - - $scope.groupByPopOver = "

" + i18n._("Select which groups to create automatically. ") + - $rootScope.BRAND_NAME + i18n._(" will create group names similar to the following examples based on the options selected:") + "

    " + - "
  • " + i18n._("Availability Zone:") + "zones » us-east-1b
    • " + - "
  • " + i18n._("Image ID:") + "images » ami-b007ab1e
    • " + - "
  • " + i18n._("Instance ID:") + "instances » i-ca11ab1e
    • " + - "
  • " + i18n._("Instance Type:") + "types » type_m1_medium
    • " + - "
  • " + i18n._("Key Name:") + "keys » key_testing
    • " + - "
  • " + i18n._("Region:") + "regions » us-east-1
    • " + - "
  • " + i18n._("Security Group:") + "security_groups » security_group_default
    • " + - "
  • " + i18n._("Tags:") + "tags » tag_Name » tag_Name_host1
    • " + - "
  • " + i18n._("VPC ID:") + "vpcs » vpc-5ca1ab1e
    • " + - "
  • " + i18n._("Tag None:") + "tags » tag_none
    • " + - "

" + i18n._("If blank, all groups above are created except") + "" + i18n._("Instance ID") + ".

"; - - - $scope.instanceFilterPopOver = "

" + i18n._("Provide a comma-separated list of filter expressions. ") + - i18n._("Hosts are imported to ") + $rootScope.BRAND_NAME + i18n._(" when ") + "" + i18n._("ANY") + "" + i18n._(" of the filters match.") + "

" + - i18n._("Limit to hosts having a tag:") + "
\n" + - "
tag-key=TowerManaged
\n" + - i18n._("Limit to hosts using either key pair:") + "
\n" + - "
key-name=staging, key-name=production
\n" + - i18n._("Limit to hosts where the Name tag begins with ") + "" + i18n._("test") + ":
\n" + - "
tag:Name=test*
\n" + - "

" + i18n._("View the ") + "" + i18n._("Describe Instances documentation") + " " + - i18n._("for a complete list of supported filters.") + "

"; - - } - if( _.get($scope, 'source') === 'vmware' || _.get($scope.source, 'value') === 'vmware') { - add_new = true; - $scope.group_by_choices = (inventorySourceData.group_by) ? inventorySourceData.group_by.split(',') - .map((i) => ({name: i, label: i, value: i})) : []; - $scope.group_by = $scope.group_by_choices; - $scope.groupByPopOver = i18n._(`Specify which groups to create automatically. Group names will be created similar to the options selected. If blank, all groups above are created. Refer to Ansible Tower documentation for more detail.`); - $scope.instanceFilterPopOver = i18n._(`Provide a comma-separated list of filter expressions. Hosts are imported when all of the filters match. Refer to Ansible Tower documentation for more detail.`); - } - if( _.get($scope, 'source') === 'tower' || _.get($scope.source, 'value') === 'tower') { - $scope.instanceFilterPopOver = i18n._(`Provide the named URL encoded name or id of the remote Tower inventory to be imported.`); - } - CreateSelect2({ - element: '#inventory_source_group_by', - multiple: true, - addNew: add_new - }); - } - $scope.lookupProject = function(){ $state.go('.project', { project_search: { @@ -346,7 +238,6 @@ export default ['$state', '$scope', 'ParseVariableString', 'ParseTypeChange', name: $scope.name, description: $scope.description, inventory: inventoryData.id, - instance_filters: $scope.instance_filters, source_script: $scope.inventory_script, credential: $scope.credential, overwrite: $scope.overwrite, @@ -355,9 +246,6 @@ export default ['$state', '$scope', 'ParseVariableString', 'ParseTypeChange', update_cache_timeout: $scope.update_cache_timeout || 0, verbosity: $scope.verbosity.value, custom_virtualenv: $scope.custom_virtualenv || null, - // comma-delimited strings - group_by: SourcesService.encodeGroupBy($scope.source, $scope.group_by), - source_regions: _.map($scope.source_regions, 'value').join(',') }; if ($scope.source) { @@ -417,20 +305,10 @@ export default ['$state', '$scope', 'ParseVariableString', 'ParseTypeChange', }); } - // reset fields - $scope.group_by_choices = source === 'ec2' ? $scope.ec2_group_by : null; - // azure_rm regions choices are keyed as "azure" in an OPTIONS request to the inventory_sources endpoint - $scope.source_region_choices = source === 'azure_rm' ? $scope.azure_regions : $scope[source + '_regions']; $scope.cloudCredentialRequired = source !== '' && source !== 'scm' && source !== 'custom' && source !== 'ec2' ? true : false; - $scope.source_regions = null; $scope.credential = null; $scope.credential_name = null; - $scope.group_by = null; - $scope.group_by_choices = []; $scope.overwrite_vars = false; - - initRegionSelect(); - }; } ]; diff --git a/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js b/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js index 3c76dd2e61..2594ad29da 100644 --- a/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js +++ b/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js @@ -126,46 +126,6 @@ export default ['NotificationsList', 'i18n', function(NotificationsList, i18n){ includeInventoryFileNotFoundError: true, subForm: 'sourceSubForm' }, - source_regions: { - label: i18n._('Regions'), - type: 'select', - ngOptions: 'source.label for source in source_region_choices track by source.value', - multiSelect: true, - ngShow: "source && (source.value == 'rax' || source.value == 'ec2' || source.value == 'gce' || source.value == 'azure_rm')", - dataTitle: i18n._('Source Regions'), - dataPlacement: 'right', - awPopOver: "

" + i18n._("Click on the regions field to see a list of regions for your cloud provider. You can select multiple regions, or choose") + - "" + i18n._("All") + " " + i18n._("to include all regions. Only Hosts associated with the selected regions will be updated.") + "

", - dataContainer: 'body', - ngDisabled: '!(inventory_source_obj.summary_fields.user_capabilities.edit || canAdd)', - subForm: 'sourceSubForm' - }, - instance_filters: { - label: i18n._("Instance Filters"), - type: 'text', - ngShow: "source && (source.value == 'ec2' || source.value == 'vmware' || source.value == 'tower')", - dataTitle: i18n._('Instance Filters'), - dataPlacement: 'right', - awPopOverWatch: 'instanceFilterPopOver', - awPopOver: '{{ instanceFilterPopOver }}', - dataContainer: 'body', - ngDisabled: '!(inventory_source_obj.summary_fields.user_capabilities.edit || canAdd)', - subForm: 'sourceSubForm' - }, - group_by: { - label: i18n._('Only Group By'), - type: 'select', - ngShow: "source && (source.value == 'ec2' || source.value == 'vmware')", - ngOptions: 'source.label for source in group_by_choices track by source.value', - multiSelect: true, - dataTitle: i18n._("Only Group By"), - dataPlacement: 'right', - awPopOverWatch: 'groupByPopOver', - awPopOver: '{{ groupByPopOver }}', - dataContainer: 'body', - ngDisabled: '!(inventory_source_obj.summary_fields.user_capabilities.edit || canAdd)', - subForm: 'sourceSubForm' - }, inventory_script: { label : i18n._("Custom Inventory Script"), type: 'lookup', diff --git a/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.service.js b/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.service.js index ce65a7dae5..a42ac51097 100644 --- a/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.service.js +++ b/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.service.js @@ -116,24 +116,6 @@ export default .catch(this.error.bind(this)) .finally(Wait('stop')); }, - encodeGroupBy(source, group_by){ - source = source && source.value ? source.value : ''; - if(source === 'ec2'){ - return _.map(group_by, 'value').join(','); - } - - if(source === 'vmware'){ - group_by = _.map(group_by, (i) => {return i.value;}); - $("#inventory_source_group_by").siblings(".select2").first().find(".select2-selection__choice").each(function(optionIndex, option){ - group_by.push(option.title); - }); - group_by = (Array.isArray(group_by)) ? _.uniq(group_by).join() : ""; - return group_by; - } - else { - return; - } - }, deleteHosts(id) { this.url = GetBasePath('inventory_sources') + id + '/hosts/'; Rest.setUrl(this.url); From 42e70bc85207e0409925f37eccf7ebcc9cc85b9e Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Wed, 29 Jul 2020 18:33:46 -0400 Subject: [PATCH 10/27] Delete inventory source fields --- .../InventorySourceDetail.jsx | 57 +--- .../InventorySourceDetail.test.jsx | 26 -- .../Inventory/shared/InventorySourceForm.jsx | 6 - .../InventorySourceSubForms/AzureSubForm.jsx | 14 +- .../AzureSubForm.test.jsx | 10 +- .../CloudFormsSubForm.test.jsx | 3 - .../InventorySourceSubForms/EC2SubForm.jsx | 25 +- .../EC2SubForm.test.jsx | 15 +- .../InventorySourceSubForms/GCESubForm.jsx | 9 +- .../GCESubForm.test.jsx | 10 +- .../OpenStackSubForm.test.jsx | 3 - .../SCMSubForm.test.jsx | 3 - .../SatelliteSubForm.test.jsx | 3 - .../InventorySourceSubForms/SharedFields.jsx | 249 +----------------- .../InventorySourceSubForms/TowerSubForm.jsx | 7 +- .../TowerSubForm.test.jsx | 4 - .../InventorySourceSubForms/VMwareSubForm.jsx | 10 +- .../VMwareSubForm.test.jsx | 11 +- .../VirtualizationSubForm.test.jsx | 3 - .../shared/data.inventory_source.json | 3 - 20 files changed, 16 insertions(+), 455 deletions(-) diff --git a/awx/ui_next/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.jsx b/awx/ui_next/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.jsx index 42fad28421..badd28fb26 100644 --- a/awx/ui_next/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.jsx +++ b/awx/ui_next/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.jsx @@ -3,10 +3,9 @@ import { Link, useHistory } from 'react-router-dom'; import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; -import { Button, Chip, List, ListItem } from '@patternfly/react-core'; +import { Button, List, ListItem } from '@patternfly/react-core'; import AlertModal from '../../../components/AlertModal'; import { CardBody, CardActionsRow } from '../../../components/Card'; -import ChipGroup from '../../../components/ChipGroup'; import { VariablesDetail } from '../../../components/CodeMirrorInput'; import ContentError from '../../../components/ContentError'; import ContentLoading from '../../../components/ContentLoading'; @@ -28,16 +27,13 @@ function InventorySourceDetail({ inventorySource, i18n }) { created, custom_virtualenv, description, - group_by, id, - instance_filters, modified, name, overwrite, overwrite_vars, source, source_path, - source_regions, source_vars, update_cache_timeout, update_on_launch, @@ -233,57 +229,6 @@ function InventorySourceDetail({ inventorySource, i18n }) { ))} /> )} - {source_regions && ( - - {source_regions.split(',').map(region => ( - - {region} - - ))} - - } - /> - )} - {instance_filters && ( - - {instance_filters.split(',').map(filter => ( - - {filter} - - ))} - - } - /> - )} - {group_by && ( - - {group_by.split(',').map(group => ( - - {group} - - ))} - - } - /> - )} {optionsList && ( )} diff --git a/awx/ui_next/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.test.jsx b/awx/ui_next/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.test.jsx index c8905143ee..0d7f9a9b79 100644 --- a/awx/ui_next/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.test.jsx +++ b/awx/ui_next/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.test.jsx @@ -64,32 +64,6 @@ describe('InventorySourceDetail', () => { assertDetail(wrapper, 'Inventory file', 'foo'); assertDetail(wrapper, 'Verbosity', '2 (Debug)'); assertDetail(wrapper, 'Cache timeout', '2 seconds'); - expect( - wrapper - .find('Detail[label="Regions"]') - .containsAllMatchingElements([ - us-east-1, - us-east-2, - ]) - ).toEqual(true); - expect( - wrapper - .find('Detail[label="Instance filters"]') - .containsAllMatchingElements([ - filter1, - filter2, - filter3, - ]) - ).toEqual(true); - expect( - wrapper - .find('Detail[label="Only group by"]') - .containsAllMatchingElements([ - group1, - group2, - group3, - ]) - ).toEqual(true); expect(wrapper.find('CredentialChip').text()).toBe('Cloud: mock cred'); expect(wrapper.find('VariablesDetail').prop('value')).toEqual( '---\nfoo: bar' diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceForm.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceForm.jsx index a694d15243..08b32f1f1a 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceForm.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceForm.jsx @@ -75,14 +75,11 @@ const InventorySourceFormFields = ({ sourceOptions, i18n }) => { } else { const defaults = { credential: null, - group_by: '', - instance_filters: '', overwrite: false, overwrite_vars: false, source: sourceType, source_path: '', source_project: null, - source_regions: '', source_script: null, source_vars: '---\n', update_cache_timeout: 0, @@ -200,15 +197,12 @@ const InventorySourceForm = ({ credential: source?.summary_fields?.credential || null, custom_virtualenv: source?.custom_virtualenv || '', description: source?.description || '', - group_by: source?.group_by || '', - instance_filters: source?.instance_filters || '', name: source?.name || '', overwrite: source?.overwrite || false, overwrite_vars: source?.overwrite_vars || false, source: source?.source || '', source_path: source?.source_path === '' ? '/ (project root)' : '', source_project: source?.summary_fields?.source_project || null, - source_regions: source?.source_regions || '', source_script: source?.summary_fields?.source_script || null, source_vars: source?.source_vars || '---\n', update_cache_timeout: source?.update_cache_timeout || 0, diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/AzureSubForm.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/AzureSubForm.jsx index 7bc6b49975..99b83ca6b5 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/AzureSubForm.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/AzureSubForm.jsx @@ -3,14 +3,9 @@ import { useField } from 'formik'; import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; import CredentialLookup from '../../../../components/Lookup/CredentialLookup'; -import { - OptionsField, - RegionsField, - SourceVarsField, - VerbosityField, -} from './SharedFields'; +import { OptionsField, SourceVarsField, VerbosityField } from './SharedFields'; -const AzureSubForm = ({ i18n, sourceOptions }) => { +const AzureSubForm = ({ i18n }) => { const [credentialField, credentialMeta, credentialHelpers] = useField( 'credential' ); @@ -29,11 +24,6 @@ const AzureSubForm = ({ i18n, sourceOptions }) => { value={credentialField.value} required /> - diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/AzureSubForm.test.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/AzureSubForm.test.jsx index a7dae124fd..b363f7f42b 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/AzureSubForm.test.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/AzureSubForm.test.jsx @@ -10,13 +10,10 @@ jest.mock('../../../../api/models/Credentials'); const initialValues = { credential: null, custom_virtualenv: '', - group_by: '', - instance_filters: '', overwrite: false, overwrite_vars: false, source_path: '', source_project: null, - source_regions: '', source_script: null, source_vars: '---\n', update_cache_timeout: 0, @@ -27,11 +24,7 @@ const initialValues = { const mockSourceOptions = { actions: { - POST: { - source_regions: { - azure_rm_region_choices: [], - }, - }, + POST: {}, }, }; @@ -58,7 +51,6 @@ describe('', () => { test('should render subform fields', () => { expect(wrapper.find('FormGroup[label="Credential"]')).toHaveLength(1); - expect(wrapper.find('FormGroup[label="Regions"]')).toHaveLength(1); expect(wrapper.find('FormGroup[label="Verbosity"]')).toHaveLength(1); expect(wrapper.find('FormGroup[label="Update options"]')).toHaveLength(1); expect( diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/CloudFormsSubForm.test.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/CloudFormsSubForm.test.jsx index e46ac8d8fa..8e46b042fd 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/CloudFormsSubForm.test.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/CloudFormsSubForm.test.jsx @@ -10,13 +10,10 @@ jest.mock('../../../../api/models/Credentials'); const initialValues = { credential: null, custom_virtualenv: '', - group_by: '', - instance_filters: '', overwrite: false, overwrite_vars: false, source_path: '', source_project: null, - source_regions: '', source_script: null, source_vars: '---\n', update_cache_timeout: 0, diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/EC2SubForm.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/EC2SubForm.jsx index d74e770895..32fc581742 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/EC2SubForm.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/EC2SubForm.jsx @@ -3,24 +3,10 @@ import { useField } from 'formik'; import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; import CredentialLookup from '../../../../components/Lookup/CredentialLookup'; -import { - GroupByField, - InstanceFiltersField, - OptionsField, - RegionsField, - SourceVarsField, - VerbosityField, -} from './SharedFields'; +import { OptionsField, SourceVarsField, VerbosityField } from './SharedFields'; -const EC2SubForm = ({ i18n, sourceOptions }) => { +const EC2SubForm = ({ i18n }) => { const [credentialField, , credentialHelpers] = useField('credential'); - const groupByOptionsObj = Object.assign( - {}, - ...sourceOptions?.actions?.POST?.group_by?.ec2_group_by_choices.map( - ([key, val]) => ({ [key]: val }) - ) - ); - return ( <> { credentialHelpers.setValue(value); }} /> - - - diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/EC2SubForm.test.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/EC2SubForm.test.jsx index fc15d03ea9..7a41471ec2 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/EC2SubForm.test.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/EC2SubForm.test.jsx @@ -10,13 +10,10 @@ jest.mock('../../../../api/models/Credentials'); const initialValues = { credential: null, custom_virtualenv: '', - group_by: '', - instance_filters: '', overwrite: false, overwrite_vars: false, source_path: '', source_project: null, - source_regions: '', source_script: null, source_vars: '---\n', update_cache_timeout: 0, @@ -27,14 +24,7 @@ const initialValues = { const mockSourceOptions = { actions: { - POST: { - source_regions: { - ec2_region_choices: [], - }, - group_by: { - ec2_group_by_choices: [], - }, - }, + POST: {}, }, }; @@ -61,9 +51,6 @@ describe('', () => { test('should render subform fields', () => { expect(wrapper.find('FormGroup[label="Credential"]')).toHaveLength(1); - expect(wrapper.find('FormGroup[label="Regions"]')).toHaveLength(1); - expect(wrapper.find('FormGroup[label="Instance filters"]')).toHaveLength(1); - expect(wrapper.find('FormGroup[label="Only group by"]')).toHaveLength(1); expect(wrapper.find('FormGroup[label="Verbosity"]')).toHaveLength(1); expect(wrapper.find('FormGroup[label="Update options"]')).toHaveLength(1); expect( diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/GCESubForm.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/GCESubForm.jsx index 0451e80b86..fcbbffa040 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/GCESubForm.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/GCESubForm.jsx @@ -3,9 +3,9 @@ import { useField } from 'formik'; import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; import CredentialLookup from '../../../../components/Lookup/CredentialLookup'; -import { OptionsField, RegionsField, VerbosityField } from './SharedFields'; +import { OptionsField, VerbosityField } from './SharedFields'; -const GCESubForm = ({ i18n, sourceOptions }) => { +const GCESubForm = ({ i18n }) => { const [credentialField, credentialMeta, credentialHelpers] = useField( 'credential' ); @@ -24,11 +24,6 @@ const GCESubForm = ({ i18n, sourceOptions }) => { value={credentialField.value} required /> - diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/GCESubForm.test.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/GCESubForm.test.jsx index a7845972c1..4655d0ad3f 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/GCESubForm.test.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/GCESubForm.test.jsx @@ -10,13 +10,10 @@ jest.mock('../../../../api/models/Credentials'); const initialValues = { credential: null, custom_virtualenv: '', - group_by: '', - instance_filters: '', overwrite: false, overwrite_vars: false, source_path: '', source_project: null, - source_regions: '', source_script: null, source_vars: '---\n', update_cache_timeout: 0, @@ -27,11 +24,7 @@ const initialValues = { const mockSourceOptions = { actions: { - POST: { - source_regions: { - gce_region_choices: [], - }, - }, + POST: {}, }, }; @@ -58,7 +51,6 @@ describe('', () => { test('should render subform fields', () => { expect(wrapper.find('FormGroup[label="Credential"]')).toHaveLength(1); - expect(wrapper.find('FormGroup[label="Regions"]')).toHaveLength(1); expect(wrapper.find('FormGroup[label="Verbosity"]')).toHaveLength(1); expect(wrapper.find('FormGroup[label="Update options"]')).toHaveLength(1); expect( diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/OpenStackSubForm.test.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/OpenStackSubForm.test.jsx index b4be2c1aff..f912186816 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/OpenStackSubForm.test.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/OpenStackSubForm.test.jsx @@ -10,13 +10,10 @@ jest.mock('../../../../api/models/Credentials'); const initialValues = { credential: null, custom_virtualenv: '', - group_by: '', - instance_filters: '', overwrite: false, overwrite_vars: false, source_path: '', source_project: null, - source_regions: '', source_script: null, source_vars: '---\n', update_cache_timeout: 0, diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.test.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.test.jsx index 6d763b78a7..fdd4fdb317 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.test.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.test.jsx @@ -11,13 +11,10 @@ jest.mock('../../../../api/models/Credentials'); const initialValues = { credential: null, custom_virtualenv: '', - group_by: '', - instance_filters: '', overwrite: false, overwrite_vars: false, source_path: '', source_project: null, - source_regions: '', source_script: null, source_vars: '---\n', update_cache_timeout: 0, diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SatelliteSubForm.test.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SatelliteSubForm.test.jsx index 5da6c02db6..2934390896 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SatelliteSubForm.test.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SatelliteSubForm.test.jsx @@ -10,13 +10,10 @@ jest.mock('../../../../api/models/Credentials'); const initialValues = { credential: null, custom_virtualenv: '', - group_by: '', - instance_filters: '', overwrite: false, overwrite_vars: false, source_path: '', source_project: null, - source_regions: '', source_script: null, source_vars: '---\n', update_cache_timeout: 0, diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SharedFields.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SharedFields.jsx index dcfc4b70fc..3ca698641b 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SharedFields.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SharedFields.jsx @@ -1,16 +1,9 @@ -import React, { useEffect, useState } from 'react'; +import React, { useEffect } from 'react'; import { withI18n } from '@lingui/react'; -import { t, Trans } from '@lingui/macro'; +import { t } from '@lingui/macro'; import { useField } from 'formik'; -import { - FormGroup, - Select, - SelectOption, - SelectVariant, -} from '@patternfly/react-core'; -import { arrayToString, stringToArray } from '../../../../util/strings'; +import { FormGroup } from '@patternfly/react-core'; import { minMaxValue } from '../../../../util/validators'; -import { BrandName } from '../../../../variables'; import AnsibleSelect from '../../../../components/AnsibleSelect'; import { VariablesField } from '../../../../components/CodeMirrorInput'; import FormField, { @@ -32,196 +25,6 @@ export const SourceVarsField = withI18n()(({ i18n }) => ( )); -export const RegionsField = withI18n()(({ i18n, regionOptions }) => { - const [field, meta, helpers] = useField('source_regions'); - const [isOpen, setIsOpen] = useState(false); - const options = Object.assign( - {}, - ...regionOptions.map(([key, val]) => ({ [key]: val })) - ); - const selected = stringToArray(field?.value) - .filter(i => options[i]) - .map(val => options[val]); - - return ( - - Click on the regions field to see a list of regions for your cloud - provider. You can select multiple regions, or choose - All to include all regions. Only Hosts associated with - the selected regions will be updated. - - } - /> - } - > - - - ); -}); - -export const GroupByField = withI18n()( - ({ i18n, fixedOptions, isCreatable = false }) => { - const [field, meta, helpers] = useField('group_by'); - const fixedOptionLabels = fixedOptions && Object.values(fixedOptions); - const selections = fixedOptions - ? stringToArray(field.value).map(o => fixedOptions[o]) - : stringToArray(field.value); - const [options, setOptions] = useState(selections); - const [isOpen, setIsOpen] = useState(false); - - const renderOptions = opts => { - return opts.map(option => ( - - {option} - - )); - }; - - const handleFilter = event => { - const str = event.target.value.toLowerCase(); - let matches; - if (fixedOptions) { - matches = fixedOptionLabels.filter(o => o.toLowerCase().includes(str)); - } else { - matches = options.filter(o => o.toLowerCase().includes(str)); - } - return renderOptions(matches); - }; - - const handleSelect = (e, option) => { - let selectedValues; - if (selections.includes(option)) { - selectedValues = selections.filter(o => o !== option); - } else { - selectedValues = selections.concat(option); - } - if (fixedOptions) { - selectedValues = selectedValues.map(val => - Object.keys(fixedOptions).find(key => fixedOptions[key] === val) - ); - } - helpers.setValue(arrayToString(selectedValues)); - }; - - return ( - - Select which groups to create automatically. AWX will create - group names similar to the following examples based on the - options selected: -
-
-
    -
  • - Availability Zone: zones » us-east-1b -
    • -
  • - Image ID: images » ami-b007ab1e -
    • -
  • - Instance ID: instances » i-ca11ab1e -
    • -
  • - Instance Type: types » type_m1_medium -
    • -
  • - Key Name: keys » key_testing -
    • -
  • - Region: regions » us-east-1 -
    • -
  • - Security Group:{' '} - - security_groups » security_group_default - -
    • -
  • - Tags: tags » tag_Name_host1 -
    • -
  • - VPC ID: vpcs » vpc-5ca1ab1e -
    • -
  • - Tag None: tags » tag_none -
    • -
-
- If blank, all groups above are created except{' '} - Instance ID. - - } - /> - } - > - - - ); - } -); - export const VerbosityField = withI18n()(({ i18n }) => { const [field, meta, helpers] = useField('verbosity'); const isValid = !(meta.touched && meta.error); @@ -351,49 +154,3 @@ export const OptionsField = withI18n()( ); } ); - -export const InstanceFiltersField = withI18n()(({ i18n }) => { - // Setting BrandName to a variable here is necessary to get the jest tests - // passing. Attempting to use BrandName in the template literal results - // in failing tests. - const brandName = BrandName; - return ( - - Provide a comma-separated list of filter expressions. Hosts are - imported to {brandName} when ANY of the filters match. -
-
- Limit to hosts having a tag: -
- tag-key=TowerManaged -
-
- Limit to hosts using either key pair: -
- key-name=staging, key-name=production -
-
- Limit to hosts where the Name tag begins with test:
- tag:Name=test* -
-
- View the - - {' '} - Describe Instances documentation{' '} - - for a complete list of supported filters. - - } - /> - ); -}); diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/TowerSubForm.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/TowerSubForm.jsx index f3fe26d3a9..3af0f9a5c6 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/TowerSubForm.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/TowerSubForm.jsx @@ -3,11 +3,7 @@ import { useField } from 'formik'; import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; import CredentialLookup from '../../../../components/Lookup/CredentialLookup'; -import { - InstanceFiltersField, - OptionsField, - VerbosityField, -} from './SharedFields'; +import { OptionsField, VerbosityField } from './SharedFields'; const TowerSubForm = ({ i18n }) => { const [credentialField, credentialMeta, credentialHelpers] = useField( @@ -28,7 +24,6 @@ const TowerSubForm = ({ i18n }) => { value={credentialField.value} required /> - diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/TowerSubForm.test.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/TowerSubForm.test.jsx index 71bc801823..fd7ee0488a 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/TowerSubForm.test.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/TowerSubForm.test.jsx @@ -10,13 +10,10 @@ jest.mock('../../../../api/models/Credentials'); const initialValues = { credential: null, custom_virtualenv: '', - group_by: '', - instance_filters: '', overwrite: false, overwrite_vars: false, source_path: '', source_project: null, - source_regions: '', source_script: null, source_vars: '---\n', update_cache_timeout: 0, @@ -48,7 +45,6 @@ describe('', () => { test('should render subform fields', () => { expect(wrapper.find('FormGroup[label="Credential"]')).toHaveLength(1); - expect(wrapper.find('FormGroup[label="Instance filters"]')).toHaveLength(1); expect(wrapper.find('FormGroup[label="Verbosity"]')).toHaveLength(1); expect(wrapper.find('FormGroup[label="Update options"]')).toHaveLength(1); expect( diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/VMwareSubForm.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/VMwareSubForm.jsx index e975e789b1..555b0498e3 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/VMwareSubForm.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/VMwareSubForm.jsx @@ -3,13 +3,7 @@ import { useField } from 'formik'; import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; import CredentialLookup from '../../../../components/Lookup/CredentialLookup'; -import { - InstanceFiltersField, - GroupByField, - OptionsField, - SourceVarsField, - VerbosityField, -} from './SharedFields'; +import { OptionsField, SourceVarsField, VerbosityField } from './SharedFields'; const VMwareSubForm = ({ i18n }) => { const [credentialField, credentialMeta, credentialHelpers] = useField( @@ -30,8 +24,6 @@ const VMwareSubForm = ({ i18n }) => { value={credentialField.value} required /> - - diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/VMwareSubForm.test.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/VMwareSubForm.test.jsx index ba4777733d..e86bc49d50 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/VMwareSubForm.test.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/VMwareSubForm.test.jsx @@ -10,13 +10,10 @@ jest.mock('../../../../api/models/Credentials'); const initialValues = { credential: null, custom_virtualenv: '', - group_by: '', - instance_filters: '', overwrite: false, overwrite_vars: false, source_path: '', source_project: null, - source_regions: '', source_script: null, source_vars: '---\n', update_cache_timeout: 0, @@ -27,11 +24,7 @@ const initialValues = { const mockSourceOptions = { actions: { - POST: { - source_regions: { - gce_region_choices: [], - }, - }, + POST: {}, }, }; @@ -58,8 +51,6 @@ describe('', () => { test('should render subform fields', () => { expect(wrapper.find('FormGroup[label="Credential"]')).toHaveLength(1); - expect(wrapper.find('FormGroup[label="Instance filters"]')).toHaveLength(1); - expect(wrapper.find('FormGroup[label="Only group by"]')).toHaveLength(1); expect(wrapper.find('FormGroup[label="Verbosity"]')).toHaveLength(1); expect(wrapper.find('FormGroup[label="Update options"]')).toHaveLength(1); expect( diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/VirtualizationSubForm.test.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/VirtualizationSubForm.test.jsx index 1d1526a42d..35f3933bb9 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/VirtualizationSubForm.test.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/VirtualizationSubForm.test.jsx @@ -10,13 +10,10 @@ jest.mock('../../../../api/models/Credentials'); const initialValues = { credential: null, custom_virtualenv: '', - group_by: '', - instance_filters: '', overwrite: false, overwrite_vars: false, source_path: '', source_project: null, - source_regions: '', source_script: null, source_vars: '---\n', update_cache_timeout: 0, diff --git a/awx/ui_next/src/screens/Inventory/shared/data.inventory_source.json b/awx/ui_next/src/screens/Inventory/shared/data.inventory_source.json index c6fbf26365..ad1e313611 100644 --- a/awx/ui_next/src/screens/Inventory/shared/data.inventory_source.json +++ b/awx/ui_next/src/screens/Inventory/shared/data.inventory_source.json @@ -96,9 +96,6 @@ "source_script": "Mock Script", "source_vars":"---\nfoo: bar", "credential": 8, - "source_regions": "us-east-1,us-east-2", - "instance_filters": "filter1,filter2,filter3", - "group_by": "group1,group2,group3", "overwrite":true, "overwrite_vars":true, "custom_virtualenv":"/venv/custom", From b253540047a17b90989c4317b3335546e3177cdd Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Tue, 4 Aug 2020 16:04:04 -0400 Subject: [PATCH 11/27] Add new common inventory source fields --- .../sources/add/sources-add.controller.js | 3 ++ .../sources/edit/sources-edit.controller.js | 5 ++++ .../related/sources/sources.form.js | 30 +++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/awx/ui/client/src/inventories-hosts/inventories/related/sources/add/sources-add.controller.js b/awx/ui/client/src/inventories-hosts/inventories/related/sources/add/sources-add.controller.js index 0ee7b8392b..05bfe07718 100644 --- a/awx/ui/client/src/inventories-hosts/inventories/related/sources/add/sources-add.controller.js +++ b/awx/ui/client/src/inventories-hosts/inventories/related/sources/add/sources-add.controller.js @@ -194,6 +194,9 @@ export default ['$state', 'ConfigData', '$scope', 'SourcesFormDefinition', 'Pars verbosity: $scope.verbosity.value, update_cache_timeout: $scope.update_cache_timeout || 0, custom_virtualenv: $scope.custom_virtualenv || null, + enabled_var: $scope.enabled_var, + enabled_value: $scope.enabled_value, + host_filter: $scope.host_filter }; if ($scope.source) { diff --git a/awx/ui/client/src/inventories-hosts/inventories/related/sources/edit/sources-edit.controller.js b/awx/ui/client/src/inventories-hosts/inventories/related/sources/edit/sources-edit.controller.js index 1bbaecaf4b..6da60e6b3b 100644 --- a/awx/ui/client/src/inventories-hosts/inventories/related/sources/edit/sources-edit.controller.js +++ b/awx/ui/client/src/inventories-hosts/inventories/related/sources/edit/sources-edit.controller.js @@ -233,6 +233,8 @@ export default ['$state', '$scope', 'ParseVariableString', 'ParseTypeChange', $scope.formSave = function() { var params; + console.log($scope); + params = { id: inventorySourceData.id, name: $scope.name, @@ -246,6 +248,9 @@ export default ['$state', '$scope', 'ParseVariableString', 'ParseTypeChange', update_cache_timeout: $scope.update_cache_timeout || 0, verbosity: $scope.verbosity.value, custom_virtualenv: $scope.custom_virtualenv || null, + enabled_var: $scope.enabled_var, + enabled_value: $scope.enabled_value, + host_filter: $scope.host_filter }; if ($scope.source) { diff --git a/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js b/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js index 2594ad29da..c49904b2e3 100644 --- a/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js +++ b/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js @@ -300,6 +300,36 @@ export default ['NotificationsList', 'i18n', function(NotificationsList, i18n){ ngDisabled: '!(inventory_source_obj.summary_fields.user_capabilities.edit || canAdd)', subForm: 'sourceSubForm' }, + host_filter: { + label: i18n._("Host Filter"), + type: 'text', + dataTitle: i18n._('Host Filter'), + dataPlacement: 'right', + awPopOver: "

" + i18n._("Regular expression where only matching hosts will be imported.") + "

", + dataContainer: 'body', + ngDisabled: '!(inventory_source_obj.summary_fields.user_capabilities.edit || canAdd)', + subForm: 'sourceSubForm' + }, + enabled_var: { + label: i18n._("Enabled Variable"), + type: 'text', + dataTitle: i18n._('Enabled Variable'), + dataPlacement: 'right', + awPopOver: "

" + i18n._("Retrieve the enabled state from the given dict of host variables. The enabled variable may be specified using dot notation, e.g: 'foo.bar'") + "

", + dataContainer: 'body', + ngDisabled: '!(inventory_source_obj.summary_fields.user_capabilities.edit || canAdd)', + subForm: 'sourceSubForm' + }, + enabled_value: { + label: i18n._("Enabled Value"), + type: 'text', + dataTitle: i18n._('Enabled Value'), + dataPlacement: 'right', + awPopOver: "

" + i18n._("If the enabled variable matches this value, the host will be enabled on import.") + "

", + dataContainer: 'body', + ngDisabled: '!(inventory_source_obj.summary_fields.user_capabilities.edit || canAdd)', + subForm: 'sourceSubForm' + }, checkbox_group: { label: i18n._('Update Options'), type: 'checkbox_group', From 7d4493e109f344a70d7b1cd7f239338f99414cbf Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Tue, 4 Aug 2020 10:54:28 -0400 Subject: [PATCH 12/27] rename inventory migration --- .../{0118_v380_inventory_plugins.py => 0118_inventory_plugins.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename awx/main/migrations/{0118_v380_inventory_plugins.py => 0118_inventory_plugins.py} (100%) diff --git a/awx/main/migrations/0118_v380_inventory_plugins.py b/awx/main/migrations/0118_inventory_plugins.py similarity index 100% rename from awx/main/migrations/0118_v380_inventory_plugins.py rename to awx/main/migrations/0118_inventory_plugins.py From 12cf607e8aa2f81ebb5a51351c0088f07a1fa840 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Tue, 4 Aug 2020 16:40:15 -0400 Subject: [PATCH 13/27] simplify InventorySourceOption inheritance hack --- awx/main/migrations/0118_inventory_plugins.py | 30 ++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/awx/main/migrations/0118_inventory_plugins.py b/awx/main/migrations/0118_inventory_plugins.py index 5ac36b6c1f..e69a3b7740 100644 --- a/awx/main/migrations/0118_inventory_plugins.py +++ b/awx/main/migrations/0118_inventory_plugins.py @@ -2,10 +2,11 @@ import logging import json +import yaml from django.db import migrations -from awx.main.models.inventory import InventorySourceOptions +from awx.main.models.base import VarsDictProperty from ._inventory_source_vars import FrozenInjectors @@ -14,29 +15,23 @@ logger = logging.getLogger('awx.main.migrations') BACKUP_FILENAME = '/tmp/tower_migration_inventory_source_vars.json' -class InventorySourceOptionsWrapper(InventorySourceOptions): - ''' - InventorySource inherits from InventorySourceOptions but that is not - "recorded" by Django's app registry model tracking. This will, effectively, - reintroduce the inheritance. - ''' - def __init__(self, *args, **kw): - self.target = kw.pop('target') - super().__init__(self, *args, **kw) - def __getattr__(self, attr): - return getattr(self.target, attr) - - def _get_inventory_sources(InventorySource): return InventorySource.objects.filter(source__in=['ec2', 'gce', 'azure_rm', 'vmware', 'satellite6', 'openstack', 'rhv', 'tower']) def inventory_source_vars_forward(apps, schema_editor): InventorySource = apps.get_model("main", "InventorySource") + ''' + The Django app registry does not keep track of model inheritance. The + source_vars_dict property comes from InventorySourceOptions via inheritance. + This adds that property. Luckily, other properteries and functionality from + InventorySourceOptions is not needed by the injector logic. + ''' + setattr(InventorySource, 'source_vars_dict', VarsDictProperty('source_vars')) source_vars_backup = dict() for inv_source_obj in _get_inventory_sources(InventorySource): - inv_source_obj = InventorySourceOptionsWrapper(target=inv_source_obj) + if inv_source_obj.source in FrozenInjectors: source_vars_backup[inv_source_obj.id] = dict(inv_source_obj.source_vars_dict) with open(BACKUP_FILENAME, 'w') as fh: @@ -44,11 +39,12 @@ def inventory_source_vars_forward(apps, schema_editor): injector = FrozenInjectors[inv_source_obj.source]() new_inv_source_vars = injector.inventory_as_dict(inv_source_obj, None) - inv_source_obj.source_vars = new_inv_source_vars + inv_source_obj.source_vars = yaml.dump(new_inv_source_vars) inv_source_obj.save() def inventory_source_vars_backward(apps, schema_editor): + InventorySource = apps.get_model("main", "InventorySource") try: with open(BACKUP_FILENAME, 'r') as fh: source_vars_backup = json.load(fh) @@ -56,7 +52,7 @@ def inventory_source_vars_backward(apps, schema_editor): print(f"Rollback file not found {BACKUP_FILENAME}") return - for inv_source_obj in _get_inventory_sources(): + for inv_source_obj in _get_inventory_sources(InventorySource): if inv_source_obj.id in source_vars_backup: inv_source_obj.source_vars = source_vars_backup[inv_source_obj.id] inv_source_obj.save() From 2fdeba47a5ff90bab4f89b8097209493415f0048 Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Tue, 4 Aug 2020 17:35:12 -0400 Subject: [PATCH 14/27] Add new common inventory source fields --- .../InventorySourceDetail.jsx | 6 +++ .../Inventory/shared/InventorySourceForm.jsx | 6 +++ .../InventorySourceSubForms/AzureSubForm.jsx | 12 +++++- .../CloudFormsSubForm.jsx | 12 +++++- .../InventorySourceSubForms/EC2SubForm.jsx | 12 +++++- .../InventorySourceSubForms/GCESubForm.jsx | 11 ++++- .../OpenStackSubForm.jsx | 12 +++++- .../InventorySourceSubForms/SCMSubForm.jsx | 12 +++++- .../SatelliteSubForm.jsx | 12 +++++- .../InventorySourceSubForms/SharedFields.jsx | 41 +++++++++++++++++++ .../InventorySourceSubForms/TowerSubForm.jsx | 11 ++++- .../InventorySourceSubForms/VMwareSubForm.jsx | 12 +++++- .../VirtualizationSubForm.jsx | 11 ++++- 13 files changed, 160 insertions(+), 10 deletions(-) diff --git a/awx/ui_next/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.jsx b/awx/ui_next/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.jsx index badd28fb26..a67c3c2064 100644 --- a/awx/ui_next/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.jsx +++ b/awx/ui_next/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.jsx @@ -39,6 +39,9 @@ function InventorySourceDetail({ inventorySource, i18n }) { update_on_launch, update_on_project_update, verbosity, + enabled_var, + enabled_value, + host_filter, summary_fields: { created_by, credentials, @@ -220,6 +223,9 @@ function InventorySourceDetail({ inventorySource, i18n }) { label={i18n._(t`Cache timeout`)} value={`${update_cache_timeout} ${i18n._(t`seconds`)}`} /> + + + {credentials?.length > 0 && ( { update_on_launch: false, update_on_project_update: false, verbosity: 1, + enabled_var: '', + enabled_value: '', + host_filter: '', }; Object.keys(defaults).forEach(label => { setFieldValue(label, defaults[label]); @@ -209,6 +212,9 @@ const InventorySourceForm = ({ update_on_launch: source?.update_on_launch || false, update_on_project_update: source?.update_on_project_update || false, verbosity: source?.verbosity || 1, + enabled_var: source?.enabled_var || '', + enabled_value: source?.enabled_value || '', + host_filter: source?.host_filter || '', }; const { diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/AzureSubForm.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/AzureSubForm.jsx index 99b83ca6b5..535b364691 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/AzureSubForm.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/AzureSubForm.jsx @@ -3,7 +3,14 @@ import { useField } from 'formik'; import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; import CredentialLookup from '../../../../components/Lookup/CredentialLookup'; -import { OptionsField, SourceVarsField, VerbosityField } from './SharedFields'; +import { + OptionsField, + SourceVarsField, + VerbosityField, + EnabledVarField, + EnabledValueField, + HostFilterField, +} from './SharedFields'; const AzureSubForm = ({ i18n }) => { const [credentialField, credentialMeta, credentialHelpers] = useField( @@ -25,6 +32,9 @@ const AzureSubForm = ({ i18n }) => { required /> + + + diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/CloudFormsSubForm.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/CloudFormsSubForm.jsx index 68aeed4d76..7db4431fdd 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/CloudFormsSubForm.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/CloudFormsSubForm.jsx @@ -3,7 +3,14 @@ import { useField } from 'formik'; import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; import CredentialLookup from '../../../../components/Lookup/CredentialLookup'; -import { OptionsField, SourceVarsField, VerbosityField } from './SharedFields'; +import { + OptionsField, + SourceVarsField, + VerbosityField, + EnabledVarField, + EnabledValueField, + HostFilterField, +} from './SharedFields'; const CloudFormsSubForm = ({ i18n }) => { const [credentialField, credentialMeta, credentialHelpers] = useField( @@ -25,6 +32,9 @@ const CloudFormsSubForm = ({ i18n }) => { required /> + + + diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/EC2SubForm.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/EC2SubForm.jsx index 32fc581742..33447a2229 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/EC2SubForm.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/EC2SubForm.jsx @@ -3,7 +3,14 @@ import { useField } from 'formik'; import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; import CredentialLookup from '../../../../components/Lookup/CredentialLookup'; -import { OptionsField, SourceVarsField, VerbosityField } from './SharedFields'; +import { + OptionsField, + SourceVarsField, + VerbosityField, + EnabledVarField, + EnabledValueField, + HostFilterField, +} from './SharedFields'; const EC2SubForm = ({ i18n }) => { const [credentialField, , credentialHelpers] = useField('credential'); @@ -18,6 +25,9 @@ const EC2SubForm = ({ i18n }) => { }} /> + + + diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/GCESubForm.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/GCESubForm.jsx index fcbbffa040..a6c8ce5cbd 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/GCESubForm.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/GCESubForm.jsx @@ -3,7 +3,13 @@ import { useField } from 'formik'; import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; import CredentialLookup from '../../../../components/Lookup/CredentialLookup'; -import { OptionsField, VerbosityField } from './SharedFields'; +import { + OptionsField, + VerbosityField, + EnabledVarField, + EnabledValueField, + HostFilterField, +} from './SharedFields'; const GCESubForm = ({ i18n }) => { const [credentialField, credentialMeta, credentialHelpers] = useField( @@ -25,6 +31,9 @@ const GCESubForm = ({ i18n }) => { required /> + + + ); diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/OpenStackSubForm.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/OpenStackSubForm.jsx index 7c61fb5d16..55a142e936 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/OpenStackSubForm.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/OpenStackSubForm.jsx @@ -3,7 +3,14 @@ import { useField } from 'formik'; import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; import CredentialLookup from '../../../../components/Lookup/CredentialLookup'; -import { OptionsField, SourceVarsField, VerbosityField } from './SharedFields'; +import { + OptionsField, + SourceVarsField, + VerbosityField, + EnabledVarField, + EnabledValueField, + HostFilterField, +} from './SharedFields'; const OpenStackSubForm = ({ i18n }) => { const [credentialField, credentialMeta, credentialHelpers] = useField( @@ -25,6 +32,9 @@ const OpenStackSubForm = ({ i18n }) => { required /> + + + diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.jsx index b338088a8d..858e209cc5 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SCMSubForm.jsx @@ -11,7 +11,14 @@ import AnsibleSelect from '../../../../components/AnsibleSelect'; import { FieldTooltip } from '../../../../components/FormField'; import CredentialLookup from '../../../../components/Lookup/CredentialLookup'; import ProjectLookup from '../../../../components/Lookup/ProjectLookup'; -import { VerbosityField, OptionsField, SourceVarsField } from './SharedFields'; +import { + OptionsField, + SourceVarsField, + VerbosityField, + EnabledVarField, + EnabledValueField, + HostFilterField, +} from './SharedFields'; const SCMSubForm = ({ i18n }) => { const [credentialField, , credentialHelpers] = useField('credential'); @@ -121,6 +128,9 @@ const SCMSubForm = ({ i18n }) => { /> + + + diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SatelliteSubForm.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SatelliteSubForm.jsx index 641539f978..573be61679 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SatelliteSubForm.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SatelliteSubForm.jsx @@ -3,7 +3,14 @@ import { useField } from 'formik'; import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; import CredentialLookup from '../../../../components/Lookup/CredentialLookup'; -import { OptionsField, SourceVarsField, VerbosityField } from './SharedFields'; +import { + OptionsField, + SourceVarsField, + VerbosityField, + EnabledVarField, + EnabledValueField, + HostFilterField, +} from './SharedFields'; const SatelliteSubForm = ({ i18n }) => { const [credentialField, credentialMeta, credentialHelpers] = useField( @@ -25,6 +32,9 @@ const SatelliteSubForm = ({ i18n }) => { required /> + + + diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SharedFields.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SharedFields.jsx index 3ca698641b..14e0bbd896 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SharedFields.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SharedFields.jsx @@ -154,3 +154,44 @@ export const OptionsField = withI18n()( ); } ); + +export const EnabledVarField = withI18n()(({ i18n }) => { + return ( + + ); +}); + +export const EnabledValueField = withI18n()(({ i18n }) => { + return ( + + ); +}); + +export const HostFilterField = withI18n()(({ i18n }) => { + return ( + + ); +}); diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/TowerSubForm.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/TowerSubForm.jsx index 3af0f9a5c6..0dd6d7be52 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/TowerSubForm.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/TowerSubForm.jsx @@ -3,7 +3,13 @@ import { useField } from 'formik'; import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; import CredentialLookup from '../../../../components/Lookup/CredentialLookup'; -import { OptionsField, VerbosityField } from './SharedFields'; +import { + OptionsField, + VerbosityField, + EnabledVarField, + EnabledValueField, + HostFilterField, +} from './SharedFields'; const TowerSubForm = ({ i18n }) => { const [credentialField, credentialMeta, credentialHelpers] = useField( @@ -25,6 +31,9 @@ const TowerSubForm = ({ i18n }) => { required /> + + + ); diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/VMwareSubForm.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/VMwareSubForm.jsx index 555b0498e3..08f1342cfe 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/VMwareSubForm.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/VMwareSubForm.jsx @@ -3,7 +3,14 @@ import { useField } from 'formik'; import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; import CredentialLookup from '../../../../components/Lookup/CredentialLookup'; -import { OptionsField, SourceVarsField, VerbosityField } from './SharedFields'; +import { + OptionsField, + SourceVarsField, + VerbosityField, + EnabledVarField, + EnabledValueField, + HostFilterField, +} from './SharedFields'; const VMwareSubForm = ({ i18n }) => { const [credentialField, credentialMeta, credentialHelpers] = useField( @@ -25,6 +32,9 @@ const VMwareSubForm = ({ i18n }) => { required /> + + + diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/VirtualizationSubForm.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/VirtualizationSubForm.jsx index 4d558b74a6..fda2a58ec9 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/VirtualizationSubForm.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/VirtualizationSubForm.jsx @@ -3,7 +3,13 @@ import { useField } from 'formik'; import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; import CredentialLookup from '../../../../components/Lookup/CredentialLookup'; -import { OptionsField, VerbosityField } from './SharedFields'; +import { + OptionsField, + VerbosityField, + EnabledVarField, + EnabledValueField, + HostFilterField, +} from './SharedFields'; const VirtualizationSubForm = ({ i18n }) => { const [credentialField, credentialMeta, credentialHelpers] = useField( @@ -25,6 +31,9 @@ const VirtualizationSubForm = ({ i18n }) => { required /> + + + ); From c7794fc3e4084d3322720f970f31cc9355abdb32 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Wed, 5 Aug 2020 09:38:55 -0400 Subject: [PATCH 15/27] fix missed import --- awx/main/migrations/_inventory_source_vars.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/awx/main/migrations/_inventory_source_vars.py b/awx/main/migrations/_inventory_source_vars.py index 857bb1c07e..edd2e6827a 100644 --- a/awx/main/migrations/_inventory_source_vars.py +++ b/awx/main/migrations/_inventory_source_vars.py @@ -1,3 +1,5 @@ +import json + from django.utils.translation import ugettext_lazy as _ From 48fb1e973c9c2e7e13f7808de37d56208186fb1e Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Wed, 5 Aug 2020 10:02:18 -0400 Subject: [PATCH 16/27] overwrite `plugin:` at runtime * Before, we were re-writing `plugin:` when users updated the InventorySource via the API. Now, we just override at run-time. This makes for a more sane API interaction --- awx/main/models/inventory.py | 20 ++++++------ .../tests/functional/api/test_inventory.py | 32 ------------------- 2 files changed, 11 insertions(+), 41 deletions(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 7d9069e4f0..5121f081ed 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -1158,13 +1158,6 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions, CustomVirtualE raise ValidationError(_("Cannot set source_path if not SCM type.")) return self.source_path - def clean_source_vars(self): - injector = self.injectors.get(self.source) - source_vars = dict(self.source_vars_dict) # make a copy - if injector and self.source_vars_dict.get('plugin', '') != injector.get_proper_name(): - source_vars['plugin'] = injector.get_proper_name() - return json.dumps(source_vars) - ''' RelatedJobsMixin ''' @@ -1369,13 +1362,21 @@ class PluginFileInjector(object): """Returns a string that is the content for the inventory file for the inventory plugin """ return yaml.safe_dump( - inventory_update.source_vars_dict, + self.inventory_as_dict(inventory_update, private_data_dir), default_flow_style=False, width=1000 ) def inventory_as_dict(self, inventory_update, private_data_dir): - return inventory_update.source_vars_dict + source_vars = dict(inventory_update.source_vars_dict) # make a copy + proper_name = self.get_proper_name() + ''' + None conveys that we should use the user-provided plugin. + Note that a plugin value of '' should still be overridden. + ''' + if proper_name is not None: + source_vars['plugin'] = proper_name + return source_vars def build_env(self, inventory_update, env, private_data_dir, private_data_files): injector_env = self.get_plugin_env(inventory_update, private_data_dir, private_data_files) @@ -1463,6 +1464,7 @@ class gce(PluginFileInjector): def inventory_as_dict(self, inventory_update, private_data_dir): ret = super().inventory_as_dict(inventory_update, private_data_dir) credential = inventory_update.get_cloud_credential() + # TODO: Align precedence of ENV vs. inventory config w/ Ansible behavior ret['projects'] = [credential.get_input('project', default='')] return ret diff --git a/awx/main/tests/functional/api/test_inventory.py b/awx/main/tests/functional/api/test_inventory.py index a04a68c9bb..5bad1b6f30 100644 --- a/awx/main/tests/functional/api/test_inventory.py +++ b/awx/main/tests/functional/api/test_inventory.py @@ -456,38 +456,6 @@ def test_inventory_source_vars_prohibition(post, inventory, admin_user): assert 'FOOBAR' in r.data['source_vars'][0] -@pytest.mark.django_db -@pytest.mark.parametrize('source,source_var_actual,source_var_expected,description', [ - ('ec2', {'plugin': 'blah'}, {'plugin': 'amazon.aws.aws_ec2'}, 'source plugin mismatch'), - ('ec2', {'plugin': 'amazon.aws.aws_ec2'}, {'plugin': 'amazon.aws.aws_ec2'}, 'valid plugin'), -]) -def test_inventory_source_vars_source_plugin_ok(post, inventory, admin_user, source, source_var_actual, source_var_expected, description): - r = post(reverse('api:inventory_source_list'), - {'name': 'new inv src', 'source_vars': json.dumps(source_var_actual), 'inventory': inventory.pk, 'source': source}, - admin_user, expect=201) - - assert r.data['source_vars'] == json.dumps(source_var_expected) - - -@pytest.mark.django_db -@pytest.mark.parametrize('source_var_actual,description', [ - ({'plugin': 'namespace.collection.script'}, 'scm source type source_vars are ignored valid'), - ({'plugin': 'namespace.collection.script'}, 'scm source type source_vars are ignored invalid'), - ({'plugin': ''}, 'scm source type source_vars are ignored blank'), - ({}, 'scm source type source_vars are ignored non-existent'), -]) -def test_inventory_source_vars_source_plugin_scm_ok(post, inventory, admin_user, project, source_var_actual, description): - r = post(reverse('api:inventory_source_list'), - {'name': 'new inv src', - 'source_vars': json.dumps(source_var_actual), - 'inventory': inventory.pk, - 'source': 'scm', - 'source_project': project.id,}, - admin_user, expect=201) - - assert r.data['source_vars'] == json.dumps(source_var_actual) - - @pytest.mark.django_db @pytest.mark.parametrize('role,expect', [ ('admin_role', 200), From d518891520a3e04d99b8f9b752c273ceb78d19ba Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Wed, 5 Aug 2020 12:06:37 -0400 Subject: [PATCH 17/27] inventory plugin inventory parameter take precedence over env vars --- awx/main/models/inventory.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 5121f081ed..6e3a22434c 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -1464,8 +1464,9 @@ class gce(PluginFileInjector): def inventory_as_dict(self, inventory_update, private_data_dir): ret = super().inventory_as_dict(inventory_update, private_data_dir) credential = inventory_update.get_cloud_credential() - # TODO: Align precedence of ENV vs. inventory config w/ Ansible behavior - ret['projects'] = [credential.get_input('project', default='')] + # InventorySource.source_vars take precedence over ENV vars + if 'projects' not in ret: + ret['projects'] = [credential.get_input('project', default='')] return ret From a9cdf076904f1782ddfcbbe48bff7b068b40d934 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Wed, 5 Aug 2020 13:33:24 -0400 Subject: [PATCH 18/27] push global invsource fields onto invsource obj --- awx/api/serializers.py | 2 +- awx/main/migrations/0118_inventory_plugins.py | 32 ++++++++++++++- awx/main/models/inventory.py | 34 ++++++++++++++- awx/main/tasks.py | 30 ++++++-------- awx/settings/defaults.py | 41 +------------------ 5 files changed, 79 insertions(+), 60 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 9cf953262b..83575025e7 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1937,7 +1937,7 @@ class InventorySourceOptionsSerializer(BaseSerializer): class Meta: fields = ('*', 'source', 'source_path', 'source_script', 'source_vars', 'credential', - 'overwrite', 'overwrite_vars', + 'enabled_var', 'enabled_value', 'host_filter', 'overwrite', 'overwrite_vars', 'custom_virtualenv', 'timeout', 'verbosity') def get_related(self, obj): diff --git a/awx/main/migrations/0118_inventory_plugins.py b/awx/main/migrations/0118_inventory_plugins.py index e69a3b7740..da21337d69 100644 --- a/awx/main/migrations/0118_inventory_plugins.py +++ b/awx/main/migrations/0118_inventory_plugins.py @@ -4,7 +4,7 @@ import logging import json import yaml -from django.db import migrations +from django.db import migrations, models from awx.main.models.base import VarsDictProperty @@ -90,4 +90,34 @@ class Migration(migrations.Migration): model_name='inventoryupdate', name='source_regions', ), + migrations.AddField( + model_name='inventorysource', + name='enabled_value', + field=models.TextField(blank=True, default='', help_text='Only used when enabled_var is set. Value when the host is considered enabled. For example if enabled_var="status.power_state"and enabled_value="powered_on" with host variables:{ "status": { "power_state": "powered_on", "created": "2020-08-04T18:13:04+00:00", "healthy": true }, "name": "foobar", "ip_address": "192.168.2.1"}The host would be marked enabled. If power_state where any value other than powered_on then the host would be disabled when imported into Tower. If the key is not found then the host will be enabled'), + ), + migrations.AddField( + model_name='inventorysource', + name='enabled_var', + field=models.TextField(blank=True, default='', help_text='Retrieve the enabled state from the given dict of host variables. The enabled variable may be specified as "foo.bar", in which case the lookup will traverse into nested dicts, equivalent to: from_dict.get("foo", {}).get("bar", default)'), + ), + migrations.AddField( + model_name='inventorysource', + name='host_filter', + field=models.TextField(blank=True, default='', help_text='Regex where only matching hosts will be imported into Tower.'), + ), + migrations.AddField( + model_name='inventoryupdate', + name='enabled_value', + field=models.TextField(blank=True, default='', help_text='Only used when enabled_var is set. Value when the host is considered enabled. For example if enabled_var="status.power_state"and enabled_value="powered_on" with host variables:{ "status": { "power_state": "powered_on", "created": "2020-08-04T18:13:04+00:00", "healthy": true }, "name": "foobar", "ip_address": "192.168.2.1"}The host would be marked enabled. If power_state where any value other than powered_on then the host would be disabled when imported into Tower. If the key is not found then the host will be enabled'), + ), + migrations.AddField( + model_name='inventoryupdate', + name='enabled_var', + field=models.TextField(blank=True, default='', help_text='Retrieve the enabled state from the given dict of host variables. The enabled variable may be specified as "foo.bar", in which case the lookup will traverse into nested dicts, equivalent to: from_dict.get("foo", {}).get("bar", default)'), + ), + migrations.AddField( + model_name='inventoryupdate', + name='host_filter', + field=models.TextField(blank=True, default='', help_text='Regex where only matching hosts will be imported into Tower.'), + ), ] diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 6e3a22434c..79568ce2a2 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -7,7 +7,6 @@ import time import logging import re import copy -import json import os.path from urllib.parse import urljoin import yaml @@ -863,6 +862,39 @@ class InventorySourceOptions(BaseModel): default='', help_text=_('Inventory source variables in YAML or JSON format.'), ) + enabled_var = models.TextField( + blank=True, + default='', + help_text=_('Retrieve the enabled state from the given dict of host ' + 'variables. The enabled variable may be specified as "foo.bar", ' + 'in which case the lookup will traverse into nested dicts, ' + 'equivalent to: from_dict.get("foo", {}).get("bar", default)'), + ) + enabled_value = models.TextField( + blank=True, + default='', + help_text=_('Only used when enabled_var is set. Value when the host is ' + 'considered enabled. For example if enabled_var="status.power_state"' + 'and enabled_value="powered_on" with host variables:' + '{' + ' "status": {' + ' "power_state": "powered_on",' + ' "created": "2020-08-04T18:13:04+00:00",' + ' "healthy": true' + ' },' + ' "name": "foobar",' + ' "ip_address": "192.168.2.1"' + '}' + 'The host would be marked enabled. If power_state where any ' + 'value other than powered_on then the host would be disabled ' + 'when imported into Tower. If the key is not found then the ' + 'host will be enabled'), + ) + host_filter = models.TextField( + blank=True, + default='', + help_text=_('Regex where only matching hosts will be imported into Tower.'), + ) overwrite = models.BooleanField( default=False, help_text=_('Overwrite local groups and hosts from remote inventory source.'), diff --git a/awx/main/tasks.py b/awx/main/tasks.py index acd7548a91..4809d2c136 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -23,6 +23,7 @@ import fcntl from pathlib import Path from uuid import uuid4 import urllib.parse as urlparse +import shlex # Django from django.conf import settings @@ -2559,23 +2560,18 @@ class RunInventoryUpdate(BaseTask): args.extend(['--venv', inventory_update.ansible_virtualenv_path]) src = inventory_update.source - # Add several options to the shell arguments based on the - # inventory-source-specific setting in the AWX configuration. - # These settings are "per-source"; it's entirely possible that - # they will be different between cloud providers if an AWX user - # actively uses more than one. - if getattr(settings, '%s_ENABLED_VAR' % src.upper(), False): - args.extend(['--enabled-var', - getattr(settings, '%s_ENABLED_VAR' % src.upper())]) - if getattr(settings, '%s_ENABLED_VALUE' % src.upper(), False): - args.extend(['--enabled-value', - getattr(settings, '%s_ENABLED_VALUE' % src.upper())]) - if getattr(settings, '%s_GROUP_FILTER' % src.upper(), False): - args.extend(['--group-filter', - getattr(settings, '%s_GROUP_FILTER' % src.upper())]) - if getattr(settings, '%s_HOST_FILTER' % src.upper(), False): - args.extend(['--host-filter', - getattr(settings, '%s_HOST_FILTER' % src.upper())]) + if inventory_update.enabled_var: + args.extend(['--enabled-var', shlex.quote(inventory_update.enabled_var)]) + args.extend(['--enabled-value', shlex.quote(inventory_update.enabled_value)]) + else: + if getattr(settings, '%s_ENABLED_VAR' % src.upper(), False): + args.extend(['--enabled-var', + getattr(settings, '%s_ENABLED_VAR' % src.upper())]) + if getattr(settings, '%s_ENABLED_VALUE' % src.upper(), False): + args.extend(['--enabled-value', + getattr(settings, '%s_ENABLED_VALUE' % src.upper())]) + if inventory_update.host_filter: + args.extend(['--host-filter', shlex.quote(inventory_update.host_filter)]) if getattr(settings, '%s_EXCLUDE_EMPTY_GROUPS' % src.upper()): args.append('--exclude-empty-groups') if getattr(settings, '%s_INSTANCE_ID_VAR' % src.upper(), False): diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 93e373e401..5aa0b834ea 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -669,59 +669,32 @@ INV_ENV_VARIABLE_BLOCKED = ("HOME", "USER", "_", "TERM") # ---------------- # -- Amazon EC2 -- # ---------------- - -# Inventory variable name/values for determining if host is active/enabled. EC2_ENABLED_VAR = 'ec2_state' EC2_ENABLED_VALUE = 'running' - -# Inventory variable name containing unique instance ID. EC2_INSTANCE_ID_VAR = 'ec2_id' - -# Filter for allowed group/host names when importing inventory from EC2. -EC2_GROUP_FILTER = r'^.+$' -EC2_HOST_FILTER = r'^.+$' EC2_EXCLUDE_EMPTY_GROUPS = True - # ------------ # -- VMware -- # ------------ -# Inventory variable name/values for determining whether a host is -# active in vSphere. VMWARE_ENABLED_VAR = 'guest.gueststate' VMWARE_ENABLED_VALUE = 'running' - -# Inventory variable name containing the unique instance ID. VMWARE_INSTANCE_ID_VAR = 'config.instanceUuid, config.instanceuuid' - -# Filter for allowed group and host names when importing inventory -# from VMware. -VMWARE_GROUP_FILTER = r'^.+$' -VMWARE_HOST_FILTER = r'^.+$' VMWARE_EXCLUDE_EMPTY_GROUPS = True VMWARE_VALIDATE_CERTS = False + # --------------------------- # -- Google Compute Engine -- # --------------------------- - -# Inventory variable name/value for determining whether a host is active -# in Google Compute Engine. GCE_ENABLED_VAR = 'status' GCE_ENABLED_VALUE = 'running' - -# Filter for allowed group and host names when importing inventory from -# Google Compute Engine. -GCE_GROUP_FILTER = r'^.+$' -GCE_HOST_FILTER = r'^.+$' GCE_EXCLUDE_EMPTY_GROUPS = True GCE_INSTANCE_ID_VAR = 'gce_id' # -------------------------------------- # -- Microsoft Azure Resource Manager -- # -------------------------------------- -AZURE_RM_GROUP_FILTER = r'^.+$' -AZURE_RM_HOST_FILTER = r'^.+$' AZURE_RM_ENABLED_VAR = 'powerstate' AZURE_RM_ENABLED_VALUE = 'running' AZURE_RM_INSTANCE_ID_VAR = 'id' @@ -732,8 +705,6 @@ AZURE_RM_EXCLUDE_EMPTY_GROUPS = True # --------------------- OPENSTACK_ENABLED_VAR = 'status' OPENSTACK_ENABLED_VALUE = 'ACTIVE' -OPENSTACK_GROUP_FILTER = r'^.+$' -OPENSTACK_HOST_FILTER = r'^.+$' OPENSTACK_EXCLUDE_EMPTY_GROUPS = True OPENSTACK_INSTANCE_ID_VAR = 'openstack.id' @@ -742,8 +713,6 @@ OPENSTACK_INSTANCE_ID_VAR = 'openstack.id' # --------------------- RHV_ENABLED_VAR = 'status' RHV_ENABLED_VALUE = 'up' -RHV_GROUP_FILTER = r'^.+$' -RHV_HOST_FILTER = r'^.+$' RHV_EXCLUDE_EMPTY_GROUPS = True RHV_INSTANCE_ID_VAR = 'id' @@ -752,8 +721,6 @@ RHV_INSTANCE_ID_VAR = 'id' # --------------------- TOWER_ENABLED_VAR = 'remote_tower_enabled' TOWER_ENABLED_VALUE = 'true' -TOWER_GROUP_FILTER = r'^.+$' -TOWER_HOST_FILTER = r'^.+$' TOWER_EXCLUDE_EMPTY_GROUPS = True TOWER_INSTANCE_ID_VAR = 'remote_tower_id' @@ -762,8 +729,6 @@ TOWER_INSTANCE_ID_VAR = 'remote_tower_id' # --------------------- SATELLITE6_ENABLED_VAR = 'foreman.enabled' SATELLITE6_ENABLED_VALUE = 'True' -SATELLITE6_GROUP_FILTER = r'^.+$' -SATELLITE6_HOST_FILTER = r'^.+$' SATELLITE6_EXCLUDE_EMPTY_GROUPS = True SATELLITE6_INSTANCE_ID_VAR = 'foreman.id' # SATELLITE6_GROUP_PREFIX and SATELLITE6_GROUP_PATTERNS defined in source vars @@ -773,8 +738,6 @@ SATELLITE6_INSTANCE_ID_VAR = 'foreman.id' # --------------------- #CUSTOM_ENABLED_VAR = #CUSTOM_ENABLED_VALUE = -CUSTOM_GROUP_FILTER = r'^.+$' -CUSTOM_HOST_FILTER = r'^.+$' CUSTOM_EXCLUDE_EMPTY_GROUPS = False #CUSTOM_INSTANCE_ID_VAR = @@ -783,8 +746,6 @@ CUSTOM_EXCLUDE_EMPTY_GROUPS = False # --------------------- #SCM_ENABLED_VAR = #SCM_ENABLED_VALUE = -SCM_GROUP_FILTER = r'^.+$' -SCM_HOST_FILTER = r'^.+$' SCM_EXCLUDE_EMPTY_GROUPS = False #SCM_INSTANCE_ID_VAR = From f04aff81c49f33f0e3d0ded9a88256ba2304771e Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Thu, 27 Aug 2020 11:42:55 -0400 Subject: [PATCH 19/27] Add details to inv source field help text --- .../inventories/related/sources/sources.form.js | 4 ++-- .../Inventory/shared/InventorySourceSubForms/SharedFields.jsx | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js b/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js index c49904b2e3..66b6693d58 100644 --- a/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js +++ b/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js @@ -305,7 +305,7 @@ export default ['NotificationsList', 'i18n', function(NotificationsList, i18n){ type: 'text', dataTitle: i18n._('Host Filter'), dataPlacement: 'right', - awPopOver: "

" + i18n._("Regular expression where only matching hosts will be imported.") + "

", + awPopOver: "

" + i18n._("Regular expression where only matching host names will be imported. The filter is applied as a post-processing step after any inventory plugin filters are applied.") + "

", dataContainer: 'body', ngDisabled: '!(inventory_source_obj.summary_fields.user_capabilities.edit || canAdd)', subForm: 'sourceSubForm' @@ -325,7 +325,7 @@ export default ['NotificationsList', 'i18n', function(NotificationsList, i18n){ type: 'text', dataTitle: i18n._('Enabled Value'), dataPlacement: 'right', - awPopOver: "

" + i18n._("If the enabled variable matches this value, the host will be enabled on import.") + "

", + awPopOver: "

" + i18n._("This field is ignored unless an Enabled Variable is set. If the enabled variable matches this value, the host will be enabled on import.") + "

", dataContainer: 'body', ngDisabled: '!(inventory_source_obj.summary_fields.user_capabilities.edit || canAdd)', subForm: 'sourceSubForm' diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SharedFields.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SharedFields.jsx index 14e0bbd896..8150a9a45f 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SharedFields.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SharedFields.jsx @@ -174,7 +174,7 @@ export const EnabledValueField = withI18n()(({ i18n }) => { id="inventory-enabled-value" label={i18n._(t`Enabled Value`)} tooltip={i18n._( - t`If the enabled variable matches this value, the host will be enabled on import.` + t`This field is ignored unless an Enabled Variable is set. If the enabled variable matches this value, the host will be enabled on import.` )} name="enabled_value" type="text" @@ -188,7 +188,7 @@ export const HostFilterField = withI18n()(({ i18n }) => { id="host-filter" label={i18n._(t`Host Filter`)} tooltip={i18n._( - t`Regular expression where only matching hosts will be imported.` + t`Regular expression where only matching host names will be imported. The filter is applied as a post-processing step after any inventory plugin filters are applied.` )} name="host_filter" type="text" From dcf5917a4e996c306005a9641857d7b01b695773 Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Thu, 27 Aug 2020 12:13:40 -0400 Subject: [PATCH 20/27] Check that host_filter is regexp --- .../shared/InventorySourceSubForms/SharedFields.jsx | 3 ++- awx/ui_next/src/util/validators.jsx | 11 +++++++++++ awx/ui_next/src/util/validators.test.js | 10 ++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SharedFields.jsx b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SharedFields.jsx index 8150a9a45f..35e2c3fc30 100644 --- a/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SharedFields.jsx +++ b/awx/ui_next/src/screens/Inventory/shared/InventorySourceSubForms/SharedFields.jsx @@ -3,7 +3,7 @@ import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; import { useField } from 'formik'; import { FormGroup } from '@patternfly/react-core'; -import { minMaxValue } from '../../../../util/validators'; +import { minMaxValue, regExp } from '../../../../util/validators'; import AnsibleSelect from '../../../../components/AnsibleSelect'; import { VariablesField } from '../../../../components/CodeMirrorInput'; import FormField, { @@ -192,6 +192,7 @@ export const HostFilterField = withI18n()(({ i18n }) => { )} name="host_filter" type="text" + validate={regExp(i18n)} /> ); }); diff --git a/awx/ui_next/src/util/validators.jsx b/awx/ui_next/src/util/validators.jsx index ee97cd4702..035b4dfb56 100644 --- a/awx/ui_next/src/util/validators.jsx +++ b/awx/ui_next/src/util/validators.jsx @@ -88,3 +88,14 @@ export function combine(validators) { return undefined; }; } + +export function regExp(i18n) { + return value => { + try { + RegExp(value); + } catch { + return i18n._(t`This field must be a regular expression`); + } + return undefined; + }; +} diff --git a/awx/ui_next/src/util/validators.test.js b/awx/ui_next/src/util/validators.test.js index f3e37c6cde..11b1a3bfd9 100644 --- a/awx/ui_next/src/util/validators.test.js +++ b/awx/ui_next/src/util/validators.test.js @@ -5,6 +5,7 @@ import { noWhiteSpace, integer, combine, + regExp, } from './validators'; const i18n = { _: val => val }; @@ -128,4 +129,13 @@ describe('validators', () => { }); expect(combine(validators)('ok')).toBeUndefined(); }); + + test('regExp rejects invalid regular expression', () => { + expect(regExp(i18n)('[')).toEqual({ + id: 'This field must be a regular expression', + }); + expect(regExp(i18n)('')).toBeUndefined(); + expect(regExp(i18n)('ok')).toBeUndefined(); + expect(regExp(i18n)('[^a-zA-Z]')).toBeUndefined(); + }); }); From 03ad1aa1412a22b98a14f54c83fe840065d259af Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Thu, 27 Aug 2020 14:32:17 -0400 Subject: [PATCH 21/27] remove backwords migraiton support for inv plugins * Do not write out inventory source_vars to a file on disk as they _may_ contain sensitive information. This also removes support for backwards migrations. This is fine, backwards migration is really only useful during development. --- awx/main/migrations/0118_inventory_plugins.py | 21 +------------------ 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/awx/main/migrations/0118_inventory_plugins.py b/awx/main/migrations/0118_inventory_plugins.py index da21337d69..991de733d3 100644 --- a/awx/main/migrations/0118_inventory_plugins.py +++ b/awx/main/migrations/0118_inventory_plugins.py @@ -1,7 +1,6 @@ # Generated by Django 2.2.11 on 2020-07-20 19:56 import logging -import json import yaml from django.db import migrations, models @@ -12,7 +11,6 @@ from ._inventory_source_vars import FrozenInjectors logger = logging.getLogger('awx.main.migrations') -BACKUP_FILENAME = '/tmp/tower_migration_inventory_source_vars.json' def _get_inventory_sources(InventorySource): @@ -34,8 +32,6 @@ def inventory_source_vars_forward(apps, schema_editor): if inv_source_obj.source in FrozenInjectors: source_vars_backup[inv_source_obj.id] = dict(inv_source_obj.source_vars_dict) - with open(BACKUP_FILENAME, 'w') as fh: - json.dump(source_vars_backup, fh) injector = FrozenInjectors[inv_source_obj.source]() new_inv_source_vars = injector.inventory_as_dict(inv_source_obj, None) @@ -43,21 +39,6 @@ def inventory_source_vars_forward(apps, schema_editor): inv_source_obj.save() -def inventory_source_vars_backward(apps, schema_editor): - InventorySource = apps.get_model("main", "InventorySource") - try: - with open(BACKUP_FILENAME, 'r') as fh: - source_vars_backup = json.load(fh) - except FileNotFoundError: - print(f"Rollback file not found {BACKUP_FILENAME}") - return - - for inv_source_obj in _get_inventory_sources(InventorySource): - if inv_source_obj.id in source_vars_backup: - inv_source_obj.source_vars = source_vars_backup[inv_source_obj.id] - inv_source_obj.save() - - class Migration(migrations.Migration): dependencies = [ @@ -65,7 +46,7 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(inventory_source_vars_forward, inventory_source_vars_backward,), + migrations.RunPython(inventory_source_vars_forward), migrations.RemoveField( model_name='inventorysource', name='group_by', From 99aff939309ea05948acfea83dc25ac0e75b2e16 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Thu, 27 Aug 2020 14:23:50 -0400 Subject: [PATCH 22/27] Get rid of ansible version checking --- awx/main/management/commands/inventory_import.py | 14 ++++---------- awx/main/models/inventory.py | 4 ---- awx/main/tasks.py | 14 ++++---------- awx/main/tests/unit/test_tasks.py | 7 ------- awx/main/utils/common.py | 10 +++------- 5 files changed, 11 insertions(+), 38 deletions(-) diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index 2e51283445..f0aa2feff2 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -39,7 +39,6 @@ from awx.main.utils import ( build_proot_temp_dir, get_licenser ) -from awx.main.utils.common import _get_ansible_version from awx.main.signals import disable_activity_stream from awx.main.constants import STANDARD_INVENTORY_UPDATE_ENV from awx.main.utils.pglock import advisory_lock @@ -136,15 +135,10 @@ class AnsibleInventoryLoader(object): # inside of /venv/ansible, so we override the specified interpreter # https://github.com/ansible/ansible/issues/50714 bargs = ['python', ansible_inventory_path, '-i', self.source] - ansible_version = _get_ansible_version(ansible_inventory_path[:-len('-inventory')]) - if ansible_version != 'unknown': - this_version = Version(ansible_version) - if this_version >= Version('2.5'): - bargs.extend(['--playbook-dir', self.source_dir]) - if this_version >= Version('2.8'): - if self.verbosity: - # INFO: -vvv, DEBUG: -vvvvv, for inventory, any more than 3 makes little difference - bargs.append('-{}'.format('v' * min(5, self.verbosity * 2 + 1))) + bargs.extend(['--playbook-dir', self.source_dir]) + if self.verbosity: + # INFO: -vvv, DEBUG: -vvvvv, for inventory, any more than 3 makes little difference + bargs.append('-{}'.format('v' * min(5, self.verbosity * 2 + 1))) logger.debug('Using base command: {}'.format(' '.join(bargs))) return bargs diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 79568ce2a2..5305e6e532 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -1373,10 +1373,6 @@ class PluginFileInjector(object): collection = None collection_migration = '2.9' # Starting with this version, we use collections - def __init__(self, ansible_version): - # This is InventoryOptions instance, could be source or inventory update - self.ansible_version = ansible_version - @classmethod def get_proper_name(cls): if cls.plugin_name is None: diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 4809d2c136..ac1a9dca04 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -73,7 +73,7 @@ from awx.main.utils import (update_scm_url, ignore_inventory_group_removal, extract_ansible_vars, schedule_task_manager, get_awx_version) from awx.main.utils.ansible import read_ansible_config -from awx.main.utils.common import _get_ansible_version, get_custom_venv_choices +from awx.main.utils.common import get_custom_venv_choices from awx.main.utils.external_logging import reconfigure_rsyslog from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja from awx.main.utils.reload import stop_local_services @@ -841,12 +841,6 @@ class BaseTask(object): logger.error('Failed to update %s after %d retries.', self.model._meta.object_name, _attempt) - def get_ansible_version(self, instance): - if not hasattr(self, '_ansible_version'): - self._ansible_version = _get_ansible_version( - ansible_path=self.get_path_to_ansible(instance, executable='ansible')) - return self._ansible_version - def get_path_to(self, *args): ''' Return absolute path relative to this file. @@ -2460,7 +2454,7 @@ class RunInventoryUpdate(BaseTask): If no private data is needed, return None. """ if inventory_update.source in InventorySource.injectors: - injector = InventorySource.injectors[inventory_update.source](self.get_ansible_version(inventory_update)) + injector = InventorySource.injectors[inventory_update.source]() return injector.build_private_data(inventory_update, private_data_dir) def build_env(self, inventory_update, private_data_dir, isolated, private_data_files=None): @@ -2488,7 +2482,7 @@ class RunInventoryUpdate(BaseTask): injector = None if inventory_update.source in InventorySource.injectors: - injector = InventorySource.injectors[inventory_update.source](self.get_ansible_version(inventory_update)) + injector = InventorySource.injectors[inventory_update.source]() if injector is not None: env = injector.build_env(inventory_update, env, private_data_dir, private_data_files) @@ -2601,7 +2595,7 @@ class RunInventoryUpdate(BaseTask): injector = None if inventory_update.source in InventorySource.injectors: - injector = InventorySource.injectors[src](self.get_ansible_version(inventory_update)) + injector = InventorySource.injectors[src]() if injector is not None: content = injector.inventory_contents(inventory_update, private_data_dir) diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index cda720b6ab..5ce64894e6 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -1880,13 +1880,6 @@ class TestProjectUpdateCredentials(TestJobExecution): assert env['FOO'] == 'BAR' -@pytest.fixture -def mock_ansible_version(): - with mock.patch('awx.main.tasks._get_ansible_version', mock.MagicMock(return_value='2.10')) as _fixture: - yield _fixture - - -@pytest.mark.usefixtures("mock_ansible_version") class TestInventoryUpdateCredentials(TestJobExecution): @pytest.fixture def inventory_update(self): diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index a017dba61b..a65120c8e8 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -162,13 +162,14 @@ def memoize_delete(function_name): return cache.delete(function_name) -def _get_ansible_version(ansible_path): +@memoize() +def get_ansible_version(): ''' Return Ansible version installed. Ansible path needs to be provided to account for custom virtual environments ''' try: - proc = subprocess.Popen([ansible_path, '--version'], + proc = subprocess.Popen(['ansible', '--version'], stdout=subprocess.PIPE) result = smart_str(proc.communicate()[0]) return result.split('\n')[0].replace('ansible', '').strip() @@ -176,11 +177,6 @@ def _get_ansible_version(ansible_path): return 'unknown' -@memoize() -def get_ansible_version(): - return _get_ansible_version('ansible') - - def get_awx_version(): ''' Return AWX version as reported by setuptools. From a6712cfd60a6de5f874c043a416707b8bb36b0e1 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Thu, 27 Aug 2020 16:20:06 -0400 Subject: [PATCH 23/27] bump inv plugin migration to avoid conflict --- .../{0118_inventory_plugins.py => 0119_inventory_plugins.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename awx/main/migrations/{0118_inventory_plugins.py => 0119_inventory_plugins.py} (98%) diff --git a/awx/main/migrations/0118_inventory_plugins.py b/awx/main/migrations/0119_inventory_plugins.py similarity index 98% rename from awx/main/migrations/0118_inventory_plugins.py rename to awx/main/migrations/0119_inventory_plugins.py index 991de733d3..670fb7887b 100644 --- a/awx/main/migrations/0118_inventory_plugins.py +++ b/awx/main/migrations/0119_inventory_plugins.py @@ -42,7 +42,7 @@ def inventory_source_vars_forward(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('main', '0117_v400_remove_cloudforms_inventory'), + ('main', '0118_add_remote_archive_scm_type'), ] operations = [ From 043a7f8599dd79f4b7ef10e613deeef91b9fb55f Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Thu, 27 Aug 2020 16:38:00 -0400 Subject: [PATCH 24/27] more get_ansible_version removal --- .../functional/test_inventory_source_injectors.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/awx/main/tests/functional/test_inventory_source_injectors.py b/awx/main/tests/functional/test_inventory_source_injectors.py index 994c2b4c11..4601668d25 100644 --- a/awx/main/tests/functional/test_inventory_source_injectors.py +++ b/awx/main/tests/functional/test_inventory_source_injectors.py @@ -209,7 +209,7 @@ def test_inventory_update_injected_content(this_kind, inventory, fake_credential env, content = read_content(private_data_dir, envvars, inventory_update) # Assert inventory plugin inventory file is in private_data_dir - inventory_filename = InventorySource.injectors[inventory_update.source]('2.9').filename + inventory_filename = InventorySource.injectors[inventory_update.source]().filename assert len([True for k in content.keys() if k.endswith(inventory_filename)]) > 0, \ f"'{inventory_filename}' file not found in inventory update runtime files {content.keys()}" @@ -252,8 +252,7 @@ def test_inventory_update_injected_content(this_kind, inventory, fake_credential with mock.patch('awx.main.queue.CallbackQueueDispatcher.dispatch', lambda self, obj: None): # Also do not send websocket status updates with mock.patch.object(UnifiedJob, 'websocket_emit_status', mock.Mock()): - with mock.patch.object(task, 'get_ansible_version', return_value='2.13'): - # The point of this test is that we replace run with assertions - with mock.patch('awx.main.tasks.ansible_runner.interface.run', substitute_run): - # so this sets up everything for a run and then yields control over to substitute_run - task.run(inventory_update.pk) + # The point of this test is that we replace run with assertions + with mock.patch('awx.main.tasks.ansible_runner.interface.run', substitute_run): + # so this sets up everything for a run and then yields control over to substitute_run + task.run(inventory_update.pk) From 72fc314da1b143b54c829f18ee3368a81c502379 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Thu, 27 Aug 2020 16:57:43 -0400 Subject: [PATCH 25/27] flake8 fix --- awx/main/management/commands/inventory_import.py | 1 - 1 file changed, 1 deletion(-) diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index f0aa2feff2..53a1660c4f 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -12,7 +12,6 @@ import sys import time import traceback import shutil -from distutils.version import LooseVersion as Version # Django from django.conf import settings From 924273f5892768270b8c7f38e6674f72641b5eff Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Thu, 27 Aug 2020 19:05:40 -0400 Subject: [PATCH 26/27] more get_ansible_version test failure fixes --- awx/main/tests/functional/models/test_inventory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/awx/main/tests/functional/models/test_inventory.py b/awx/main/tests/functional/models/test_inventory.py index 2b3c747868..04b92d5a1d 100644 --- a/awx/main/tests/functional/models/test_inventory.py +++ b/awx/main/tests/functional/models/test_inventory.py @@ -255,7 +255,7 @@ class TestInventorySourceInjectors: are named correctly, because Ansible will reject files that do not have these exact names """ - injector = InventorySource.injectors[source]('2.7.7') + injector = InventorySource.injectors[source]() assert injector.filename == filename @pytest.mark.parametrize('source,proper_name', [ @@ -269,7 +269,7 @@ class TestInventorySourceInjectors: ('tower', 'awx.awx.tower'), ]) def test_plugin_proper_names(self, source, proper_name): - injector = InventorySource.injectors[source]('2.9') + injector = InventorySource.injectors[source]() assert injector.get_proper_name() == proper_name From 059999c7c3076438880ed9302a595a661dbd2733 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Tue, 1 Sep 2020 13:15:07 -0400 Subject: [PATCH 27/27] update the inventory source module to respect new fields --- .../plugins/modules/tower_inventory_source.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/awx_collection/plugins/modules/tower_inventory_source.py b/awx_collection/plugins/modules/tower_inventory_source.py index d3517b3f5a..4755670291 100644 --- a/awx_collection/plugins/modules/tower_inventory_source.py +++ b/awx_collection/plugins/modules/tower_inventory_source.py @@ -57,6 +57,18 @@ options: description: - The variables or environment fields to apply to this source type. type: dict + enabled_var: + description: + - The variable to use to determine enabled state e.g., "status.power_state" + type: str + enabled_value: + description: + - Value when the host is considered enabled, e.g., "powered_on" + type: str + host_filter: + description: + - If specified, AWX will only import hosts that match this regular expression. + type: str credential: description: - Credential to use for the source. @@ -152,6 +164,9 @@ def main(): source_path=dict(), source_script=dict(), source_vars=dict(type='dict'), + enabled_var=dict(), + enabled_value=dict(), + host_filter=dict(), credential=dict(), overwrite=dict(type='bool'), overwrite_vars=dict(type='bool'), @@ -232,7 +247,7 @@ def main(): 'description', 'source', 'source_path', 'source_vars', 'overwrite', 'overwrite_vars', 'custom_virtualenv', 'timeout', 'verbosity', 'update_on_launch', 'update_cache_timeout', - 'update_on_project_update' + 'update_on_project_update', 'enabled_var', 'enabled_value', 'host_filter', ) # Layer in all remaining optional information