diff --git a/awx/api/metadata.py b/awx/api/metadata.py index 8bbfb906ef..847e353890 100644 --- a/awx/api/metadata.py +++ b/awx/api/metadata.py @@ -23,7 +23,7 @@ from rest_framework.request import clone_request # AWX from awx.api.fields import ChoiceNullField from awx.main.fields import JSONField, ImplicitRoleField -from awx.main.models import InventorySource, NotificationTemplate +from awx.main.models import NotificationTemplate from awx.main.scheduler.kubernetes import PodManager @@ -115,19 +115,6 @@ class Metadata(metadata.SimpleMetadata): if getattr(field, 'write_only', False): field_info['write_only'] = True - # Special handling of inventory source_region choices that vary based on - # selected inventory source. - if field.field_name == 'source_regions': - for cp in ('azure_rm', 'ec2', 'gce'): - get_regions = getattr(InventorySource, 'get_%s_region_choices' % cp) - field_info['%s_region_choices' % cp] = get_regions() - - # Special handling of group_by choices for EC2. - if field.field_name == 'group_by': - for cp in ('ec2',): - get_group_by_choices = getattr(InventorySource, 'get_%s_group_by_choices' % cp) - field_info['%s_group_by_choices' % cp] = get_group_by_choices() - # Special handling of notification configuration where the required properties # are conditional on the type selected. if field.field_name == 'notification_configuration': diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 6ce32b3f51..83575025e7 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1937,7 +1937,7 @@ class InventorySourceOptionsSerializer(BaseSerializer): class Meta: fields = ('*', 'source', 'source_path', 'source_script', 'source_vars', 'credential', - 'source_regions', 'instance_filters', 'group_by', 'overwrite', 'overwrite_vars', + 'enabled_var', 'enabled_value', 'host_filter', 'overwrite', 'overwrite_vars', 'custom_virtualenv', 'timeout', 'verbosity') def get_related(self, obj): @@ -1957,7 +1957,7 @@ class InventorySourceOptionsSerializer(BaseSerializer): return ret def validate(self, attrs): - # TODO: Validate source, validate source_regions + # TODO: Validate source errors = {} source = attrs.get('source', self.instance and self.instance.source or '') diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index 2e51283445..53a1660c4f 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -12,7 +12,6 @@ import sys import time import traceback import shutil -from distutils.version import LooseVersion as Version # Django from django.conf import settings @@ -39,7 +38,6 @@ from awx.main.utils import ( build_proot_temp_dir, get_licenser ) -from awx.main.utils.common import _get_ansible_version from awx.main.signals import disable_activity_stream from awx.main.constants import STANDARD_INVENTORY_UPDATE_ENV from awx.main.utils.pglock import advisory_lock @@ -136,15 +134,10 @@ class AnsibleInventoryLoader(object): # inside of /venv/ansible, so we override the specified interpreter # https://github.com/ansible/ansible/issues/50714 bargs = ['python', ansible_inventory_path, '-i', self.source] - ansible_version = _get_ansible_version(ansible_inventory_path[:-len('-inventory')]) - if ansible_version != 'unknown': - this_version = Version(ansible_version) - if this_version >= Version('2.5'): - bargs.extend(['--playbook-dir', self.source_dir]) - if this_version >= Version('2.8'): - if self.verbosity: - # INFO: -vvv, DEBUG: -vvvvv, for inventory, any more than 3 makes little difference - bargs.append('-{}'.format('v' * min(5, self.verbosity * 2 + 1))) + bargs.extend(['--playbook-dir', self.source_dir]) + if self.verbosity: + # INFO: -vvv, DEBUG: -vvvvv, for inventory, any more than 3 makes little difference + bargs.append('-{}'.format('v' * min(5, self.verbosity * 2 + 1))) logger.debug('Using base command: {}'.format(' '.join(bargs))) return bargs diff --git a/awx/main/migrations/0119_inventory_plugins.py b/awx/main/migrations/0119_inventory_plugins.py new file mode 100644 index 0000000000..670fb7887b --- /dev/null +++ b/awx/main/migrations/0119_inventory_plugins.py @@ -0,0 +1,104 @@ +# Generated by Django 2.2.11 on 2020-07-20 19:56 + +import logging +import yaml + +from django.db import migrations, models + +from awx.main.models.base import VarsDictProperty + +from ._inventory_source_vars import FrozenInjectors + + +logger = logging.getLogger('awx.main.migrations') + + +def _get_inventory_sources(InventorySource): + return InventorySource.objects.filter(source__in=['ec2', 'gce', 'azure_rm', 'vmware', 'satellite6', 'openstack', 'rhv', 'tower']) + + +def inventory_source_vars_forward(apps, schema_editor): + InventorySource = apps.get_model("main", "InventorySource") + ''' + The Django app registry does not keep track of model inheritance. The + source_vars_dict property comes from InventorySourceOptions via inheritance. + This adds that property. Luckily, other properteries and functionality from + InventorySourceOptions is not needed by the injector logic. + ''' + setattr(InventorySource, 'source_vars_dict', VarsDictProperty('source_vars')) + source_vars_backup = dict() + + for inv_source_obj in _get_inventory_sources(InventorySource): + + if inv_source_obj.source in FrozenInjectors: + source_vars_backup[inv_source_obj.id] = dict(inv_source_obj.source_vars_dict) + + injector = FrozenInjectors[inv_source_obj.source]() + new_inv_source_vars = injector.inventory_as_dict(inv_source_obj, None) + inv_source_obj.source_vars = yaml.dump(new_inv_source_vars) + inv_source_obj.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0118_add_remote_archive_scm_type'), + ] + + operations = [ + migrations.RunPython(inventory_source_vars_forward), + migrations.RemoveField( + model_name='inventorysource', + name='group_by', + ), + migrations.RemoveField( + model_name='inventoryupdate', + name='group_by', + ), + migrations.RemoveField( + model_name='inventorysource', + name='instance_filters', + ), + migrations.RemoveField( + model_name='inventoryupdate', + name='instance_filters', + ), + migrations.RemoveField( + model_name='inventorysource', + name='source_regions', + ), + migrations.RemoveField( + model_name='inventoryupdate', + name='source_regions', + ), + migrations.AddField( + model_name='inventorysource', + name='enabled_value', + field=models.TextField(blank=True, default='', help_text='Only used when enabled_var is set. Value when the host is considered enabled. For example if enabled_var="status.power_state"and enabled_value="powered_on" with host variables:{ "status": { "power_state": "powered_on", "created": "2020-08-04T18:13:04+00:00", "healthy": true }, "name": "foobar", "ip_address": "192.168.2.1"}The host would be marked enabled. If power_state where any value other than powered_on then the host would be disabled when imported into Tower. If the key is not found then the host will be enabled'), + ), + migrations.AddField( + model_name='inventorysource', + name='enabled_var', + field=models.TextField(blank=True, default='', help_text='Retrieve the enabled state from the given dict of host variables. The enabled variable may be specified as "foo.bar", in which case the lookup will traverse into nested dicts, equivalent to: from_dict.get("foo", {}).get("bar", default)'), + ), + migrations.AddField( + model_name='inventorysource', + name='host_filter', + field=models.TextField(blank=True, default='', help_text='Regex where only matching hosts will be imported into Tower.'), + ), + migrations.AddField( + model_name='inventoryupdate', + name='enabled_value', + field=models.TextField(blank=True, default='', help_text='Only used when enabled_var is set. Value when the host is considered enabled. For example if enabled_var="status.power_state"and enabled_value="powered_on" with host variables:{ "status": { "power_state": "powered_on", "created": "2020-08-04T18:13:04+00:00", "healthy": true }, "name": "foobar", "ip_address": "192.168.2.1"}The host would be marked enabled. If power_state where any value other than powered_on then the host would be disabled when imported into Tower. If the key is not found then the host will be enabled'), + ), + migrations.AddField( + model_name='inventoryupdate', + name='enabled_var', + field=models.TextField(blank=True, default='', help_text='Retrieve the enabled state from the given dict of host variables. The enabled variable may be specified as "foo.bar", in which case the lookup will traverse into nested dicts, equivalent to: from_dict.get("foo", {}).get("bar", default)'), + ), + migrations.AddField( + model_name='inventoryupdate', + name='host_filter', + field=models.TextField(blank=True, default='', help_text='Regex where only matching hosts will be imported into Tower.'), + ), + ] diff --git a/awx/main/migrations/_inventory_source_vars.py b/awx/main/migrations/_inventory_source_vars.py new file mode 100644 index 0000000000..edd2e6827a --- /dev/null +++ b/awx/main/migrations/_inventory_source_vars.py @@ -0,0 +1,751 @@ +import json + +from django.utils.translation import ugettext_lazy as _ + + +FrozenInjectors = dict() + + +class PluginFileInjector(object): + plugin_name = None # Ansible core name used to reference plugin + # every source should have collection, these are for the collection name + namespace = None + collection = None + + def inventory_as_dict(self, inventory_source, private_data_dir): + """Default implementation of inventory plugin file contents. + There are some valid cases when all parameters can be obtained from + the environment variables, example "plugin: linode" is valid + ideally, however, some options should be filled from the inventory source data + """ + if self.plugin_name is None: + raise NotImplementedError('At minimum the plugin name is needed for inventory plugin use.') + proper_name = f'{self.namespace}.{self.collection}.{self.plugin_name}' + return {'plugin': proper_name} + + +class azure_rm(PluginFileInjector): + plugin_name = 'azure_rm' + namespace = 'azure' + collection = 'azcollection' + + def inventory_as_dict(self, inventory_source, private_data_dir): + ret = super(azure_rm, self).inventory_as_dict(inventory_source, private_data_dir) + + source_vars = inventory_source.source_vars_dict + + ret['fail_on_template_errors'] = False + + group_by_hostvar = { + 'location': {'prefix': '', 'separator': '', 'key': 'location'}, + 'tag': {'prefix': '', 'separator': '', 'key': 'tags.keys() | list if tags else []'}, + # Introduced with https://github.com/ansible/ansible/pull/53046 + 'security_group': {'prefix': '', 'separator': '', 'key': 'security_group'}, + 'resource_group': {'prefix': '', 'separator': '', 'key': 'resource_group'}, + # Note, os_family was not documented correctly in script, but defaulted to grouping by it + 'os_family': {'prefix': '', 'separator': '', 'key': 'os_disk.operating_system_type'} + } + # by default group by everything + # always respect user setting, if they gave it + group_by = [ + grouping_name for grouping_name in group_by_hostvar + if source_vars.get('group_by_{}'.format(grouping_name), True) + ] + ret['keyed_groups'] = [group_by_hostvar[grouping_name] for grouping_name in group_by] + if 'tag' in group_by: + # Nasty syntax to reproduce "key_value" group names in addition to "key" + ret['keyed_groups'].append({ + 'prefix': '', 'separator': '', + 'key': r'dict(tags.keys() | map("regex_replace", "^(.*)$", "\1_") | list | zip(tags.values() | list)) if tags else []' + }) + + # Compatibility content + # TODO: add proper support for instance_filters non-specific to compatibility + # TODO: add proper support for group_by non-specific to compatibility + # Dashes were not configurable in azure_rm.py script, we do not want unicode, so always use this + ret['use_contrib_script_compatible_sanitization'] = True + # use same host names as script + ret['plain_host_names'] = True + # By default the script did not filter hosts + ret['default_host_filters'] = [] + # User-given host filters + user_filters = [] + old_filterables = [ + ('resource_groups', 'resource_group'), + ('tags', 'tags') + # locations / location would be an entry + # but this would conflict with source_regions + ] + for key, loc in old_filterables: + value = source_vars.get(key, None) + if value and isinstance(value, str): + # tags can be list of key:value pairs + # e.g. 'Creator:jmarshall, peanutbutter:jelly' + # or tags can be a list of keys + # e.g. 'Creator, peanutbutter' + if key == "tags": + # grab each key value pair + for kvpair in value.split(','): + # split into key and value + kv = kvpair.split(':') + # filter out any host that does not have key + # in their tags.keys() variable + user_filters.append('"{}" not in tags.keys()'.format(kv[0].strip())) + # if a value is provided, check that the key:value pair matches + if len(kv) > 1: + user_filters.append('tags["{}"] != "{}"'.format(kv[0].strip(), kv[1].strip())) + else: + user_filters.append('{} not in {}'.format( + loc, value.split(',') + )) + if user_filters: + ret.setdefault('exclude_host_filters', []) + ret['exclude_host_filters'].extend(user_filters) + + ret['conditional_groups'] = {'azure': True} + ret['hostvar_expressions'] = { + 'provisioning_state': 'provisioning_state | title', + 'computer_name': 'name', + 'type': 'resource_type', + 'private_ip': 'private_ipv4_addresses[0] if private_ipv4_addresses else None', + 'public_ip': 'public_ipv4_addresses[0] if public_ipv4_addresses else None', + 'public_ip_name': 'public_ip_name if public_ip_name is defined else None', + 'public_ip_id': 'public_ip_id if public_ip_id is defined else None', + 'tags': 'tags if tags else None' + } + # Special functionality from script + if source_vars.get('use_private_ip', False): + ret['hostvar_expressions']['ansible_host'] = 'private_ipv4_addresses[0]' + # end compatibility content + + if inventory_source.source_regions and 'all' not in inventory_source.source_regions: + # initialize a list for this section in inventory file + ret.setdefault('exclude_host_filters', []) + # make a python list of the regions we will use + python_regions = [x.strip() for x in inventory_source.source_regions.split(',')] + # convert that list in memory to python syntax in a string + # now put that in jinja2 syntax operating on hostvar key "location" + # and put that as an entry in the exclusions list + ret['exclude_host_filters'].append("location not in {}".format(repr(python_regions))) + return ret + +class ec2(PluginFileInjector): + plugin_name = 'aws_ec2' + namespace = 'amazon' + collection = 'aws' + + + def _get_ec2_group_by_choices(self): + return [ + ('ami_id', _('Image ID')), + ('availability_zone', _('Availability Zone')), + ('aws_account', _('Account')), + ('instance_id', _('Instance ID')), + ('instance_state', _('Instance State')), + ('platform', _('Platform')), + ('instance_type', _('Instance Type')), + ('key_pair', _('Key Name')), + ('region', _('Region')), + ('security_group', _('Security Group')), + ('tag_keys', _('Tags')), + ('tag_none', _('Tag None')), + ('vpc_id', _('VPC ID')), + ] + + def _compat_compose_vars(self): + return { + # vars that change + 'ec2_block_devices': ( + "dict(block_device_mappings | map(attribute='device_name') | list | zip(block_device_mappings " + "| map(attribute='ebs.volume_id') | list))" + ), + 'ec2_dns_name': 'public_dns_name', + 'ec2_group_name': 'placement.group_name', + 'ec2_instance_profile': 'iam_instance_profile | default("")', + 'ec2_ip_address': 'public_ip_address', + 'ec2_kernel': 'kernel_id | default("")', + 'ec2_monitored': "monitoring.state in ['enabled', 'pending']", + 'ec2_monitoring_state': 'monitoring.state', + 'ec2_placement': 'placement.availability_zone', + 'ec2_ramdisk': 'ramdisk_id | default("")', + 'ec2_reason': 'state_transition_reason', + 'ec2_security_group_ids': "security_groups | map(attribute='group_id') | list | join(',')", + 'ec2_security_group_names': "security_groups | map(attribute='group_name') | list | join(',')", + 'ec2_tag_Name': 'tags.Name', + 'ec2_state': 'state.name', + 'ec2_state_code': 'state.code', + 'ec2_state_reason': 'state_reason.message if state_reason is defined else ""', + 'ec2_sourceDestCheck': 'source_dest_check | default(false) | lower | string', # snake_case syntax intended + 'ec2_account_id': 'owner_id', + # vars that just need ec2_ prefix + 'ec2_ami_launch_index': 'ami_launch_index | string', + 'ec2_architecture': 'architecture', + 'ec2_client_token': 'client_token', + 'ec2_ebs_optimized': 'ebs_optimized', + 'ec2_hypervisor': 'hypervisor', + 'ec2_image_id': 'image_id', + 'ec2_instance_type': 'instance_type', + 'ec2_key_name': 'key_name', + 'ec2_launch_time': r'launch_time | regex_replace(" ", "T") | regex_replace("(\+)(\d\d):(\d)(\d)$", ".\g<2>\g<3>Z")', + 'ec2_platform': 'platform | default("")', + 'ec2_private_dns_name': 'private_dns_name', + 'ec2_private_ip_address': 'private_ip_address', + 'ec2_public_dns_name': 'public_dns_name', + 'ec2_region': 'placement.region', + 'ec2_root_device_name': 'root_device_name', + 'ec2_root_device_type': 'root_device_type', + # many items need blank defaults because the script tended to keep a common schema + 'ec2_spot_instance_request_id': 'spot_instance_request_id | default("")', + 'ec2_subnet_id': 'subnet_id | default("")', + 'ec2_virtualization_type': 'virtualization_type', + 'ec2_vpc_id': 'vpc_id | default("")', + # same as ec2_ip_address, the script provided this + 'ansible_host': 'public_ip_address', + # new with https://github.com/ansible/ansible/pull/53645 + 'ec2_eventsSet': 'events | default("")', + 'ec2_persistent': 'persistent | default(false)', + 'ec2_requester_id': 'requester_id | default("")' + } + + def inventory_as_dict(self, inventory_source, private_data_dir): + ret = super(ec2, self).inventory_as_dict(inventory_source, private_data_dir) + + keyed_groups = [] + group_by_hostvar = { + 'ami_id': {'prefix': '', 'separator': '', 'key': 'image_id', 'parent_group': 'images'}, + # 2 entries for zones for same groups to establish 2 parentage trees + 'availability_zone': {'prefix': '', 'separator': '', 'key': 'placement.availability_zone', 'parent_group': 'zones'}, + 'aws_account': {'prefix': '', 'separator': '', 'key': 'ec2_account_id', 'parent_group': 'accounts'}, # composed var + 'instance_id': {'prefix': '', 'separator': '', 'key': 'instance_id', 'parent_group': 'instances'}, # normally turned off + 'instance_state': {'prefix': 'instance_state', 'key': 'ec2_state', 'parent_group': 'instance_states'}, # composed var + # ec2_platform is a composed var, but group names do not match up to hostvar exactly + 'platform': {'prefix': 'platform', 'key': 'platform | default("undefined")', 'parent_group': 'platforms'}, + 'instance_type': {'prefix': 'type', 'key': 'instance_type', 'parent_group': 'types'}, + 'key_pair': {'prefix': 'key', 'key': 'key_name', 'parent_group': 'keys'}, + 'region': {'prefix': '', 'separator': '', 'key': 'placement.region', 'parent_group': 'regions'}, + # Security requires some ninja jinja2 syntax, credit to s-hertel + 'security_group': {'prefix': 'security_group', 'key': 'security_groups | map(attribute="group_name")', 'parent_group': 'security_groups'}, + # tags cannot be parented in exactly the same way as the script due to + # https://github.com/ansible/ansible/pull/53812 + 'tag_keys': [ + {'prefix': 'tag', 'key': 'tags', 'parent_group': 'tags'}, + {'prefix': 'tag', 'key': 'tags.keys()', 'parent_group': 'tags'} + ], + # 'tag_none': None, # grouping by no tags isn't a different thing with plugin + # naming is redundant, like vpc_id_vpc_8c412cea, but intended + 'vpc_id': {'prefix': 'vpc_id', 'key': 'vpc_id', 'parent_group': 'vpcs'}, + } + # -- same-ish as script here -- + group_by = [x.strip().lower() for x in inventory_source.group_by.split(',') if x.strip()] + for choice in self._get_ec2_group_by_choices(): + value = bool((group_by and choice[0] in group_by) or (not group_by and choice[0] != 'instance_id')) + # -- end sameness to script -- + if value: + this_keyed_group = group_by_hostvar.get(choice[0], None) + # If a keyed group syntax does not exist, there is nothing we can do to get this group + if this_keyed_group is not None: + if isinstance(this_keyed_group, list): + keyed_groups.extend(this_keyed_group) + else: + keyed_groups.append(this_keyed_group) + # special case, this parentage is only added if both zones and regions are present + if not group_by or ('region' in group_by and 'availability_zone' in group_by): + keyed_groups.append({'prefix': '', 'separator': '', 'key': 'placement.availability_zone', 'parent_group': '{{ placement.region }}'}) + + source_vars = inventory_source.source_vars_dict + # This is a setting from the script, hopefully no one used it + # if true, it replaces dashes, but not in region / loc names + replace_dash = bool(source_vars.get('replace_dash_in_groups', True)) + # Compatibility content + legacy_regex = { + True: r"[^A-Za-z0-9\_]", + False: r"[^A-Za-z0-9\_\-]" # do not replace dash, dash is allowed + }[replace_dash] + list_replacer = 'map("regex_replace", "{rx}", "_") | list'.format(rx=legacy_regex) + # this option, a plugin option, will allow dashes, but not unicode + # when set to False, unicode will be allowed, but it was not allowed by script + # thus, we always have to use this option, and always use our custom regex + ret['use_contrib_script_compatible_sanitization'] = True + for grouping_data in keyed_groups: + if grouping_data['key'] in ('placement.region', 'placement.availability_zone'): + # us-east-2 is always us-east-2 according to ec2.py + # no sanitization in region-ish groups for the script standards, ever ever + continue + if grouping_data['key'] == 'tags': + # dict jinja2 transformation + grouping_data['key'] = 'dict(tags.keys() | {replacer} | zip(tags.values() | {replacer}))'.format( + replacer=list_replacer + ) + elif grouping_data['key'] == 'tags.keys()' or grouping_data['prefix'] == 'security_group': + # list jinja2 transformation + grouping_data['key'] += ' | {replacer}'.format(replacer=list_replacer) + else: + # string transformation + grouping_data['key'] += ' | regex_replace("{rx}", "_")'.format(rx=legacy_regex) + # end compatibility content + + if source_vars.get('iam_role_arn', None): + ret['iam_role_arn'] = source_vars['iam_role_arn'] + + # This was an allowed ec2.ini option, also plugin option, so pass through + if source_vars.get('boto_profile', None): + ret['boto_profile'] = source_vars['boto_profile'] + + elif not replace_dash: + # Using the plugin, but still want dashes allowed + ret['use_contrib_script_compatible_sanitization'] = True + + if source_vars.get('nested_groups') is False: + for this_keyed_group in keyed_groups: + this_keyed_group.pop('parent_group', None) + + if keyed_groups: + ret['keyed_groups'] = keyed_groups + + # Instance ID not part of compat vars, because of settings.EC2_INSTANCE_ID_VAR + compose_dict = {'ec2_id': 'instance_id'} + inst_filters = {} + + # Compatibility content + compose_dict.update(self._compat_compose_vars()) + # plugin provides "aws_ec2", but not this which the script gave + ret['groups'] = {'ec2': True} + if source_vars.get('hostname_variable') is not None: + hnames = [] + for expr in source_vars.get('hostname_variable').split(','): + if expr == 'public_dns_name': + hnames.append('dns-name') + elif not expr.startswith('tag:') and '_' in expr: + hnames.append(expr.replace('_', '-')) + else: + hnames.append(expr) + ret['hostnames'] = hnames + else: + # public_ip as hostname is non-default plugin behavior, script behavior + ret['hostnames'] = [ + 'network-interface.addresses.association.public-ip', + 'dns-name', + 'private-dns-name' + ] + # The script returned only running state by default, the plugin does not + # https://docs.aws.amazon.com/cli/latest/reference/ec2/describe-instances.html#options + # options: pending | running | shutting-down | terminated | stopping | stopped + inst_filters['instance-state-name'] = ['running'] + # end compatibility content + + if source_vars.get('destination_variable') or source_vars.get('vpc_destination_variable'): + for fd in ('destination_variable', 'vpc_destination_variable'): + if source_vars.get(fd): + compose_dict['ansible_host'] = source_vars.get(fd) + break + + if compose_dict: + ret['compose'] = compose_dict + + if inventory_source.instance_filters: + # logic used to live in ec2.py, now it belongs to us. Yay more code? + filter_sets = [f for f in inventory_source.instance_filters.split(',') if f] + + for instance_filter in filter_sets: + # AND logic not supported, unclear how to... + instance_filter = instance_filter.strip() + if not instance_filter or '=' not in instance_filter: + continue + filter_key, filter_value = [x.strip() for x in instance_filter.split('=', 1)] + if not filter_key: + continue + inst_filters[filter_key] = filter_value + + if inst_filters: + ret['filters'] = inst_filters + + if inventory_source.source_regions and 'all' not in inventory_source.source_regions: + ret['regions'] = inventory_source.source_regions.split(',') + + return ret + + +class gce(PluginFileInjector): + plugin_name = 'gcp_compute' + namespace = 'google' + collection = 'cloud' + + def _compat_compose_vars(self): + # missing: gce_image, gce_uuid + # https://github.com/ansible/ansible/issues/51884 + return { + 'gce_description': 'description if description else None', + 'gce_machine_type': 'machineType', + 'gce_name': 'name', + 'gce_network': 'networkInterfaces[0].network.name', + 'gce_private_ip': 'networkInterfaces[0].networkIP', + 'gce_public_ip': 'networkInterfaces[0].accessConfigs[0].natIP | default(None)', + 'gce_status': 'status', + 'gce_subnetwork': 'networkInterfaces[0].subnetwork.name', + 'gce_tags': 'tags.get("items", [])', + 'gce_zone': 'zone', + 'gce_metadata': 'metadata.get("items", []) | items2dict(key_name="key", value_name="value")', + # NOTE: image hostvar is enabled via retrieve_image_info option + 'gce_image': 'image', + # We need this as long as hostnames is non-default, otherwise hosts + # will not be addressed correctly, was returned in script + 'ansible_ssh_host': 'networkInterfaces[0].accessConfigs[0].natIP | default(networkInterfaces[0].networkIP)' + } + + def inventory_as_dict(self, inventory_source, private_data_dir): + ret = super(gce, self).inventory_as_dict(inventory_source, private_data_dir) + + # auth related items + ret['auth_kind'] = "serviceaccount" + + filters = [] + # TODO: implement gce group_by options + # gce never processed the group_by field, if it had, we would selectively + # apply those options here, but it did not, so all groups are added here + keyed_groups = [ + # the jinja2 syntax is duplicated with compose + # https://github.com/ansible/ansible/issues/51883 + {'prefix': 'network', 'key': 'gce_subnetwork'}, # composed var + {'prefix': '', 'separator': '', 'key': 'gce_private_ip'}, # composed var + {'prefix': '', 'separator': '', 'key': 'gce_public_ip'}, # composed var + {'prefix': '', 'separator': '', 'key': 'machineType'}, + {'prefix': '', 'separator': '', 'key': 'zone'}, + {'prefix': 'tag', 'key': 'gce_tags'}, # composed var + {'prefix': 'status', 'key': 'status | lower'}, + # NOTE: image hostvar is enabled via retrieve_image_info option + {'prefix': '', 'separator': '', 'key': 'image'}, + ] + # This will be used as the gce instance_id, must be universal, non-compat + compose_dict = {'gce_id': 'id'} + + # Compatibility content + # TODO: proper group_by and instance_filters support, irrelevant of compat mode + # The gce.py script never sanitized any names in any way + ret['use_contrib_script_compatible_sanitization'] = True + # Perform extra API query to get the image hostvar + ret['retrieve_image_info'] = True + # Add in old hostvars aliases + compose_dict.update(self._compat_compose_vars()) + # Non-default names to match script + ret['hostnames'] = ['name', 'public_ip', 'private_ip'] + # end compatibility content + + if keyed_groups: + ret['keyed_groups'] = keyed_groups + if filters: + ret['filters'] = filters + if compose_dict: + ret['compose'] = compose_dict + if inventory_source.source_regions and 'all' not in inventory_source.source_regions: + ret['zones'] = inventory_source.source_regions.split(',') + return ret + + +class vmware(PluginFileInjector): + plugin_name = 'vmware_vm_inventory' + namespace = 'community' + collection = 'vmware' + + def inventory_as_dict(self, inventory_source, private_data_dir): + ret = super(vmware, self).inventory_as_dict(inventory_source, private_data_dir) + ret['strict'] = False + # Documentation of props, see + # https://github.com/ansible/ansible/blob/devel/docs/docsite/rst/scenario_guides/vmware_scenarios/vmware_inventory_vm_attributes.rst + UPPERCASE_PROPS = [ + "availableField", + "configIssue", + "configStatus", + "customValue", # optional + "datastore", + "effectiveRole", + "guestHeartbeatStatus", # optional + "layout", # optional + "layoutEx", # optional + "name", + "network", + "overallStatus", + "parentVApp", # optional + "permission", + "recentTask", + "resourcePool", + "rootSnapshot", + "snapshot", # optional + "triggeredAlarmState", + "value" + ] + NESTED_PROPS = [ + "capability", + "config", + "guest", + "runtime", + "storage", + "summary", # repeat of other properties + ] + ret['properties'] = UPPERCASE_PROPS + NESTED_PROPS + ret['compose'] = {'ansible_host': 'guest.ipAddress'} # default value + ret['compose']['ansible_ssh_host'] = ret['compose']['ansible_host'] + # the ansible_uuid was unique every host, every import, from the script + ret['compose']['ansible_uuid'] = '99999999 | random | to_uuid' + for prop in UPPERCASE_PROPS: + if prop == prop.lower(): + continue + ret['compose'][prop.lower()] = prop + ret['with_nested_properties'] = True + # ret['property_name_format'] = 'lower_case' # only dacrystal/topic/vmware-inventory-plugin-property-format + + # process custom options + vmware_opts = dict(inventory_source.source_vars_dict.items()) + if inventory_source.instance_filters: + vmware_opts.setdefault('host_filters', inventory_source.instance_filters) + if inventory_source.group_by: + vmware_opts.setdefault('groupby_patterns', inventory_source.group_by) + + alias_pattern = vmware_opts.get('alias_pattern') + if alias_pattern: + ret.setdefault('hostnames', []) + for alias in alias_pattern.split(','): # make best effort + striped_alias = alias.replace('{', '').replace('}', '').strip() # make best effort + if not striped_alias: + continue + ret['hostnames'].append(striped_alias) + + host_pattern = vmware_opts.get('host_pattern') # not working in script + if host_pattern: + stripped_hp = host_pattern.replace('{', '').replace('}', '').strip() # make best effort + ret['compose']['ansible_host'] = stripped_hp + ret['compose']['ansible_ssh_host'] = stripped_hp + + host_filters = vmware_opts.get('host_filters') + if host_filters: + ret.setdefault('filters', []) + for hf in host_filters.split(','): + striped_hf = hf.replace('{', '').replace('}', '').strip() # make best effort + if not striped_hf: + continue + ret['filters'].append(striped_hf) + else: + # default behavior filters by power state + ret['filters'] = ['runtime.powerState == "poweredOn"'] + + groupby_patterns = vmware_opts.get('groupby_patterns') + ret.setdefault('keyed_groups', []) + if groupby_patterns: + for pattern in groupby_patterns.split(','): + stripped_pattern = pattern.replace('{', '').replace('}', '').strip() # make best effort + ret['keyed_groups'].append({ + 'prefix': '', 'separator': '', + 'key': stripped_pattern + }) + else: + # default groups from script + for entry in ('config.guestId', '"templates" if config.template else "guests"'): + ret['keyed_groups'].append({ + 'prefix': '', 'separator': '', + 'key': entry + }) + + return ret + + +class openstack(PluginFileInjector): + plugin_name = 'openstack' + namespace = 'openstack' + collection = 'cloud' + + def inventory_as_dict(self, inventory_source, private_data_dir): + def use_host_name_for_name(a_bool_maybe): + if not isinstance(a_bool_maybe, bool): + # Could be specified by user via "host" or "uuid" + return a_bool_maybe + elif a_bool_maybe: + return 'name' # plugin default + else: + return 'uuid' + + ret = super(openstack, self).inventory_as_dict(inventory_source, private_data_dir) + ret['fail_on_errors'] = True + ret['expand_hostvars'] = True + ret['inventory_hostname'] = use_host_name_for_name(False) + # Note: mucking with defaults will break import integrity + # For the plugin, we need to use the same defaults as the old script + # or else imports will conflict. To find script defaults you have + # to read source code of the script. + # + # Script Defaults Plugin Defaults + # 'use_hostnames': False, 'name' (True) + # 'expand_hostvars': True, 'no' (False) + # 'fail_on_errors': True, 'no' (False) + # + # These are, yet again, different from ansible_variables in script logic + # but those are applied inconsistently + source_vars = inventory_source.source_vars_dict + for var_name in ['expand_hostvars', 'fail_on_errors']: + if var_name in source_vars: + ret[var_name] = source_vars[var_name] + if 'use_hostnames' in source_vars: + ret['inventory_hostname'] = use_host_name_for_name(source_vars['use_hostnames']) + return ret + +class rhv(PluginFileInjector): + """ovirt uses the custom credential templating, and that is all + """ + plugin_name = 'ovirt' + initial_version = '2.9' + namespace = 'ovirt' + collection = 'ovirt' + + def inventory_as_dict(self, inventory_source, private_data_dir): + ret = super(rhv, self).inventory_as_dict(inventory_source, private_data_dir) + ret['ovirt_insecure'] = False # Default changed from script + # TODO: process strict option upstream + ret['compose'] = { + 'ansible_host': '(devices.values() | list)[0][0] if devices else None' + } + ret['keyed_groups'] = [] + for key in ('cluster', 'status'): + ret['keyed_groups'].append({'prefix': key, 'separator': '_', 'key': key}) + ret['keyed_groups'].append({'prefix': 'tag', 'separator': '_', 'key': 'tags'}) + ret['ovirt_hostname_preference'] = ['name', 'fqdn'] + source_vars = inventory_source.source_vars_dict + for key, value in source_vars.items(): + if key == 'plugin': + continue + ret[key] = value + return ret + + +class satellite6(PluginFileInjector): + plugin_name = 'foreman' + namespace = 'theforeman' + collection = 'foreman' + + def inventory_as_dict(self, inventory_source, private_data_dir): + ret = super(satellite6, self).inventory_as_dict(inventory_source, private_data_dir) + ret['validate_certs'] = False + + group_patterns = '[]' + group_prefix = 'foreman_' + want_hostcollections = False + want_ansible_ssh_host = False + want_facts = True + + foreman_opts = inventory_source.source_vars_dict.copy() + for k, v in foreman_opts.items(): + if k == 'satellite6_group_patterns' and isinstance(v, str): + group_patterns = v + elif k == 'satellite6_group_prefix' and isinstance(v, str): + group_prefix = v + elif k == 'satellite6_want_hostcollections' and isinstance(v, bool): + want_hostcollections = v + elif k == 'satellite6_want_ansible_ssh_host' and isinstance(v, bool): + want_ansible_ssh_host = v + elif k == 'satellite6_want_facts' and isinstance(v, bool): + want_facts = v + # add backwards support for ssl_verify + # plugin uses new option, validate_certs, instead + elif k == 'ssl_verify' and isinstance(v, bool): + ret['validate_certs'] = v + else: + ret[k] = str(v) + + # Compatibility content + group_by_hostvar = { + "environment": {"prefix": "{}environment_".format(group_prefix), + "separator": "", + "key": "foreman['environment_name'] | lower | regex_replace(' ', '') | " + "regex_replace('[^A-Za-z0-9_]', '_') | regex_replace('none', '')"}, + "location": {"prefix": "{}location_".format(group_prefix), + "separator": "", + "key": "foreman['location_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_')"}, + "organization": {"prefix": "{}organization_".format(group_prefix), + "separator": "", + "key": "foreman['organization_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_')"}, + "lifecycle_environment": {"prefix": "{}lifecycle_environment_".format(group_prefix), + "separator": "", + "key": "foreman['content_facet_attributes']['lifecycle_environment_name'] | " + "lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_')"}, + "content_view": {"prefix": "{}content_view_".format(group_prefix), + "separator": "", + "key": "foreman['content_facet_attributes']['content_view_name'] | " + "lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_')"} + } + + ret['legacy_hostvars'] = True # convert hostvar structure to the form used by the script + ret['want_params'] = True + ret['group_prefix'] = group_prefix + ret['want_hostcollections'] = want_hostcollections + ret['want_facts'] = want_facts + + if want_ansible_ssh_host: + ret['compose'] = {'ansible_ssh_host': "foreman['ip6'] | default(foreman['ip'], true)"} + ret['keyed_groups'] = [group_by_hostvar[grouping_name] for grouping_name in group_by_hostvar] + + def form_keyed_group(group_pattern): + """ + Converts foreman group_pattern to + inventory plugin keyed_group + + e.g. {app_param}-{tier_param}-{dc_param} + becomes + "%s-%s-%s" | format(app_param, tier_param, dc_param) + """ + if type(group_pattern) is not str: + return None + params = re.findall('{[^}]*}', group_pattern) + if len(params) == 0: + return None + + param_names = [] + for p in params: + param_names.append(p[1:-1].strip()) # strip braces and space + + # form keyed_group key by + # replacing curly braces with '%s' + # (for use with jinja's format filter) + key = group_pattern + for p in params: + key = key.replace(p, '%s', 1) + + # apply jinja filter to key + key = '"{}" | format({})'.format(key, ', '.join(param_names)) + + keyed_group = {'key': key, + 'separator': ''} + return keyed_group + + try: + group_patterns = json.loads(group_patterns) + + if type(group_patterns) is list: + for group_pattern in group_patterns: + keyed_group = form_keyed_group(group_pattern) + if keyed_group: + ret['keyed_groups'].append(keyed_group) + except json.JSONDecodeError: + logger.warning('Could not parse group_patterns. Expected JSON-formatted string, found: {}' + .format(group_patterns)) + + return ret + + +class tower(PluginFileInjector): + plugin_name = 'tower' + namespace = 'awx' + collection = 'awx' + + def inventory_as_dict(self, inventory_source, private_data_dir): + ret = super(tower, self).inventory_as_dict(inventory_source, private_data_dir) + # Credentials injected as env vars, same as script + try: + # plugin can take an actual int type + identifier = int(inventory_source.instance_filters) + except ValueError: + # inventory_id could be a named URL + identifier = iri_to_uri(inventory_source.instance_filters) + ret['inventory_id'] = identifier + ret['include_metadata'] = True # used for license check + return ret + + +for cls in PluginFileInjector.__subclasses__(): + FrozenInjectors[cls.__name__] = cls diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 137c056111..5305e6e532 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -4,7 +4,6 @@ # Python import datetime import time -import json import logging import re import copy @@ -19,7 +18,6 @@ from django.utils.translation import ugettext_lazy as _ from django.db import transaction from django.core.exceptions import ValidationError from django.utils.timezone import now -from django.utils.encoding import iri_to_uri from django.db.models import Q # REST Framework @@ -56,7 +54,7 @@ from awx.main.models.notifications import ( JobNotificationMixin, ) from awx.main.models.credential.injectors import _openstack_data -from awx.main.utils import _inventory_updates, region_sorting +from awx.main.utils import _inventory_updates from awx.main.utils.safe_yaml import sanitize_jinja @@ -838,89 +836,6 @@ class InventorySourceOptions(BaseModel): (2, '2 (DEBUG)'), ] - # Use tools/scripts/get_ec2_filter_names.py to build this list. - INSTANCE_FILTER_NAMES = [ - "architecture", - "association.allocation-id", - "association.association-id", - "association.ip-owner-id", - "association.public-ip", - "availability-zone", - "block-device-mapping.attach-time", - "block-device-mapping.delete-on-termination", - "block-device-mapping.device-name", - "block-device-mapping.status", - "block-device-mapping.volume-id", - "client-token", - "dns-name", - "group-id", - "group-name", - "hypervisor", - "iam-instance-profile.arn", - "image-id", - "instance-id", - "instance-lifecycle", - "instance-state-code", - "instance-state-name", - "instance-type", - "instance.group-id", - "instance.group-name", - "ip-address", - "kernel-id", - "key-name", - "launch-index", - "launch-time", - "monitoring-state", - "network-interface-private-dns-name", - "network-interface.addresses.association.ip-owner-id", - "network-interface.addresses.association.public-ip", - "network-interface.addresses.primary", - "network-interface.addresses.private-ip-address", - "network-interface.attachment.attach-time", - "network-interface.attachment.attachment-id", - "network-interface.attachment.delete-on-termination", - "network-interface.attachment.device-index", - "network-interface.attachment.instance-id", - "network-interface.attachment.instance-owner-id", - "network-interface.attachment.status", - "network-interface.availability-zone", - "network-interface.description", - "network-interface.group-id", - "network-interface.group-name", - "network-interface.mac-address", - "network-interface.network-interface.id", - "network-interface.owner-id", - "network-interface.requester-id", - "network-interface.requester-managed", - "network-interface.source-destination-check", - "network-interface.status", - "network-interface.subnet-id", - "network-interface.vpc-id", - "owner-id", - "placement-group-name", - "platform", - "private-dns-name", - "private-ip-address", - "product-code", - "product-code.type", - "ramdisk-id", - "reason", - "requester-id", - "reservation-id", - "root-device-name", - "root-device-type", - "source-dest-check", - "spot-instance-request-id", - "state-reason-code", - "state-reason-message", - "subnet-id", - "tag-key", - "tag-value", - "tenancy", - "virtualization-type", - "vpc-id" - ] - class Meta: abstract = True @@ -947,22 +862,38 @@ class InventorySourceOptions(BaseModel): default='', help_text=_('Inventory source variables in YAML or JSON format.'), ) - source_regions = models.CharField( - max_length=1024, + enabled_var = models.TextField( blank=True, default='', + help_text=_('Retrieve the enabled state from the given dict of host ' + 'variables. The enabled variable may be specified as "foo.bar", ' + 'in which case the lookup will traverse into nested dicts, ' + 'equivalent to: from_dict.get("foo", {}).get("bar", default)'), ) - instance_filters = models.CharField( - max_length=1024, + enabled_value = models.TextField( blank=True, default='', - help_text=_('Comma-separated list of filter expressions (EC2 only). Hosts are imported when ANY of the filters match.'), + help_text=_('Only used when enabled_var is set. Value when the host is ' + 'considered enabled. For example if enabled_var="status.power_state"' + 'and enabled_value="powered_on" with host variables:' + '{' + ' "status": {' + ' "power_state": "powered_on",' + ' "created": "2020-08-04T18:13:04+00:00",' + ' "healthy": true' + ' },' + ' "name": "foobar",' + ' "ip_address": "192.168.2.1"' + '}' + 'The host would be marked enabled. If power_state where any ' + 'value other than powered_on then the host would be disabled ' + 'when imported into Tower. If the key is not found then the ' + 'host will be enabled'), ) - group_by = models.CharField( - max_length=1024, + host_filter = models.TextField( blank=True, default='', - help_text=_('Limit groups automatically created from inventory source (EC2 only).'), + help_text=_('Regex where only matching hosts will be imported into Tower.'), ) overwrite = models.BooleanField( default=False, @@ -983,97 +914,6 @@ class InventorySourceOptions(BaseModel): default=1, ) - @classmethod - def get_ec2_region_choices(cls): - ec2_region_names = getattr(settings, 'EC2_REGION_NAMES', {}) - ec2_name_replacements = { - 'us': 'US', - 'ap': 'Asia Pacific', - 'eu': 'Europe', - 'sa': 'South America', - } - import boto.ec2 - regions = [('all', 'All')] - for region in boto.ec2.regions(): - label = ec2_region_names.get(region.name, '') - if not label: - label_parts = [] - for part in region.name.split('-'): - part = ec2_name_replacements.get(part.lower(), part.title()) - label_parts.append(part) - label = ' '.join(label_parts) - regions.append((region.name, label)) - return sorted(regions, key=region_sorting) - - @classmethod - def get_ec2_group_by_choices(cls): - return [ - ('ami_id', _('Image ID')), - ('availability_zone', _('Availability Zone')), - ('aws_account', _('Account')), - ('instance_id', _('Instance ID')), - ('instance_state', _('Instance State')), - ('platform', _('Platform')), - ('instance_type', _('Instance Type')), - ('key_pair', _('Key Name')), - ('region', _('Region')), - ('security_group', _('Security Group')), - ('tag_keys', _('Tags')), - ('tag_none', _('Tag None')), - ('vpc_id', _('VPC ID')), - ] - - @classmethod - def get_gce_region_choices(self): - """Return a complete list of regions in GCE, as a list of - two-tuples. - """ - # It's not possible to get a list of regions from GCE without - # authenticating first. Therefore, use a list from settings. - regions = list(getattr(settings, 'GCE_REGION_CHOICES', [])) - regions.insert(0, ('all', 'All')) - return sorted(regions, key=region_sorting) - - @classmethod - def get_azure_rm_region_choices(self): - """Return a complete list of regions in Microsoft Azure, as a list of - two-tuples. - """ - # It's not possible to get a list of regions from Azure without - # authenticating first (someone reading these might think there's - # a pattern here!). Therefore, you guessed it, use a list from - # settings. - regions = list(getattr(settings, 'AZURE_RM_REGION_CHOICES', [])) - regions.insert(0, ('all', 'All')) - return sorted(regions, key=region_sorting) - - @classmethod - def get_vmware_region_choices(self): - """Return a complete list of regions in VMware, as a list of two-tuples - (but note that VMware doesn't actually have regions!). - """ - return [('all', 'All')] - - @classmethod - def get_openstack_region_choices(self): - """I don't think openstack has regions""" - return [('all', 'All')] - - @classmethod - def get_satellite6_region_choices(self): - """Red Hat Satellite 6 region choices (not implemented)""" - return [('all', 'All')] - - @classmethod - def get_rhv_region_choices(self): - """No region supprt""" - return [('all', 'All')] - - @classmethod - def get_tower_region_choices(self): - """No region supprt""" - return [('all', 'All')] - @staticmethod def cloud_credential_validation(source, cred): if not source: @@ -1138,78 +978,9 @@ class InventorySourceOptions(BaseModel): if cred is not None: return cred.pk - def clean_source_regions(self): - regions = self.source_regions - - if self.source in CLOUD_PROVIDERS: - get_regions = getattr(self, 'get_%s_region_choices' % self.source) - valid_regions = [x[0] for x in get_regions()] - region_transform = lambda x: x.strip().lower() - else: - return '' - all_region = region_transform('all') - valid_regions = [region_transform(x) for x in valid_regions] - regions = [region_transform(x) for x in regions.split(',') if x.strip()] - if all_region in regions: - return all_region - invalid_regions = [] - for r in regions: - if r not in valid_regions and r not in invalid_regions: - invalid_regions.append(r) - if invalid_regions: - raise ValidationError(_('Invalid %(source)s region: %(region)s') % { - 'source': self.source, 'region': ', '.join(invalid_regions)}) - return ','.join(regions) source_vars_dict = VarsDictProperty('source_vars') - def clean_instance_filters(self): - instance_filters = str(self.instance_filters or '') - if self.source == 'ec2': - invalid_filters = [] - instance_filter_re = re.compile(r'^((tag:.+)|([a-z][a-z\.-]*[a-z]))=.*$') - for instance_filter in instance_filters.split(','): - instance_filter = instance_filter.strip() - if not instance_filter: - continue - if not instance_filter_re.match(instance_filter): - invalid_filters.append(instance_filter) - continue - instance_filter_name = instance_filter.split('=', 1)[0] - if instance_filter_name.startswith('tag:'): - continue - if instance_filter_name not in self.INSTANCE_FILTER_NAMES: - invalid_filters.append(instance_filter) - if invalid_filters: - raise ValidationError(_('Invalid filter expression: %(filter)s') % - {'filter': ', '.join(invalid_filters)}) - return instance_filters - elif self.source in ('vmware', 'tower'): - return instance_filters - else: - return '' - - def clean_group_by(self): - group_by = str(self.group_by or '') - if self.source == 'ec2': - get_choices = getattr(self, 'get_%s_group_by_choices' % self.source) - valid_choices = [x[0] for x in get_choices()] - choice_transform = lambda x: x.strip().lower() - valid_choices = [choice_transform(x) for x in valid_choices] - choices = [choice_transform(x) for x in group_by.split(',') if x.strip()] - invalid_choices = [] - for c in choices: - if c not in valid_choices and c not in invalid_choices: - invalid_choices.append(c) - if invalid_choices: - raise ValidationError(_('Invalid group by choice: %(choice)s') % - {'choice': ', '.join(invalid_choices)}) - return ','.join(choices) - elif self.source == 'vmware': - return group_by - else: - return '' - class InventorySource(UnifiedJobTemplate, InventorySourceOptions, CustomVirtualEnvMixin, RelatedJobsMixin): @@ -1602,9 +1373,11 @@ class PluginFileInjector(object): collection = None collection_migration = '2.9' # Starting with this version, we use collections - def __init__(self, ansible_version): - # This is InventoryOptions instance, could be source or inventory update - self.ansible_version = ansible_version + @classmethod + def get_proper_name(cls): + if cls.plugin_name is None: + return None + return f'{cls.namespace}.{cls.collection}.{cls.plugin_name}' @property def filename(self): @@ -1613,17 +1386,6 @@ class PluginFileInjector(object): """ return '{0}.yml'.format(self.plugin_name) - def inventory_as_dict(self, inventory_update, private_data_dir): - """Default implementation of inventory plugin file contents. - There are some valid cases when all parameters can be obtained from - the environment variables, example "plugin: linode" is valid - ideally, however, some options should be filled from the inventory source data - """ - if self.plugin_name is None: - raise NotImplementedError('At minimum the plugin name is needed for inventory plugin use.') - proper_name = f'{self.namespace}.{self.collection}.{self.plugin_name}' - return {'plugin': proper_name} - def inventory_contents(self, inventory_update, private_data_dir): """Returns a string that is the content for the inventory file for the inventory plugin """ @@ -1633,6 +1395,17 @@ class PluginFileInjector(object): width=1000 ) + def inventory_as_dict(self, inventory_update, private_data_dir): + source_vars = dict(inventory_update.source_vars_dict) # make a copy + proper_name = self.get_proper_name() + ''' + None conveys that we should use the user-provided plugin. + Note that a plugin value of '' should still be overridden. + ''' + if proper_name is not None: + source_vars['plugin'] = proper_name + return source_vars + def build_env(self, inventory_update, env, private_data_dir, private_data_files): injector_env = self.get_plugin_env(inventory_update, private_data_dir, private_data_files) env.update(injector_env) @@ -1690,106 +1463,6 @@ class azure_rm(PluginFileInjector): ret['ANSIBLE_JINJA2_NATIVE'] = str(True) return ret - def inventory_as_dict(self, inventory_update, private_data_dir): - ret = super(azure_rm, self).inventory_as_dict(inventory_update, private_data_dir) - - source_vars = inventory_update.source_vars_dict - - ret['fail_on_template_errors'] = False - - group_by_hostvar = { - 'location': {'prefix': '', 'separator': '', 'key': 'location'}, - 'tag': {'prefix': '', 'separator': '', 'key': 'tags.keys() | list if tags else []'}, - # Introduced with https://github.com/ansible/ansible/pull/53046 - 'security_group': {'prefix': '', 'separator': '', 'key': 'security_group'}, - 'resource_group': {'prefix': '', 'separator': '', 'key': 'resource_group'}, - # Note, os_family was not documented correctly in script, but defaulted to grouping by it - 'os_family': {'prefix': '', 'separator': '', 'key': 'os_disk.operating_system_type'} - } - # by default group by everything - # always respect user setting, if they gave it - group_by = [ - grouping_name for grouping_name in group_by_hostvar - if source_vars.get('group_by_{}'.format(grouping_name), True) - ] - ret['keyed_groups'] = [group_by_hostvar[grouping_name] for grouping_name in group_by] - if 'tag' in group_by: - # Nasty syntax to reproduce "key_value" group names in addition to "key" - ret['keyed_groups'].append({ - 'prefix': '', 'separator': '', - 'key': r'dict(tags.keys() | map("regex_replace", "^(.*)$", "\1_") | list | zip(tags.values() | list)) if tags else []' - }) - - # Compatibility content - # TODO: add proper support for instance_filters non-specific to compatibility - # TODO: add proper support for group_by non-specific to compatibility - # Dashes were not configurable in azure_rm.py script, we do not want unicode, so always use this - ret['use_contrib_script_compatible_sanitization'] = True - # use same host names as script - ret['plain_host_names'] = True - # By default the script did not filter hosts - ret['default_host_filters'] = [] - # User-given host filters - user_filters = [] - old_filterables = [ - ('resource_groups', 'resource_group'), - ('tags', 'tags') - # locations / location would be an entry - # but this would conflict with source_regions - ] - for key, loc in old_filterables: - value = source_vars.get(key, None) - if value and isinstance(value, str): - # tags can be list of key:value pairs - # e.g. 'Creator:jmarshall, peanutbutter:jelly' - # or tags can be a list of keys - # e.g. 'Creator, peanutbutter' - if key == "tags": - # grab each key value pair - for kvpair in value.split(','): - # split into key and value - kv = kvpair.split(':') - # filter out any host that does not have key - # in their tags.keys() variable - user_filters.append('"{}" not in tags.keys()'.format(kv[0].strip())) - # if a value is provided, check that the key:value pair matches - if len(kv) > 1: - user_filters.append('tags["{}"] != "{}"'.format(kv[0].strip(), kv[1].strip())) - else: - user_filters.append('{} not in {}'.format( - loc, value.split(',') - )) - if user_filters: - ret.setdefault('exclude_host_filters', []) - ret['exclude_host_filters'].extend(user_filters) - - ret['conditional_groups'] = {'azure': True} - ret['hostvar_expressions'] = { - 'provisioning_state': 'provisioning_state | title', - 'computer_name': 'name', - 'type': 'resource_type', - 'private_ip': 'private_ipv4_addresses[0] if private_ipv4_addresses else None', - 'public_ip': 'public_ipv4_addresses[0] if public_ipv4_addresses else None', - 'public_ip_name': 'public_ip_name if public_ip_name is defined else None', - 'public_ip_id': 'public_ip_id if public_ip_id is defined else None', - 'tags': 'tags if tags else None' - } - # Special functionality from script - if source_vars.get('use_private_ip', False): - ret['hostvar_expressions']['ansible_host'] = 'private_ipv4_addresses[0]' - # end compatibility content - - if inventory_update.source_regions and 'all' not in inventory_update.source_regions: - # initialize a list for this section in inventory file - ret.setdefault('exclude_host_filters', []) - # make a python list of the regions we will use - python_regions = [x.strip() for x in inventory_update.source_regions.split(',')] - # convert that list in memory to python syntax in a string - # now put that in jinja2 syntax operating on hostvar key "location" - # and put that as an entry in the exclusions list - ret['exclude_host_filters'].append("location not in {}".format(repr(python_regions))) - return ret - class ec2(PluginFileInjector): plugin_name = 'aws_ec2' @@ -1803,218 +1476,6 @@ class ec2(PluginFileInjector): ret['ANSIBLE_JINJA2_NATIVE'] = str(True) return ret - def _compat_compose_vars(self): - return { - # vars that change - 'ec2_block_devices': ( - "dict(block_device_mappings | map(attribute='device_name') | list | zip(block_device_mappings " - "| map(attribute='ebs.volume_id') | list))" - ), - 'ec2_dns_name': 'public_dns_name', - 'ec2_group_name': 'placement.group_name', - 'ec2_instance_profile': 'iam_instance_profile | default("")', - 'ec2_ip_address': 'public_ip_address', - 'ec2_kernel': 'kernel_id | default("")', - 'ec2_monitored': "monitoring.state in ['enabled', 'pending']", - 'ec2_monitoring_state': 'monitoring.state', - 'ec2_placement': 'placement.availability_zone', - 'ec2_ramdisk': 'ramdisk_id | default("")', - 'ec2_reason': 'state_transition_reason', - 'ec2_security_group_ids': "security_groups | map(attribute='group_id') | list | join(',')", - 'ec2_security_group_names': "security_groups | map(attribute='group_name') | list | join(',')", - 'ec2_tag_Name': 'tags.Name', - 'ec2_state': 'state.name', - 'ec2_state_code': 'state.code', - 'ec2_state_reason': 'state_reason.message if state_reason is defined else ""', - 'ec2_sourceDestCheck': 'source_dest_check | default(false) | lower | string', # snake_case syntax intended - 'ec2_account_id': 'owner_id', - # vars that just need ec2_ prefix - 'ec2_ami_launch_index': 'ami_launch_index | string', - 'ec2_architecture': 'architecture', - 'ec2_client_token': 'client_token', - 'ec2_ebs_optimized': 'ebs_optimized', - 'ec2_hypervisor': 'hypervisor', - 'ec2_image_id': 'image_id', - 'ec2_instance_type': 'instance_type', - 'ec2_key_name': 'key_name', - 'ec2_launch_time': r'launch_time | regex_replace(" ", "T") | regex_replace("(\+)(\d\d):(\d)(\d)$", ".\g<2>\g<3>Z")', - 'ec2_platform': 'platform | default("")', - 'ec2_private_dns_name': 'private_dns_name', - 'ec2_private_ip_address': 'private_ip_address', - 'ec2_public_dns_name': 'public_dns_name', - 'ec2_region': 'placement.region', - 'ec2_root_device_name': 'root_device_name', - 'ec2_root_device_type': 'root_device_type', - # many items need blank defaults because the script tended to keep a common schema - 'ec2_spot_instance_request_id': 'spot_instance_request_id | default("")', - 'ec2_subnet_id': 'subnet_id | default("")', - 'ec2_virtualization_type': 'virtualization_type', - 'ec2_vpc_id': 'vpc_id | default("")', - # same as ec2_ip_address, the script provided this - 'ansible_host': 'public_ip_address', - # new with https://github.com/ansible/ansible/pull/53645 - 'ec2_eventsSet': 'events | default("")', - 'ec2_persistent': 'persistent | default(false)', - 'ec2_requester_id': 'requester_id | default("")' - } - - def inventory_as_dict(self, inventory_update, private_data_dir): - ret = super(ec2, self).inventory_as_dict(inventory_update, private_data_dir) - - keyed_groups = [] - group_by_hostvar = { - 'ami_id': {'prefix': '', 'separator': '', 'key': 'image_id', 'parent_group': 'images'}, - # 2 entries for zones for same groups to establish 2 parentage trees - 'availability_zone': {'prefix': '', 'separator': '', 'key': 'placement.availability_zone', 'parent_group': 'zones'}, - 'aws_account': {'prefix': '', 'separator': '', 'key': 'ec2_account_id', 'parent_group': 'accounts'}, # composed var - 'instance_id': {'prefix': '', 'separator': '', 'key': 'instance_id', 'parent_group': 'instances'}, # normally turned off - 'instance_state': {'prefix': 'instance_state', 'key': 'ec2_state', 'parent_group': 'instance_states'}, # composed var - # ec2_platform is a composed var, but group names do not match up to hostvar exactly - 'platform': {'prefix': 'platform', 'key': 'platform | default("undefined")', 'parent_group': 'platforms'}, - 'instance_type': {'prefix': 'type', 'key': 'instance_type', 'parent_group': 'types'}, - 'key_pair': {'prefix': 'key', 'key': 'key_name', 'parent_group': 'keys'}, - 'region': {'prefix': '', 'separator': '', 'key': 'placement.region', 'parent_group': 'regions'}, - # Security requires some ninja jinja2 syntax, credit to s-hertel - 'security_group': {'prefix': 'security_group', 'key': 'security_groups | map(attribute="group_name")', 'parent_group': 'security_groups'}, - # tags cannot be parented in exactly the same way as the script due to - # https://github.com/ansible/ansible/pull/53812 - 'tag_keys': [ - {'prefix': 'tag', 'key': 'tags', 'parent_group': 'tags'}, - {'prefix': 'tag', 'key': 'tags.keys()', 'parent_group': 'tags'} - ], - # 'tag_none': None, # grouping by no tags isn't a different thing with plugin - # naming is redundant, like vpc_id_vpc_8c412cea, but intended - 'vpc_id': {'prefix': 'vpc_id', 'key': 'vpc_id', 'parent_group': 'vpcs'}, - } - # -- same-ish as script here -- - group_by = [x.strip().lower() for x in inventory_update.group_by.split(',') if x.strip()] - for choice in inventory_update.get_ec2_group_by_choices(): - value = bool((group_by and choice[0] in group_by) or (not group_by and choice[0] != 'instance_id')) - # -- end sameness to script -- - if value: - this_keyed_group = group_by_hostvar.get(choice[0], None) - # If a keyed group syntax does not exist, there is nothing we can do to get this group - if this_keyed_group is not None: - if isinstance(this_keyed_group, list): - keyed_groups.extend(this_keyed_group) - else: - keyed_groups.append(this_keyed_group) - # special case, this parentage is only added if both zones and regions are present - if not group_by or ('region' in group_by and 'availability_zone' in group_by): - keyed_groups.append({'prefix': '', 'separator': '', 'key': 'placement.availability_zone', 'parent_group': '{{ placement.region }}'}) - - source_vars = inventory_update.source_vars_dict - # This is a setting from the script, hopefully no one used it - # if true, it replaces dashes, but not in region / loc names - replace_dash = bool(source_vars.get('replace_dash_in_groups', True)) - # Compatibility content - legacy_regex = { - True: r"[^A-Za-z0-9\_]", - False: r"[^A-Za-z0-9\_\-]" # do not replace dash, dash is allowed - }[replace_dash] - list_replacer = 'map("regex_replace", "{rx}", "_") | list'.format(rx=legacy_regex) - # this option, a plugin option, will allow dashes, but not unicode - # when set to False, unicode will be allowed, but it was not allowed by script - # thus, we always have to use this option, and always use our custom regex - ret['use_contrib_script_compatible_sanitization'] = True - for grouping_data in keyed_groups: - if grouping_data['key'] in ('placement.region', 'placement.availability_zone'): - # us-east-2 is always us-east-2 according to ec2.py - # no sanitization in region-ish groups for the script standards, ever ever - continue - if grouping_data['key'] == 'tags': - # dict jinja2 transformation - grouping_data['key'] = 'dict(tags.keys() | {replacer} | zip(tags.values() | {replacer}))'.format( - replacer=list_replacer - ) - elif grouping_data['key'] == 'tags.keys()' or grouping_data['prefix'] == 'security_group': - # list jinja2 transformation - grouping_data['key'] += ' | {replacer}'.format(replacer=list_replacer) - else: - # string transformation - grouping_data['key'] += ' | regex_replace("{rx}", "_")'.format(rx=legacy_regex) - # end compatibility content - - if source_vars.get('iam_role_arn', None): - ret['iam_role_arn'] = source_vars['iam_role_arn'] - - # This was an allowed ec2.ini option, also plugin option, so pass through - if source_vars.get('boto_profile', None): - ret['boto_profile'] = source_vars['boto_profile'] - - elif not replace_dash: - # Using the plugin, but still want dashes allowed - ret['use_contrib_script_compatible_sanitization'] = True - - if source_vars.get('nested_groups') is False: - for this_keyed_group in keyed_groups: - this_keyed_group.pop('parent_group', None) - - if keyed_groups: - ret['keyed_groups'] = keyed_groups - - # Instance ID not part of compat vars, because of settings.EC2_INSTANCE_ID_VAR - compose_dict = {'ec2_id': 'instance_id'} - inst_filters = {} - - # Compatibility content - compose_dict.update(self._compat_compose_vars()) - # plugin provides "aws_ec2", but not this which the script gave - ret['groups'] = {'ec2': True} - if source_vars.get('hostname_variable') is not None: - hnames = [] - for expr in source_vars.get('hostname_variable').split(','): - if expr == 'public_dns_name': - hnames.append('dns-name') - elif not expr.startswith('tag:') and '_' in expr: - hnames.append(expr.replace('_', '-')) - else: - hnames.append(expr) - ret['hostnames'] = hnames - else: - # public_ip as hostname is non-default plugin behavior, script behavior - ret['hostnames'] = [ - 'network-interface.addresses.association.public-ip', - 'dns-name', - 'private-dns-name' - ] - # The script returned only running state by default, the plugin does not - # https://docs.aws.amazon.com/cli/latest/reference/ec2/describe-instances.html#options - # options: pending | running | shutting-down | terminated | stopping | stopped - inst_filters['instance-state-name'] = ['running'] - # end compatibility content - - if source_vars.get('destination_variable') or source_vars.get('vpc_destination_variable'): - for fd in ('destination_variable', 'vpc_destination_variable'): - if source_vars.get(fd): - compose_dict['ansible_host'] = source_vars.get(fd) - break - - if compose_dict: - ret['compose'] = compose_dict - - if inventory_update.instance_filters: - # logic used to live in ec2.py, now it belongs to us. Yay more code? - filter_sets = [f for f in inventory_update.instance_filters.split(',') if f] - - for instance_filter in filter_sets: - # AND logic not supported, unclear how to... - instance_filter = instance_filter.strip() - if not instance_filter or '=' not in instance_filter: - continue - filter_key, filter_value = [x.strip() for x in instance_filter.split('=', 1)] - if not filter_key: - continue - inst_filters[filter_key] = filter_value - - if inst_filters: - ret['filters'] = inst_filters - - if inventory_update.source_regions and 'all' not in inventory_update.source_regions: - ret['regions'] = inventory_update.source_regions.split(',') - - return ret - class gce(PluginFileInjector): plugin_name = 'gcp_compute' @@ -2028,76 +1489,12 @@ class gce(PluginFileInjector): ret['ANSIBLE_JINJA2_NATIVE'] = str(True) return ret - def _compat_compose_vars(self): - # missing: gce_image, gce_uuid - # https://github.com/ansible/ansible/issues/51884 - return { - 'gce_description': 'description if description else None', - 'gce_machine_type': 'machineType', - 'gce_name': 'name', - 'gce_network': 'networkInterfaces[0].network.name', - 'gce_private_ip': 'networkInterfaces[0].networkIP', - 'gce_public_ip': 'networkInterfaces[0].accessConfigs[0].natIP | default(None)', - 'gce_status': 'status', - 'gce_subnetwork': 'networkInterfaces[0].subnetwork.name', - 'gce_tags': 'tags.get("items", [])', - 'gce_zone': 'zone', - 'gce_metadata': 'metadata.get("items", []) | items2dict(key_name="key", value_name="value")', - # NOTE: image hostvar is enabled via retrieve_image_info option - 'gce_image': 'image', - # We need this as long as hostnames is non-default, otherwise hosts - # will not be addressed correctly, was returned in script - 'ansible_ssh_host': 'networkInterfaces[0].accessConfigs[0].natIP | default(networkInterfaces[0].networkIP)' - } - def inventory_as_dict(self, inventory_update, private_data_dir): - ret = super(gce, self).inventory_as_dict(inventory_update, private_data_dir) + ret = super().inventory_as_dict(inventory_update, private_data_dir) credential = inventory_update.get_cloud_credential() - - # auth related items - ret['projects'] = [credential.get_input('project', default='')] - ret['auth_kind'] = "serviceaccount" - - filters = [] - # TODO: implement gce group_by options - # gce never processed the group_by field, if it had, we would selectively - # apply those options here, but it did not, so all groups are added here - keyed_groups = [ - # the jinja2 syntax is duplicated with compose - # https://github.com/ansible/ansible/issues/51883 - {'prefix': 'network', 'key': 'gce_subnetwork'}, # composed var - {'prefix': '', 'separator': '', 'key': 'gce_private_ip'}, # composed var - {'prefix': '', 'separator': '', 'key': 'gce_public_ip'}, # composed var - {'prefix': '', 'separator': '', 'key': 'machineType'}, - {'prefix': '', 'separator': '', 'key': 'zone'}, - {'prefix': 'tag', 'key': 'gce_tags'}, # composed var - {'prefix': 'status', 'key': 'status | lower'}, - # NOTE: image hostvar is enabled via retrieve_image_info option - {'prefix': '', 'separator': '', 'key': 'image'}, - ] - # This will be used as the gce instance_id, must be universal, non-compat - compose_dict = {'gce_id': 'id'} - - # Compatibility content - # TODO: proper group_by and instance_filters support, irrelevant of compat mode - # The gce.py script never sanitized any names in any way - ret['use_contrib_script_compatible_sanitization'] = True - # Perform extra API query to get the image hostvar - ret['retrieve_image_info'] = True - # Add in old hostvars aliases - compose_dict.update(self._compat_compose_vars()) - # Non-default names to match script - ret['hostnames'] = ['name', 'public_ip', 'private_ip'] - # end compatibility content - - if keyed_groups: - ret['keyed_groups'] = keyed_groups - if filters: - ret['filters'] = filters - if compose_dict: - ret['compose'] = compose_dict - if inventory_update.source_regions and 'all' not in inventory_update.source_regions: - ret['zones'] = inventory_update.source_regions.split(',') + # InventorySource.source_vars take precedence over ENV vars + if 'projects' not in ret: + ret['projects'] = [credential.get_input('project', default='')] return ret @@ -2107,106 +1504,6 @@ class vmware(PluginFileInjector): namespace = 'community' collection = 'vmware' - def inventory_as_dict(self, inventory_update, private_data_dir): - ret = super(vmware, self).inventory_as_dict(inventory_update, private_data_dir) - ret['strict'] = False - # Documentation of props, see - # https://github.com/ansible/ansible/blob/devel/docs/docsite/rst/scenario_guides/vmware_scenarios/vmware_inventory_vm_attributes.rst - UPPERCASE_PROPS = [ - "availableField", - "configIssue", - "configStatus", - "customValue", # optional - "datastore", - "effectiveRole", - "guestHeartbeatStatus", # optional - "layout", # optional - "layoutEx", # optional - "name", - "network", - "overallStatus", - "parentVApp", # optional - "permission", - "recentTask", - "resourcePool", - "rootSnapshot", - "snapshot", # optional - "triggeredAlarmState", - "value" - ] - NESTED_PROPS = [ - "capability", - "config", - "guest", - "runtime", - "storage", - "summary", # repeat of other properties - ] - ret['properties'] = UPPERCASE_PROPS + NESTED_PROPS - ret['compose'] = {'ansible_host': 'guest.ipAddress'} # default value - ret['compose']['ansible_ssh_host'] = ret['compose']['ansible_host'] - # the ansible_uuid was unique every host, every import, from the script - ret['compose']['ansible_uuid'] = '99999999 | random | to_uuid' - for prop in UPPERCASE_PROPS: - if prop == prop.lower(): - continue - ret['compose'][prop.lower()] = prop - ret['with_nested_properties'] = True - # ret['property_name_format'] = 'lower_case' # only dacrystal/topic/vmware-inventory-plugin-property-format - - # process custom options - vmware_opts = dict(inventory_update.source_vars_dict.items()) - if inventory_update.instance_filters: - vmware_opts.setdefault('host_filters', inventory_update.instance_filters) - if inventory_update.group_by: - vmware_opts.setdefault('groupby_patterns', inventory_update.group_by) - - alias_pattern = vmware_opts.get('alias_pattern') - if alias_pattern: - ret.setdefault('hostnames', []) - for alias in alias_pattern.split(','): # make best effort - striped_alias = alias.replace('{', '').replace('}', '').strip() # make best effort - if not striped_alias: - continue - ret['hostnames'].append(striped_alias) - - host_pattern = vmware_opts.get('host_pattern') # not working in script - if host_pattern: - stripped_hp = host_pattern.replace('{', '').replace('}', '').strip() # make best effort - ret['compose']['ansible_host'] = stripped_hp - ret['compose']['ansible_ssh_host'] = stripped_hp - - host_filters = vmware_opts.get('host_filters') - if host_filters: - ret.setdefault('filters', []) - for hf in host_filters.split(','): - striped_hf = hf.replace('{', '').replace('}', '').strip() # make best effort - if not striped_hf: - continue - ret['filters'].append(striped_hf) - else: - # default behavior filters by power state - ret['filters'] = ['runtime.powerState == "poweredOn"'] - - groupby_patterns = vmware_opts.get('groupby_patterns') - ret.setdefault('keyed_groups', []) - if groupby_patterns: - for pattern in groupby_patterns.split(','): - stripped_pattern = pattern.replace('{', '').replace('}', '').strip() # make best effort - ret['keyed_groups'].append({ - 'prefix': '', 'separator': '', - 'key': stripped_pattern - }) - else: - # default groups from script - for entry in ('config.guestId', '"templates" if config.template else "guests"'): - ret['keyed_groups'].append({ - 'prefix': '', 'separator': '', - 'key': entry - }) - - return ret - class openstack(PluginFileInjector): plugin_name = 'openstack' @@ -2243,40 +1540,6 @@ class openstack(PluginFileInjector): ) return private_data - def inventory_as_dict(self, inventory_update, private_data_dir): - def use_host_name_for_name(a_bool_maybe): - if not isinstance(a_bool_maybe, bool): - # Could be specified by user via "host" or "uuid" - return a_bool_maybe - elif a_bool_maybe: - return 'name' # plugin default - else: - return 'uuid' - - ret = super(openstack, self).inventory_as_dict(inventory_update, private_data_dir) - ret['fail_on_errors'] = True - ret['expand_hostvars'] = True - ret['inventory_hostname'] = use_host_name_for_name(False) - # Note: mucking with defaults will break import integrity - # For the plugin, we need to use the same defaults as the old script - # or else imports will conflict. To find script defaults you have - # to read source code of the script. - # - # Script Defaults Plugin Defaults - # 'use_hostnames': False, 'name' (True) - # 'expand_hostvars': True, 'no' (False) - # 'fail_on_errors': True, 'no' (False) - # - # These are, yet again, different from ansible_variables in script logic - # but those are applied inconsistently - source_vars = inventory_update.source_vars_dict - for var_name in ['expand_hostvars', 'fail_on_errors']: - if var_name in source_vars: - ret[var_name] = source_vars[var_name] - if 'use_hostnames' in source_vars: - ret['inventory_hostname'] = use_host_name_for_name(source_vars['use_hostnames']) - return ret - def get_plugin_env(self, inventory_update, private_data_dir, private_data_files): env = super(openstack, self).get_plugin_env(inventory_update, private_data_dir, private_data_files) credential = inventory_update.get_cloud_credential() @@ -2294,25 +1557,6 @@ class rhv(PluginFileInjector): namespace = 'ovirt' collection = 'ovirt' - def inventory_as_dict(self, inventory_update, private_data_dir): - ret = super(rhv, self).inventory_as_dict(inventory_update, private_data_dir) - ret['ovirt_insecure'] = False # Default changed from script - # TODO: process strict option upstream - ret['compose'] = { - 'ansible_host': '(devices.values() | list)[0][0] if devices else None' - } - ret['keyed_groups'] = [] - for key in ('cluster', 'status'): - ret['keyed_groups'].append({'prefix': key, 'separator': '_', 'key': key}) - ret['keyed_groups'].append({'prefix': 'tag', 'separator': '_', 'key': 'tags'}) - ret['ovirt_hostname_preference'] = ['name', 'fqdn'] - source_vars = inventory_update.source_vars_dict - for key, value in source_vars.items(): - if key == 'plugin': - continue - ret[key] = value - return ret - class satellite6(PluginFileInjector): plugin_name = 'foreman' @@ -2330,114 +1574,6 @@ class satellite6(PluginFileInjector): ret['FOREMAN_PASSWORD'] = credential.get_input('password', default='') return ret - def inventory_as_dict(self, inventory_update, private_data_dir): - ret = super(satellite6, self).inventory_as_dict(inventory_update, private_data_dir) - ret['validate_certs'] = False - - group_patterns = '[]' - group_prefix = 'foreman_' - want_hostcollections = False - want_ansible_ssh_host = False - want_facts = True - - foreman_opts = inventory_update.source_vars_dict.copy() - for k, v in foreman_opts.items(): - if k == 'satellite6_group_patterns' and isinstance(v, str): - group_patterns = v - elif k == 'satellite6_group_prefix' and isinstance(v, str): - group_prefix = v - elif k == 'satellite6_want_hostcollections' and isinstance(v, bool): - want_hostcollections = v - elif k == 'satellite6_want_ansible_ssh_host' and isinstance(v, bool): - want_ansible_ssh_host = v - elif k == 'satellite6_want_facts' and isinstance(v, bool): - want_facts = v - # add backwards support for ssl_verify - # plugin uses new option, validate_certs, instead - elif k == 'ssl_verify' and isinstance(v, bool): - ret['validate_certs'] = v - else: - ret[k] = str(v) - - # Compatibility content - group_by_hostvar = { - "environment": {"prefix": "{}environment_".format(group_prefix), - "separator": "", - "key": "foreman['environment_name'] | lower | regex_replace(' ', '') | " - "regex_replace('[^A-Za-z0-9_]', '_') | regex_replace('none', '')"}, - "location": {"prefix": "{}location_".format(group_prefix), - "separator": "", - "key": "foreman['location_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_')"}, - "organization": {"prefix": "{}organization_".format(group_prefix), - "separator": "", - "key": "foreman['organization_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_')"}, - "lifecycle_environment": {"prefix": "{}lifecycle_environment_".format(group_prefix), - "separator": "", - "key": "foreman['content_facet_attributes']['lifecycle_environment_name'] | " - "lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_')"}, - "content_view": {"prefix": "{}content_view_".format(group_prefix), - "separator": "", - "key": "foreman['content_facet_attributes']['content_view_name'] | " - "lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_')"} - } - - ret['legacy_hostvars'] = True # convert hostvar structure to the form used by the script - ret['want_params'] = True - ret['group_prefix'] = group_prefix - ret['want_hostcollections'] = want_hostcollections - ret['want_facts'] = want_facts - - if want_ansible_ssh_host: - ret['compose'] = {'ansible_ssh_host': "foreman['ip6'] | default(foreman['ip'], true)"} - ret['keyed_groups'] = [group_by_hostvar[grouping_name] for grouping_name in group_by_hostvar] - - def form_keyed_group(group_pattern): - """ - Converts foreman group_pattern to - inventory plugin keyed_group - - e.g. {app_param}-{tier_param}-{dc_param} - becomes - "%s-%s-%s" | format(app_param, tier_param, dc_param) - """ - if type(group_pattern) is not str: - return None - params = re.findall('{[^}]*}', group_pattern) - if len(params) == 0: - return None - - param_names = [] - for p in params: - param_names.append(p[1:-1].strip()) # strip braces and space - - # form keyed_group key by - # replacing curly braces with '%s' - # (for use with jinja's format filter) - key = group_pattern - for p in params: - key = key.replace(p, '%s', 1) - - # apply jinja filter to key - key = '"{}" | format({})'.format(key, ', '.join(param_names)) - - keyed_group = {'key': key, - 'separator': ''} - return keyed_group - - try: - group_patterns = json.loads(group_patterns) - - if type(group_patterns) is list: - for group_pattern in group_patterns: - keyed_group = form_keyed_group(group_pattern) - if keyed_group: - ret['keyed_groups'].append(keyed_group) - except json.JSONDecodeError: - logger.warning('Could not parse group_patterns. Expected JSON-formatted string, found: {}' - .format(group_patterns)) - - return ret - class tower(PluginFileInjector): plugin_name = 'tower' @@ -2445,19 +1581,6 @@ class tower(PluginFileInjector): namespace = 'awx' collection = 'awx' - def inventory_as_dict(self, inventory_update, private_data_dir): - ret = super(tower, self).inventory_as_dict(inventory_update, private_data_dir) - # Credentials injected as env vars, same as script - try: - # plugin can take an actual int type - identifier = int(inventory_update.instance_filters) - except ValueError: - # inventory_id could be a named URL - identifier = iri_to_uri(inventory_update.instance_filters) - ret['inventory_id'] = identifier - ret['include_metadata'] = True # used for license check - return ret - for cls in PluginFileInjector.__subclasses__(): InventorySourceOptions.injectors[cls.__name__] = cls diff --git a/awx/main/tasks.py b/awx/main/tasks.py index acd7548a91..ac1a9dca04 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -23,6 +23,7 @@ import fcntl from pathlib import Path from uuid import uuid4 import urllib.parse as urlparse +import shlex # Django from django.conf import settings @@ -72,7 +73,7 @@ from awx.main.utils import (update_scm_url, ignore_inventory_group_removal, extract_ansible_vars, schedule_task_manager, get_awx_version) from awx.main.utils.ansible import read_ansible_config -from awx.main.utils.common import _get_ansible_version, get_custom_venv_choices +from awx.main.utils.common import get_custom_venv_choices from awx.main.utils.external_logging import reconfigure_rsyslog from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja from awx.main.utils.reload import stop_local_services @@ -840,12 +841,6 @@ class BaseTask(object): logger.error('Failed to update %s after %d retries.', self.model._meta.object_name, _attempt) - def get_ansible_version(self, instance): - if not hasattr(self, '_ansible_version'): - self._ansible_version = _get_ansible_version( - ansible_path=self.get_path_to_ansible(instance, executable='ansible')) - return self._ansible_version - def get_path_to(self, *args): ''' Return absolute path relative to this file. @@ -2459,7 +2454,7 @@ class RunInventoryUpdate(BaseTask): If no private data is needed, return None. """ if inventory_update.source in InventorySource.injectors: - injector = InventorySource.injectors[inventory_update.source](self.get_ansible_version(inventory_update)) + injector = InventorySource.injectors[inventory_update.source]() return injector.build_private_data(inventory_update, private_data_dir) def build_env(self, inventory_update, private_data_dir, isolated, private_data_files=None): @@ -2487,7 +2482,7 @@ class RunInventoryUpdate(BaseTask): injector = None if inventory_update.source in InventorySource.injectors: - injector = InventorySource.injectors[inventory_update.source](self.get_ansible_version(inventory_update)) + injector = InventorySource.injectors[inventory_update.source]() if injector is not None: env = injector.build_env(inventory_update, env, private_data_dir, private_data_files) @@ -2559,23 +2554,18 @@ class RunInventoryUpdate(BaseTask): args.extend(['--venv', inventory_update.ansible_virtualenv_path]) src = inventory_update.source - # Add several options to the shell arguments based on the - # inventory-source-specific setting in the AWX configuration. - # These settings are "per-source"; it's entirely possible that - # they will be different between cloud providers if an AWX user - # actively uses more than one. - if getattr(settings, '%s_ENABLED_VAR' % src.upper(), False): - args.extend(['--enabled-var', - getattr(settings, '%s_ENABLED_VAR' % src.upper())]) - if getattr(settings, '%s_ENABLED_VALUE' % src.upper(), False): - args.extend(['--enabled-value', - getattr(settings, '%s_ENABLED_VALUE' % src.upper())]) - if getattr(settings, '%s_GROUP_FILTER' % src.upper(), False): - args.extend(['--group-filter', - getattr(settings, '%s_GROUP_FILTER' % src.upper())]) - if getattr(settings, '%s_HOST_FILTER' % src.upper(), False): - args.extend(['--host-filter', - getattr(settings, '%s_HOST_FILTER' % src.upper())]) + if inventory_update.enabled_var: + args.extend(['--enabled-var', shlex.quote(inventory_update.enabled_var)]) + args.extend(['--enabled-value', shlex.quote(inventory_update.enabled_value)]) + else: + if getattr(settings, '%s_ENABLED_VAR' % src.upper(), False): + args.extend(['--enabled-var', + getattr(settings, '%s_ENABLED_VAR' % src.upper())]) + if getattr(settings, '%s_ENABLED_VALUE' % src.upper(), False): + args.extend(['--enabled-value', + getattr(settings, '%s_ENABLED_VALUE' % src.upper())]) + if inventory_update.host_filter: + args.extend(['--host-filter', shlex.quote(inventory_update.host_filter)]) if getattr(settings, '%s_EXCLUDE_EMPTY_GROUPS' % src.upper()): args.append('--exclude-empty-groups') if getattr(settings, '%s_INSTANCE_ID_VAR' % src.upper(), False): @@ -2605,7 +2595,7 @@ class RunInventoryUpdate(BaseTask): injector = None if inventory_update.source in InventorySource.injectors: - injector = InventorySource.injectors[src](self.get_ansible_version(inventory_update)) + injector = InventorySource.injectors[src]() if injector is not None: content = injector.inventory_contents(inventory_update, private_data_dir) diff --git a/awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml b/awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml deleted file mode 100644 index 8d6c1dbfa7..0000000000 --- a/awx/main/tests/data/inventory/plugins/azure_rm/files/azure_rm.yml +++ /dev/null @@ -1,43 +0,0 @@ -conditional_groups: - azure: true -default_host_filters: [] -exclude_host_filters: -- resource_group not in ['foo_resources', 'bar_resources'] -- '"Creator" not in tags.keys()' -- tags["Creator"] != "jmarshall" -- '"peanutbutter" not in tags.keys()' -- tags["peanutbutter"] != "jelly" -- location not in ['southcentralus', 'westus'] -fail_on_template_errors: false -hostvar_expressions: - ansible_host: private_ipv4_addresses[0] - computer_name: name - private_ip: private_ipv4_addresses[0] if private_ipv4_addresses else None - provisioning_state: provisioning_state | title - public_ip: public_ipv4_addresses[0] if public_ipv4_addresses else None - public_ip_id: public_ip_id if public_ip_id is defined else None - public_ip_name: public_ip_name if public_ip_name is defined else None - tags: tags if tags else None - type: resource_type -keyed_groups: -- key: location - prefix: '' - separator: '' -- key: tags.keys() | list if tags else [] - prefix: '' - separator: '' -- key: security_group - prefix: '' - separator: '' -- key: resource_group - prefix: '' - separator: '' -- key: os_disk.operating_system_type - prefix: '' - separator: '' -- key: dict(tags.keys() | map("regex_replace", "^(.*)$", "\1_") | list | zip(tags.values() | list)) if tags else [] - prefix: '' - separator: '' -plain_host_names: true -plugin: azure.azcollection.azure_rm -use_contrib_script_compatible_sanitization: true diff --git a/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml b/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml deleted file mode 100644 index 8984d4cb56..0000000000 --- a/awx/main/tests/data/inventory/plugins/ec2/files/aws_ec2.yml +++ /dev/null @@ -1,81 +0,0 @@ -boto_profile: /tmp/my_boto_stuff -compose: - ansible_host: public_dns_name - ec2_account_id: owner_id - ec2_ami_launch_index: ami_launch_index | string - ec2_architecture: architecture - ec2_block_devices: dict(block_device_mappings | map(attribute='device_name') | list | zip(block_device_mappings | map(attribute='ebs.volume_id') | list)) - ec2_client_token: client_token - ec2_dns_name: public_dns_name - ec2_ebs_optimized: ebs_optimized - ec2_eventsSet: events | default("") - ec2_group_name: placement.group_name - ec2_hypervisor: hypervisor - ec2_id: instance_id - ec2_image_id: image_id - ec2_instance_profile: iam_instance_profile | default("") - ec2_instance_type: instance_type - ec2_ip_address: public_ip_address - ec2_kernel: kernel_id | default("") - ec2_key_name: key_name - ec2_launch_time: launch_time | regex_replace(" ", "T") | regex_replace("(\+)(\d\d):(\d)(\d)$", ".\g<2>\g<3>Z") - ec2_monitored: monitoring.state in ['enabled', 'pending'] - ec2_monitoring_state: monitoring.state - ec2_persistent: persistent | default(false) - ec2_placement: placement.availability_zone - ec2_platform: platform | default("") - ec2_private_dns_name: private_dns_name - ec2_private_ip_address: private_ip_address - ec2_public_dns_name: public_dns_name - ec2_ramdisk: ramdisk_id | default("") - ec2_reason: state_transition_reason - ec2_region: placement.region - ec2_requester_id: requester_id | default("") - ec2_root_device_name: root_device_name - ec2_root_device_type: root_device_type - ec2_security_group_ids: security_groups | map(attribute='group_id') | list | join(',') - ec2_security_group_names: security_groups | map(attribute='group_name') | list | join(',') - ec2_sourceDestCheck: source_dest_check | default(false) | lower | string - ec2_spot_instance_request_id: spot_instance_request_id | default("") - ec2_state: state.name - ec2_state_code: state.code - ec2_state_reason: state_reason.message if state_reason is defined else "" - ec2_subnet_id: subnet_id | default("") - ec2_tag_Name: tags.Name - ec2_virtualization_type: virtualization_type - ec2_vpc_id: vpc_id | default("") -filters: - instance-state-name: - - running -groups: - ec2: true -hostnames: -- dns-name -iam_role_arn: arn:aws:iam::123456789012:role/test-role -keyed_groups: -- key: placement.availability_zone - parent_group: zones - prefix: '' - separator: '' -- key: instance_type | regex_replace("[^A-Za-z0-9\_]", "_") - parent_group: types - prefix: type -- key: placement.region - parent_group: regions - prefix: '' - separator: '' -- key: dict(tags.keys() | map("regex_replace", "[^A-Za-z0-9\_]", "_") | list | zip(tags.values() | map("regex_replace", "[^A-Za-z0-9\_]", "_") | list)) - parent_group: tags - prefix: tag -- key: tags.keys() | map("regex_replace", "[^A-Za-z0-9\_]", "_") | list - parent_group: tags - prefix: tag -- key: placement.availability_zone - parent_group: '{{ placement.region }}' - prefix: '' - separator: '' -plugin: amazon.aws.aws_ec2 -regions: -- us-east-2 -- ap-south-1 -use_contrib_script_compatible_sanitization: true diff --git a/awx/main/tests/data/inventory/plugins/gce/files/gcp_compute.yml b/awx/main/tests/data/inventory/plugins/gce/files/gcp_compute.yml deleted file mode 100644 index 63f8a44f64..0000000000 --- a/awx/main/tests/data/inventory/plugins/gce/files/gcp_compute.yml +++ /dev/null @@ -1,50 +0,0 @@ -auth_kind: serviceaccount -compose: - ansible_ssh_host: networkInterfaces[0].accessConfigs[0].natIP | default(networkInterfaces[0].networkIP) - gce_description: description if description else None - gce_id: id - gce_image: image - gce_machine_type: machineType - gce_metadata: metadata.get("items", []) | items2dict(key_name="key", value_name="value") - gce_name: name - gce_network: networkInterfaces[0].network.name - gce_private_ip: networkInterfaces[0].networkIP - gce_public_ip: networkInterfaces[0].accessConfigs[0].natIP | default(None) - gce_status: status - gce_subnetwork: networkInterfaces[0].subnetwork.name - gce_tags: tags.get("items", []) - gce_zone: zone -hostnames: -- name -- public_ip -- private_ip -keyed_groups: -- key: gce_subnetwork - prefix: network -- key: gce_private_ip - prefix: '' - separator: '' -- key: gce_public_ip - prefix: '' - separator: '' -- key: machineType - prefix: '' - separator: '' -- key: zone - prefix: '' - separator: '' -- key: gce_tags - prefix: tag -- key: status | lower - prefix: status -- key: image - prefix: '' - separator: '' -plugin: google.cloud.gcp_compute -projects: -- fooo -retrieve_image_info: true -use_contrib_script_compatible_sanitization: true -zones: -- us-east4-a -- us-west1-b diff --git a/awx/main/tests/data/inventory/plugins/openstack/files/file_reference b/awx/main/tests/data/inventory/plugins/openstack/files/file_reference index 895a1eb8a8..c578942ca1 100644 --- a/awx/main/tests/data/inventory/plugins/openstack/files/file_reference +++ b/awx/main/tests/data/inventory/plugins/openstack/files/file_reference @@ -1,7 +1,3 @@ -ansible: - expand_hostvars: true - fail_on_errors: true - use_hostnames: false clouds: devstack: auth: @@ -11,5 +7,5 @@ clouds: project_domain_name: fooo project_name: fooo username: fooo - private: false + private: true verify: false diff --git a/awx/main/tests/data/inventory/plugins/openstack/files/openstack.yml b/awx/main/tests/data/inventory/plugins/openstack/files/openstack.yml deleted file mode 100644 index 36e9024b54..0000000000 --- a/awx/main/tests/data/inventory/plugins/openstack/files/openstack.yml +++ /dev/null @@ -1,4 +0,0 @@ -expand_hostvars: true -fail_on_errors: true -inventory_hostname: uuid -plugin: openstack.cloud.openstack diff --git a/awx/main/tests/data/inventory/plugins/rhv/files/ovirt.yml b/awx/main/tests/data/inventory/plugins/rhv/files/ovirt.yml deleted file mode 100644 index 67a94ae6de..0000000000 --- a/awx/main/tests/data/inventory/plugins/rhv/files/ovirt.yml +++ /dev/null @@ -1,20 +0,0 @@ -base_source_var: value_of_var -compose: - ansible_host: (devices.values() | list)[0][0] if devices else None -groups: - dev: '"dev" in tags' -keyed_groups: -- key: cluster - prefix: cluster - separator: _ -- key: status - prefix: status - separator: _ -- key: tags - prefix: tag - separator: _ -ovirt_hostname_preference: -- name -- fqdn -ovirt_insecure: false -plugin: ovirt.ovirt.ovirt diff --git a/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml b/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml deleted file mode 100644 index fcad2586f6..0000000000 --- a/awx/main/tests/data/inventory/plugins/satellite6/files/foreman.yml +++ /dev/null @@ -1,30 +0,0 @@ -base_source_var: value_of_var -compose: - ansible_ssh_host: foreman['ip6'] | default(foreman['ip'], true) -group_prefix: foo_group_prefix -keyed_groups: -- key: foreman['environment_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_') | regex_replace('none', '') - prefix: foo_group_prefixenvironment_ - separator: '' -- key: foreman['location_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_') - prefix: foo_group_prefixlocation_ - separator: '' -- key: foreman['organization_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_') - prefix: foo_group_prefixorganization_ - separator: '' -- key: foreman['content_facet_attributes']['lifecycle_environment_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_') - prefix: foo_group_prefixlifecycle_environment_ - separator: '' -- key: foreman['content_facet_attributes']['content_view_name'] | lower | regex_replace(' ', '') | regex_replace('[^A-Za-z0-9_]', '_') - prefix: foo_group_prefixcontent_view_ - separator: '' -- key: '"%s-%s-%s" | format(app, tier, color)' - separator: '' -- key: '"%s-%s" | format(app, color)' - separator: '' -legacy_hostvars: true -plugin: theforeman.foreman.foreman -validate_certs: false -want_facts: true -want_hostcollections: true -want_params: true diff --git a/awx/main/tests/data/inventory/plugins/tower/files/tower.yml b/awx/main/tests/data/inventory/plugins/tower/files/tower.yml deleted file mode 100644 index 2c41f1b55d..0000000000 --- a/awx/main/tests/data/inventory/plugins/tower/files/tower.yml +++ /dev/null @@ -1,3 +0,0 @@ -include_metadata: true -inventory_id: 42 -plugin: awx.awx.tower diff --git a/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml b/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml deleted file mode 100644 index ac1db9f4cf..0000000000 --- a/awx/main/tests/data/inventory/plugins/vmware/files/vmware_vm_inventory.yml +++ /dev/null @@ -1,55 +0,0 @@ -compose: - ansible_host: guest.ipAddress - ansible_ssh_host: guest.ipAddress - ansible_uuid: 99999999 | random | to_uuid - availablefield: availableField - configissue: configIssue - configstatus: configStatus - customvalue: customValue - effectiverole: effectiveRole - guestheartbeatstatus: guestHeartbeatStatus - layoutex: layoutEx - overallstatus: overallStatus - parentvapp: parentVApp - recenttask: recentTask - resourcepool: resourcePool - rootsnapshot: rootSnapshot - triggeredalarmstate: triggeredAlarmState -filters: -- config.zoo == "DC0_H0_VM0" -hostnames: -- config.foo -keyed_groups: -- key: config.asdf - prefix: '' - separator: '' -plugin: community.vmware.vmware_vm_inventory -properties: -- availableField -- configIssue -- configStatus -- customValue -- datastore -- effectiveRole -- guestHeartbeatStatus -- layout -- layoutEx -- name -- network -- overallStatus -- parentVApp -- permission -- recentTask -- resourcePool -- rootSnapshot -- snapshot -- triggeredAlarmState -- value -- capability -- config -- guest -- runtime -- storage -- summary -strict: false -with_nested_properties: true diff --git a/awx/main/tests/functional/api/test_inventory.py b/awx/main/tests/functional/api/test_inventory.py index e63286f7e8..5bad1b6f30 100644 --- a/awx/main/tests/functional/api/test_inventory.py +++ b/awx/main/tests/functional/api/test_inventory.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import pytest +import json from unittest import mock from django.core.exceptions import ValidationError @@ -8,8 +9,6 @@ from awx.api.versioning import reverse from awx.main.models import InventorySource, Inventory, ActivityStream -import json - @pytest.fixture def scm_inventory(inventory, project): @@ -522,7 +521,8 @@ class TestInventorySourceCredential: data={ 'inventory': inventory.pk, 'name': 'fobar', 'source': 'scm', 'source_project': project.pk, 'source_path': '', - 'credential': vault_credential.pk + 'credential': vault_credential.pk, + 'source_vars': 'plugin: a.b.c', }, expect=400, user=admin_user @@ -561,7 +561,7 @@ class TestInventorySourceCredential: data={ 'inventory': inventory.pk, 'name': 'fobar', 'source': 'scm', 'source_project': project.pk, 'source_path': '', - 'credential': os_cred.pk + 'credential': os_cred.pk, 'source_vars': 'plugin: a.b.c', }, expect=201, user=admin_user @@ -636,8 +636,14 @@ class TestControlledBySCM: assert scm_inventory.inventory_sources.count() == 0 def test_adding_inv_src_ok(self, post, scm_inventory, project, admin_user): - post(reverse('api:inventory_inventory_sources_list', kwargs={'pk': scm_inventory.id}), - {'name': 'new inv src', 'source_project': project.pk, 'update_on_project_update': False, 'source': 'scm', 'overwrite_vars': True}, + post(reverse('api:inventory_inventory_sources_list', + kwargs={'pk': scm_inventory.id}), + {'name': 'new inv src', + 'source_project': project.pk, + 'update_on_project_update': False, + 'source': 'scm', + 'overwrite_vars': True, + 'source_vars': 'plugin: a.b.c'}, admin_user, expect=201) def test_adding_inv_src_prohibited(self, post, scm_inventory, project, admin_user): @@ -657,7 +663,7 @@ class TestControlledBySCM: def test_adding_inv_src_without_proj_access_prohibited(self, post, project, inventory, rando): inventory.admin_role.members.add(rando) post(reverse('api:inventory_inventory_sources_list', kwargs={'pk': inventory.id}), - {'name': 'new inv src', 'source_project': project.pk, 'source': 'scm', 'overwrite_vars': True}, + {'name': 'new inv src', 'source_project': project.pk, 'source': 'scm', 'overwrite_vars': True, 'source_vars': 'plugin: a.b.c'}, rando, expect=403) diff --git a/awx/main/tests/functional/models/test_inventory.py b/awx/main/tests/functional/models/test_inventory.py index 6765f0e73b..04b92d5a1d 100644 --- a/awx/main/tests/functional/models/test_inventory.py +++ b/awx/main/tests/functional/models/test_inventory.py @@ -2,7 +2,6 @@ import pytest from unittest import mock -import json from django.core.exceptions import ValidationError @@ -256,33 +255,22 @@ class TestInventorySourceInjectors: are named correctly, because Ansible will reject files that do not have these exact names """ - injector = InventorySource.injectors[source]('2.7.7') + injector = InventorySource.injectors[source]() assert injector.filename == filename - def test_group_by_azure(self): - injector = InventorySource.injectors['azure_rm']('2.9') - inv_src = InventorySource( - name='azure source', source='azure_rm', - source_vars={'group_by_os_family': True} - ) - group_by_on = injector.inventory_as_dict(inv_src, '/tmp/foo') - # suspicious, yes, that is just what the script did - expected_groups = 6 - assert len(group_by_on['keyed_groups']) == expected_groups - inv_src.source_vars = json.dumps({'group_by_os_family': False}) - group_by_off = injector.inventory_as_dict(inv_src, '/tmp/foo') - # much better, everyone should turn off the flag and live in the future - assert len(group_by_off['keyed_groups']) == expected_groups - 1 - - def test_tower_plugin_named_url(self): - injector = InventorySource.injectors['tower']('2.9') - inv_src = InventorySource( - name='my tower source', source='tower', - # named URL pattern "inventory++organization" - instance_filters='Designer hair 읰++Cosmetic_products䵆' - ) - result = injector.inventory_as_dict(inv_src, '/tmp/foo') - assert result['inventory_id'] == 'Designer%20hair%20%EC%9D%B0++Cosmetic_products%E4%B5%86' + @pytest.mark.parametrize('source,proper_name', [ + ('ec2', 'amazon.aws.aws_ec2'), + ('openstack', 'openstack.cloud.openstack'), + ('gce', 'google.cloud.gcp_compute'), + ('azure_rm', 'azure.azcollection.azure_rm'), + ('vmware', 'community.vmware.vmware_vm_inventory'), + ('rhv', 'ovirt.ovirt.ovirt'), + ('satellite6', 'theforeman.foreman.foreman'), + ('tower', 'awx.awx.tower'), + ]) + def test_plugin_proper_names(self, source, proper_name): + injector = InventorySource.injectors[source]() + assert injector.get_proper_name() == proper_name @pytest.mark.django_db diff --git a/awx/main/tests/functional/test_inventory_source_injectors.py b/awx/main/tests/functional/test_inventory_source_injectors.py index 392abf8535..4601668d25 100644 --- a/awx/main/tests/functional/test_inventory_source_injectors.py +++ b/awx/main/tests/functional/test_inventory_source_injectors.py @@ -14,69 +14,6 @@ from django.conf import settings DATA = os.path.join(os.path.dirname(data.__file__), 'inventory') -TEST_SOURCE_FIELDS = { - 'vmware': { - 'instance_filters': '{{ config.name == "only_my_server" }},{{ somevar == "bar"}}', - 'group_by': 'fouo' - }, - 'ec2': { - 'instance_filters': 'foobaa', - # group_by selected to capture some non-trivial cross-interactions - 'group_by': 'availability_zone,instance_type,tag_keys,region', - 'source_regions': 'us-east-2,ap-south-1' - }, - 'gce': { - 'source_regions': 'us-east4-a,us-west1-b' # surfaced as env var - }, - 'azure_rm': { - 'source_regions': 'southcentralus,westus' - }, - 'tower': { - 'instance_filters': '42' - } -} - -INI_TEST_VARS = { - 'ec2': { - 'boto_profile': '/tmp/my_boto_stuff', - 'iam_role_arn': 'arn:aws:iam::123456789012:role/test-role', - 'hostname_variable': 'public_dns_name', - 'destination_variable': 'public_dns_name' - }, - 'gce': {}, - 'openstack': { - 'private': False, - 'use_hostnames': False, - 'expand_hostvars': True, - 'fail_on_errors': True - }, - 'tower': {}, # there are none - 'vmware': { - 'alias_pattern': "{{ config.foo }}", - 'host_filters': '{{ config.zoo == "DC0_H0_VM0" }}', - 'groupby_patterns': "{{ config.asdf }}", - # setting VMWARE_VALIDATE_CERTS is duplicated with env var - }, - 'azure_rm': { - 'use_private_ip': True, - 'resource_groups': 'foo_resources,bar_resources', - 'tags': 'Creator:jmarshall, peanutbutter:jelly' - }, - 'satellite6': { - 'satellite6_group_patterns': '["{app}-{tier}-{color}", "{app}-{color}"]', - 'satellite6_group_prefix': 'foo_group_prefix', - 'satellite6_want_hostcollections': True, - 'satellite6_want_ansible_ssh_host': True, - 'satellite6_want_facts': True - }, - 'rhv': { # options specific to the plugin - 'ovirt_insecure': False, - 'groups': { - 'dev': '"dev" in tags' - } - } -} - def generate_fake_var(element): """Given a credential type field element, makes up something acceptable. @@ -245,25 +182,21 @@ def create_reference_data(source_dir, env, content): @pytest.mark.django_db @pytest.mark.parametrize('this_kind', CLOUD_PROVIDERS) def test_inventory_update_injected_content(this_kind, inventory, fake_credential_factory): + injector = InventorySource.injectors[this_kind] + if injector.plugin_name is None: + pytest.skip('Use of inventory plugin is not enabled for this source') + src_vars = dict(base_source_var='value_of_var') - if this_kind in INI_TEST_VARS: - src_vars.update(INI_TEST_VARS[this_kind]) - extra_kwargs = {} - if this_kind in TEST_SOURCE_FIELDS: - extra_kwargs.update(TEST_SOURCE_FIELDS[this_kind]) + src_vars['plugin'] = injector.get_proper_name() inventory_source = InventorySource.objects.create( inventory=inventory, source=this_kind, source_vars=src_vars, - **extra_kwargs ) inventory_source.credentials.add(fake_credential_factory(this_kind)) inventory_update = inventory_source.create_unified_job() task = RunInventoryUpdate() - if InventorySource.injectors[this_kind].plugin_name is None: - pytest.skip('Use of inventory plugin is not enabled for this source') - def substitute_run(envvars=None, **_kw): """This method will replace run_pexpect instead of running, it will read the private data directory contents @@ -274,6 +207,12 @@ def test_inventory_update_injected_content(this_kind, inventory, fake_credential assert envvars.pop('ANSIBLE_INVENTORY_ENABLED') == 'auto' set_files = bool(os.getenv("MAKE_INVENTORY_REFERENCE_FILES", 'false').lower()[0] not in ['f', '0']) env, content = read_content(private_data_dir, envvars, inventory_update) + + # Assert inventory plugin inventory file is in private_data_dir + inventory_filename = InventorySource.injectors[inventory_update.source]().filename + assert len([True for k in content.keys() if k.endswith(inventory_filename)]) > 0, \ + f"'{inventory_filename}' file not found in inventory update runtime files {content.keys()}" + env.pop('ANSIBLE_COLLECTIONS_PATHS', None) # collection paths not relevant to this test base_dir = os.path.join(DATA, 'plugins') if not os.path.exists(base_dir): @@ -283,6 +222,8 @@ def test_inventory_update_injected_content(this_kind, inventory, fake_credential create_reference_data(source_dir, env, content) pytest.skip('You set MAKE_INVENTORY_REFERENCE_FILES, so this created files, unset to run actual test.') else: + source_dir = os.path.join(base_dir, this_kind) # this_kind is a global + if not os.path.exists(source_dir): raise FileNotFoundError( 'Maybe you never made reference files? ' @@ -292,9 +233,6 @@ def test_inventory_update_injected_content(this_kind, inventory, fake_credential expected_file_list = os.listdir(files_dir) except FileNotFoundError: expected_file_list = [] - assert set(expected_file_list) == set(content.keys()), ( - 'Inventory update runtime environment does not have expected files' - ) for f_name in expected_file_list: with open(os.path.join(files_dir, f_name), 'r') as f: ref_content = f.read() @@ -314,8 +252,7 @@ def test_inventory_update_injected_content(this_kind, inventory, fake_credential with mock.patch('awx.main.queue.CallbackQueueDispatcher.dispatch', lambda self, obj: None): # Also do not send websocket status updates with mock.patch.object(UnifiedJob, 'websocket_emit_status', mock.Mock()): - with mock.patch.object(task, 'get_ansible_version', return_value='2.13'): - # The point of this test is that we replace run with assertions - with mock.patch('awx.main.tasks.ansible_runner.interface.run', substitute_run): - # so this sets up everything for a run and then yields control over to substitute_run - task.run(inventory_update.pk) + # The point of this test is that we replace run with assertions + with mock.patch('awx.main.tasks.ansible_runner.interface.run', substitute_run): + # so this sets up everything for a run and then yields control over to substitute_run + task.run(inventory_update.pk) diff --git a/awx/main/tests/unit/models/test_inventory.py b/awx/main/tests/unit/models/test_inventory.py index dc6af0e828..26ef5e1fa9 100644 --- a/awx/main/tests/unit/models/test_inventory.py +++ b/awx/main/tests/unit/models/test_inventory.py @@ -72,23 +72,6 @@ def test_invalid_kind_clean_insights_credential(): assert json.dumps(str(e.value)) == json.dumps(str([u'Assignment not allowed for Smart Inventory'])) -@pytest.mark.parametrize('source_vars,validate_certs', [ - ({'ssl_verify': True}, True), - ({'ssl_verify': False}, False), - ({'validate_certs': True}, True), - ({'validate_certs': False}, False)]) -def test_satellite_plugin_backwards_support_for_ssl_verify(source_vars, validate_certs): - injector = InventorySource.injectors['satellite6']('2.9') - inv_src = InventorySource( - name='satellite source', source='satellite6', - source_vars=source_vars - ) - - ret = injector.inventory_as_dict(inv_src, '/tmp/foo') - assert 'validate_certs' in ret - assert ret['validate_certs'] in (validate_certs, str(validate_certs)) - - class TestControlledBySCM(): def test_clean_source_path_valid(self): inv_src = InventorySource(source_path='/not_real/', diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index 7dcea33ccd..5ce64894e6 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -1880,13 +1880,6 @@ class TestProjectUpdateCredentials(TestJobExecution): assert env['FOO'] == 'BAR' -@pytest.fixture -def mock_ansible_version(): - with mock.patch('awx.main.tasks._get_ansible_version', mock.MagicMock(return_value='2.10')) as _fixture: - yield _fixture - - -@pytest.mark.usefixtures("mock_ansible_version") class TestInventoryUpdateCredentials(TestJobExecution): @pytest.fixture def inventory_update(self): @@ -2020,7 +2013,6 @@ class TestInventoryUpdateCredentials(TestJobExecution): task = tasks.RunInventoryUpdate() azure_rm = CredentialType.defaults['azure_rm']() inventory_update.source = 'azure_rm' - inventory_update.source_regions = 'north, south, east, west' def get_cred(): cred = Credential( @@ -2059,7 +2051,6 @@ class TestInventoryUpdateCredentials(TestJobExecution): task = tasks.RunInventoryUpdate() azure_rm = CredentialType.defaults['azure_rm']() inventory_update.source = 'azure_rm' - inventory_update.source_regions = 'all' def get_cred(): cred = Credential( @@ -2097,7 +2088,6 @@ class TestInventoryUpdateCredentials(TestJobExecution): task = tasks.RunInventoryUpdate() gce = CredentialType.defaults['gce']() inventory_update.source = 'gce' - inventory_update.source_regions = 'all' def get_cred(): cred = Credential( @@ -2216,7 +2206,6 @@ class TestInventoryUpdateCredentials(TestJobExecution): task = tasks.RunInventoryUpdate() tower = CredentialType.defaults['tower']() inventory_update.source = 'tower' - inventory_update.instance_filters = '12345' inputs = { 'host': 'https://tower.example.org', 'username': 'bob', @@ -2248,7 +2237,6 @@ class TestInventoryUpdateCredentials(TestJobExecution): task = tasks.RunInventoryUpdate() tower = CredentialType.defaults['tower']() inventory_update.source = 'tower' - inventory_update.instance_filters = '12345' inputs = { 'host': 'https://tower.example.org', 'username': 'bob', diff --git a/awx/main/tests/unit/utils/test_common.py b/awx/main/tests/unit/utils/test_common.py index edacfc1423..8c07020c53 100644 --- a/awx/main/tests/unit/utils/test_common.py +++ b/awx/main/tests/unit/utils/test_common.py @@ -215,11 +215,3 @@ def test_get_custom_venv_choices(): os.path.join(temp_dir, ''), os.path.join(custom_venv_1, '') ] - - -def test_region_sorting(): - s = [('Huey', 'China1'), - ('Dewey', 'UK1'), - ('Lewie', 'US1'), - ('All', 'All')] - assert [x[1] for x in sorted(s, key=common.region_sorting)] == ['All', 'US1', 'China1', 'UK1'] diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index f34cf4e4d8..a65120c8e8 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -45,7 +45,7 @@ __all__ = [ 'get_object_or_400', 'camelcase_to_underscore', 'underscore_to_camelcase', 'memoize', 'memoize_delete', 'get_ansible_version', 'get_licenser', 'get_awx_http_client_headers', 'get_awx_version', 'update_scm_url', 'get_type_for_model', 'get_model_for_type', - 'copy_model_by_class', 'region_sorting', 'copy_m2m_relationships', + 'copy_model_by_class', 'copy_m2m_relationships', 'prefetch_page_capabilities', 'to_python_boolean', 'ignore_inventory_computed_fields', 'ignore_inventory_group_removal', '_inventory_updates', 'get_pk_from_dict', 'getattrd', 'getattr_dne', 'NoDefaultProvided', 'get_current_apps', 'set_current_apps', @@ -87,15 +87,6 @@ def to_python_boolean(value, allow_none=False): raise ValueError(_(u'Unable to convert "%s" to boolean') % value) -def region_sorting(region): - # python3's removal of sorted(cmp=...) is _stupid_ - if region[1].lower() == 'all': - return '' - elif region[1].lower().startswith('us'): - return region[1] - return 'ZZZ' + str(region[1]) - - def camelcase_to_underscore(s): ''' Convert CamelCase names to lowercase_with_underscore. @@ -171,13 +162,14 @@ def memoize_delete(function_name): return cache.delete(function_name) -def _get_ansible_version(ansible_path): +@memoize() +def get_ansible_version(): ''' Return Ansible version installed. Ansible path needs to be provided to account for custom virtual environments ''' try: - proc = subprocess.Popen([ansible_path, '--version'], + proc = subprocess.Popen(['ansible', '--version'], stdout=subprocess.PIPE) result = smart_str(proc.communicate()[0]) return result.split('\n')[0].replace('ansible', '').strip() @@ -185,11 +177,6 @@ def _get_ansible_version(ansible_path): return 'unknown' -@memoize() -def get_ansible_version(): - return _get_ansible_version('ansible') - - def get_awx_version(): ''' Return AWX version as reported by setuptools. diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 716aea3aa7..5aa0b834ea 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -8,8 +8,6 @@ from datetime import timedelta # global settings from django.conf import global_settings -# ugettext lazy -from django.utils.translation import ugettext_lazy as _ # Update this module's local settings from the global settings module. this_module = sys.modules[__name__] @@ -671,145 +669,32 @@ INV_ENV_VARIABLE_BLOCKED = ("HOME", "USER", "_", "TERM") # ---------------- # -- Amazon EC2 -- # ---------------- - -# AWS does not appear to provide pretty region names via any API, so store the -# list of names here. The available region IDs will be pulled from boto. -# http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region -EC2_REGION_NAMES = { - 'us-east-1': _('US East (Northern Virginia)'), - 'us-east-2': _('US East (Ohio)'), - 'us-west-2': _('US West (Oregon)'), - 'us-west-1': _('US West (Northern California)'), - 'ca-central-1': _('Canada (Central)'), - 'eu-central-1': _('EU (Frankfurt)'), - 'eu-west-1': _('EU (Ireland)'), - 'eu-west-2': _('EU (London)'), - 'ap-southeast-1': _('Asia Pacific (Singapore)'), - 'ap-southeast-2': _('Asia Pacific (Sydney)'), - 'ap-northeast-1': _('Asia Pacific (Tokyo)'), - 'ap-northeast-2': _('Asia Pacific (Seoul)'), - 'ap-south-1': _('Asia Pacific (Mumbai)'), - 'sa-east-1': _('South America (Sao Paulo)'), - 'us-gov-west-1': _('US West (GovCloud)'), - 'cn-north-1': _('China (Beijing)'), -} - -# Inventory variable name/values for determining if host is active/enabled. EC2_ENABLED_VAR = 'ec2_state' EC2_ENABLED_VALUE = 'running' - -# Inventory variable name containing unique instance ID. EC2_INSTANCE_ID_VAR = 'ec2_id' - -# Filter for allowed group/host names when importing inventory from EC2. -EC2_GROUP_FILTER = r'^.+$' -EC2_HOST_FILTER = r'^.+$' EC2_EXCLUDE_EMPTY_GROUPS = True - # ------------ # -- VMware -- # ------------ -# Inventory variable name/values for determining whether a host is -# active in vSphere. VMWARE_ENABLED_VAR = 'guest.gueststate' VMWARE_ENABLED_VALUE = 'running' - -# Inventory variable name containing the unique instance ID. VMWARE_INSTANCE_ID_VAR = 'config.instanceUuid, config.instanceuuid' - -# Filter for allowed group and host names when importing inventory -# from VMware. -VMWARE_GROUP_FILTER = r'^.+$' -VMWARE_HOST_FILTER = r'^.+$' VMWARE_EXCLUDE_EMPTY_GROUPS = True VMWARE_VALIDATE_CERTS = False + # --------------------------- # -- Google Compute Engine -- # --------------------------- - -# It's not possible to get zones in GCE without authenticating, so we -# provide a list here. -# Source: https://developers.google.com/compute/docs/zones -GCE_REGION_CHOICES = [ - ('us-east1-b', _('US East 1 (B)')), - ('us-east1-c', _('US East 1 (C)')), - ('us-east1-d', _('US East 1 (D)')), - ('us-east4-a', _('US East 4 (A)')), - ('us-east4-b', _('US East 4 (B)')), - ('us-east4-c', _('US East 4 (C)')), - ('us-central1-a', _('US Central (A)')), - ('us-central1-b', _('US Central (B)')), - ('us-central1-c', _('US Central (C)')), - ('us-central1-f', _('US Central (F)')), - ('us-west1-a', _('US West (A)')), - ('us-west1-b', _('US West (B)')), - ('us-west1-c', _('US West (C)')), - ('europe-west1-b', _('Europe West 1 (B)')), - ('europe-west1-c', _('Europe West 1 (C)')), - ('europe-west1-d', _('Europe West 1 (D)')), - ('europe-west2-a', _('Europe West 2 (A)')), - ('europe-west2-b', _('Europe West 2 (B)')), - ('europe-west2-c', _('Europe West 2 (C)')), - ('asia-east1-a', _('Asia East (A)')), - ('asia-east1-b', _('Asia East (B)')), - ('asia-east1-c', _('Asia East (C)')), - ('asia-southeast1-a', _('Asia Southeast (A)')), - ('asia-southeast1-b', _('Asia Southeast (B)')), - ('asia-northeast1-a', _('Asia Northeast (A)')), - ('asia-northeast1-b', _('Asia Northeast (B)')), - ('asia-northeast1-c', _('Asia Northeast (C)')), - ('australia-southeast1-a', _('Australia Southeast (A)')), - ('australia-southeast1-b', _('Australia Southeast (B)')), - ('australia-southeast1-c', _('Australia Southeast (C)')), -] -# Inventory variable name/value for determining whether a host is active -# in Google Compute Engine. GCE_ENABLED_VAR = 'status' GCE_ENABLED_VALUE = 'running' - -# Filter for allowed group and host names when importing inventory from -# Google Compute Engine. -GCE_GROUP_FILTER = r'^.+$' -GCE_HOST_FILTER = r'^.+$' GCE_EXCLUDE_EMPTY_GROUPS = True GCE_INSTANCE_ID_VAR = 'gce_id' # -------------------------------------- # -- Microsoft Azure Resource Manager -- # -------------------------------------- -# It's not possible to get zones in Azure without authenticating, so we -# provide a list here. -AZURE_RM_REGION_CHOICES = [ - ('eastus', _('US East')), - ('eastus2', _('US East 2')), - ('centralus', _('US Central')), - ('northcentralus', _('US North Central')), - ('southcentralus', _('US South Central')), - ('westcentralus', _('US West Central')), - ('westus', _('US West')), - ('westus2', _('US West 2')), - ('canadaeast', _('Canada East')), - ('canadacentral', _('Canada Central')), - ('brazilsouth', _('Brazil South')), - ('northeurope', _('Europe North')), - ('westeurope', _('Europe West')), - ('ukwest', _('UK West')), - ('uksouth', _('UK South')), - ('eastasia', _('Asia East')), - ('southestasia', _('Asia Southeast')), - ('australiaeast', _('Australia East')), - ('australiasoutheast', _('Australia Southeast')), - ('westindia', _('India West')), - ('southindia', _('India South')), - ('japaneast', _('Japan East')), - ('japanwest', _('Japan West')), - ('koreacentral', _('Korea Central')), - ('koreasouth', _('Korea South')), -] -AZURE_RM_GROUP_FILTER = r'^.+$' -AZURE_RM_HOST_FILTER = r'^.+$' AZURE_RM_ENABLED_VAR = 'powerstate' AZURE_RM_ENABLED_VALUE = 'running' AZURE_RM_INSTANCE_ID_VAR = 'id' @@ -820,8 +705,6 @@ AZURE_RM_EXCLUDE_EMPTY_GROUPS = True # --------------------- OPENSTACK_ENABLED_VAR = 'status' OPENSTACK_ENABLED_VALUE = 'ACTIVE' -OPENSTACK_GROUP_FILTER = r'^.+$' -OPENSTACK_HOST_FILTER = r'^.+$' OPENSTACK_EXCLUDE_EMPTY_GROUPS = True OPENSTACK_INSTANCE_ID_VAR = 'openstack.id' @@ -830,8 +713,6 @@ OPENSTACK_INSTANCE_ID_VAR = 'openstack.id' # --------------------- RHV_ENABLED_VAR = 'status' RHV_ENABLED_VALUE = 'up' -RHV_GROUP_FILTER = r'^.+$' -RHV_HOST_FILTER = r'^.+$' RHV_EXCLUDE_EMPTY_GROUPS = True RHV_INSTANCE_ID_VAR = 'id' @@ -840,8 +721,6 @@ RHV_INSTANCE_ID_VAR = 'id' # --------------------- TOWER_ENABLED_VAR = 'remote_tower_enabled' TOWER_ENABLED_VALUE = 'true' -TOWER_GROUP_FILTER = r'^.+$' -TOWER_HOST_FILTER = r'^.+$' TOWER_EXCLUDE_EMPTY_GROUPS = True TOWER_INSTANCE_ID_VAR = 'remote_tower_id' @@ -850,8 +729,6 @@ TOWER_INSTANCE_ID_VAR = 'remote_tower_id' # --------------------- SATELLITE6_ENABLED_VAR = 'foreman.enabled' SATELLITE6_ENABLED_VALUE = 'True' -SATELLITE6_GROUP_FILTER = r'^.+$' -SATELLITE6_HOST_FILTER = r'^.+$' SATELLITE6_EXCLUDE_EMPTY_GROUPS = True SATELLITE6_INSTANCE_ID_VAR = 'foreman.id' # SATELLITE6_GROUP_PREFIX and SATELLITE6_GROUP_PATTERNS defined in source vars @@ -861,8 +738,6 @@ SATELLITE6_INSTANCE_ID_VAR = 'foreman.id' # --------------------- #CUSTOM_ENABLED_VAR = #CUSTOM_ENABLED_VALUE = -CUSTOM_GROUP_FILTER = r'^.+$' -CUSTOM_HOST_FILTER = r'^.+$' CUSTOM_EXCLUDE_EMPTY_GROUPS = False #CUSTOM_INSTANCE_ID_VAR = @@ -871,8 +746,6 @@ CUSTOM_EXCLUDE_EMPTY_GROUPS = False # --------------------- #SCM_ENABLED_VAR = #SCM_ENABLED_VALUE = -SCM_GROUP_FILTER = r'^.+$' -SCM_HOST_FILTER = r'^.+$' SCM_EXCLUDE_EMPTY_GROUPS = False #SCM_INSTANCE_ID_VAR = diff --git a/awx/ui/client/src/inventories-hosts/inventories/related/sources/add/sources-add.controller.js b/awx/ui/client/src/inventories-hosts/inventories/related/sources/add/sources-add.controller.js index 9e254fd7e5..05bfe07718 100644 --- a/awx/ui/client/src/inventories-hosts/inventories/related/sources/add/sources-add.controller.js +++ b/awx/ui/client/src/inventories-hosts/inventories/related/sources/add/sources-add.controller.js @@ -24,49 +24,6 @@ export default ['$state', 'ConfigData', '$scope', 'SourcesFormDefinition', 'Pars const virtualEnvs = ConfigData.custom_virtualenvs || []; $scope.custom_virtualenvs_options = virtualEnvs; - GetChoices({ - scope: $scope, - field: 'source_regions', - variable: 'rax_regions', - choice_name: 'rax_region_choices', - options: inventorySourcesOptions - }); - - GetChoices({ - scope: $scope, - field: 'source_regions', - variable: 'ec2_regions', - choice_name: 'ec2_region_choices', - options: inventorySourcesOptions - }); - - GetChoices({ - scope: $scope, - field: 'source_regions', - variable: 'gce_regions', - choice_name: 'gce_region_choices', - options: inventorySourcesOptions - }); - - GetChoices({ - scope: $scope, - field: 'source_regions', - variable: 'azure_regions', - choice_name: 'azure_rm_region_choices', - options: inventorySourcesOptions - }); - - // Load options for group_by - GetChoices({ - scope: $scope, - field: 'group_by', - variable: 'ec2_group_by', - choice_name: 'ec2_group_by_choices', - options: inventorySourcesOptions - }); - - initRegionSelect(); - GetChoices({ scope: $scope, field: 'verbosity', @@ -205,20 +162,11 @@ export default ['$state', 'ConfigData', '$scope', 'SourcesFormDefinition', 'Pars $scope.projectBasePath = GetBasePath('projects') + '?not__status=never updated'; } - // reset fields - $scope.group_by_choices = source === 'ec2' ? $scope.ec2_group_by : null; - // azure_rm regions choices are keyed as "azure" in an OPTIONS request to the inventory_sources endpoint - $scope.source_region_choices = source === 'azure_rm' ? $scope.azure_regions : $scope[source + '_regions']; $scope.cloudCredentialRequired = source !== '' && source !== 'scm' && source !== 'custom' && source !== 'ec2' ? true : false; - $scope.source_regions = null; $scope.credential = null; $scope.credential_name = null; - $scope.group_by = null; - $scope.group_by_choices = []; $scope.overwrite_vars = false; - initRegionSelect(); }; - // region / source options callback $scope.$on('sourceTypeOptionsReady', function() { CreateSelect2({ @@ -227,57 +175,6 @@ export default ['$state', 'ConfigData', '$scope', 'SourcesFormDefinition', 'Pars }); }); - function initRegionSelect(){ - CreateSelect2({ - element: '#inventory_source_source_regions', - multiple: true - }); - - let add_new = false; - if( _.get($scope, 'source') === 'ec2' || _.get($scope.source, 'value') === 'ec2') { - $scope.group_by_choices = $scope.ec2_group_by; - $scope.groupByPopOver = "
" + i18n._("Select which groups to create automatically. ") + - $rootScope.BRAND_NAME + i18n._(" will create group names similar to the following examples based on the options selected:") + "
" + i18n._("If blank, all groups above are created except") + "" + i18n._("Instance ID") + ".
"; - - $scope.instanceFilterPopOver = "" + i18n._("Provide a comma-separated list of filter expressions. ") + - i18n._("Hosts are imported to ") + $rootScope.BRAND_NAME + i18n._(" when ") + "" + i18n._("ANY") + "" + i18n._(" of the filters match.") + "
" + - i18n._("Limit to hosts having a tag:") + "tag-key=TowerManaged\n" + - i18n._("Limit to hosts using either key pair:") + "
key-name=staging, key-name=production\n" + - i18n._("Limit to hosts where the Name tag begins with ") + "" + i18n._("test") + ":
tag:Name=test*\n" + - "
" + i18n._("View the ") + "" + i18n._("Describe Instances documentation") + " " + - i18n._("for a complete list of supported filters.") + "
"; - } - if( _.get($scope, 'source') === 'vmware' || _.get($scope.source, 'value') === 'vmware') { - add_new = true; - $scope.group_by_choices = []; - $scope.group_by = $scope.group_by_choices; - $scope.groupByPopOver = i18n._("Specify which groups to create automatically. Group names will be created similar to the options selected. If blank, all groups above are created. Refer to Ansible Tower documentation for more detail."); - $scope.instanceFilterPopOver = i18n._("Provide a comma-separated list of filter expressions. Hosts are imported when all of the filters match. Refer to Ansible Tower documentation for more detail."); - } - if( _.get($scope, 'source') === 'tower' || _.get($scope.source, 'value') === 'tower') { - $scope.instanceFilterPopOver = i18n._("Provide the named URL encoded name or id of the remote Tower inventory to be imported."); - } - CreateSelect2({ - element: '#inventory_source_group_by', - multiple: true, - addNew: add_new - }); - } - $scope.formCancel = function() { $state.go('^'); }; @@ -289,7 +186,6 @@ export default ['$state', 'ConfigData', '$scope', 'SourcesFormDefinition', 'Pars name: $scope.name, description: $scope.description, inventory: inventoryData.id, - instance_filters: $scope.instance_filters, source_script: $scope.inventory_script, credential: $scope.credential, overwrite: $scope.overwrite, @@ -298,9 +194,9 @@ export default ['$state', 'ConfigData', '$scope', 'SourcesFormDefinition', 'Pars verbosity: $scope.verbosity.value, update_cache_timeout: $scope.update_cache_timeout || 0, custom_virtualenv: $scope.custom_virtualenv || null, - // comma-delimited strings - group_by: SourcesService.encodeGroupBy($scope.source, $scope.group_by), - source_regions: _.map($scope.source_regions, 'value').join(','), + enabled_var: $scope.enabled_var, + enabled_value: $scope.enabled_value, + host_filter: $scope.host_filter }; if ($scope.source) { diff --git a/awx/ui/client/src/inventories-hosts/inventories/related/sources/edit/sources-edit.controller.js b/awx/ui/client/src/inventories-hosts/inventories/related/sources/edit/sources-edit.controller.js index 40dc4fc970..6da60e6b3b 100644 --- a/awx/ui/client/src/inventories-hosts/inventories/related/sources/edit/sources-edit.controller.js +++ b/awx/ui/client/src/inventories-hosts/inventories/related/sources/edit/sources-edit.controller.js @@ -34,7 +34,6 @@ export default ['$state', '$scope', 'ParseVariableString', 'ParseTypeChange', {overwrite_vars: inventorySourceData.overwrite_vars}, {update_on_launch: inventorySourceData.update_on_launch}, {update_cache_timeout: inventorySourceData.update_cache_timeout}, - {instance_filters: inventorySourceData.instance_filters}, {inventory_script: inventorySourceData.source_script}, {verbosity: inventorySourceData.verbosity}); @@ -100,56 +99,6 @@ export default ['$state', '$scope', 'ParseVariableString', 'ParseTypeChange', scope: $scope, variable: 'source_type_options' }); - GetChoices({ - scope: $scope, - field: 'source_regions', - variable: 'rax_regions', - choice_name: 'rax_region_choices', - options: inventorySourcesOptions - }); - GetChoices({ - scope: $scope, - field: 'source_regions', - variable: 'ec2_regions', - choice_name: 'ec2_region_choices', - options: inventorySourcesOptions - }); - GetChoices({ - scope: $scope, - field: 'source_regions', - variable: 'gce_regions', - choice_name: 'gce_region_choices', - options: inventorySourcesOptions - }); - GetChoices({ - scope: $scope, - field: 'source_regions', - variable: 'azure_regions', - choice_name: 'azure_rm_region_choices', - options: inventorySourcesOptions - }); - GetChoices({ - scope: $scope, - field: 'group_by', - variable: 'ec2_group_by', - choice_name: 'ec2_group_by_choices', - options: inventorySourcesOptions - }); - - var source = $scope.source === 'azure_rm' ? 'azure' : $scope.source; - var regions = inventorySourceData.source_regions.split(','); - // azure_rm regions choices are keyed as "azure" in an OPTIONS request to the inventory_sources endpoint - $scope.source_region_choices = $scope[source + '_regions']; - - // the API stores azure regions as all-lowercase strings - but the azure regions received from OPTIONS are Snake_Cased - if (source === 'azure') { - $scope.source_regions = _.map(regions, (region) => _.find($scope[source + '_regions'], (o) => o.value.toLowerCase() === region)); - } - // all other regions are 1-1 - else { - $scope.source_regions = _.map(regions, (region) => _.find($scope[source + '_regions'], (o) => o.value === region)); - } - initRegionSelect(); GetChoices({ scope: $scope, @@ -236,63 +185,6 @@ export default ['$state', '$scope', 'ParseVariableString', 'ParseTypeChange', } } - function initRegionSelect() { - CreateSelect2({ - element: '#inventory_source_source_regions', - multiple: true - }); - - let add_new = false; - if( _.get($scope, 'source') === 'ec2' || _.get($scope.source, 'value') === 'ec2') { - $scope.group_by_choices = $scope.ec2_group_by; - let group_by = inventorySourceData.group_by.split(','); - $scope.group_by = _.map(group_by, (item) => _.find($scope.ec2_group_by, { value: item })); - - $scope.groupByPopOver = "" + i18n._("Select which groups to create automatically. ") + - $rootScope.BRAND_NAME + i18n._(" will create group names similar to the following examples based on the options selected:") + "
" + i18n._("If blank, all groups above are created except") + "" + i18n._("Instance ID") + ".
"; - - - $scope.instanceFilterPopOver = "" + i18n._("Provide a comma-separated list of filter expressions. ") + - i18n._("Hosts are imported to ") + $rootScope.BRAND_NAME + i18n._(" when ") + "" + i18n._("ANY") + "" + i18n._(" of the filters match.") + "
" + - i18n._("Limit to hosts having a tag:") + "tag-key=TowerManaged\n" + - i18n._("Limit to hosts using either key pair:") + "
key-name=staging, key-name=production\n" + - i18n._("Limit to hosts where the Name tag begins with ") + "" + i18n._("test") + ":
tag:Name=test*\n" + - "
" + i18n._("View the ") + "" + i18n._("Describe Instances documentation") + " " + - i18n._("for a complete list of supported filters.") + "
"; - - } - if( _.get($scope, 'source') === 'vmware' || _.get($scope.source, 'value') === 'vmware') { - add_new = true; - $scope.group_by_choices = (inventorySourceData.group_by) ? inventorySourceData.group_by.split(',') - .map((i) => ({name: i, label: i, value: i})) : []; - $scope.group_by = $scope.group_by_choices; - $scope.groupByPopOver = i18n._(`Specify which groups to create automatically. Group names will be created similar to the options selected. If blank, all groups above are created. Refer to Ansible Tower documentation for more detail.`); - $scope.instanceFilterPopOver = i18n._(`Provide a comma-separated list of filter expressions. Hosts are imported when all of the filters match. Refer to Ansible Tower documentation for more detail.`); - } - if( _.get($scope, 'source') === 'tower' || _.get($scope.source, 'value') === 'tower') { - $scope.instanceFilterPopOver = i18n._(`Provide the named URL encoded name or id of the remote Tower inventory to be imported.`); - } - CreateSelect2({ - element: '#inventory_source_group_by', - multiple: true, - addNew: add_new - }); - } - $scope.lookupProject = function(){ $state.go('.project', { project_search: { @@ -341,12 +233,13 @@ export default ['$state', '$scope', 'ParseVariableString', 'ParseTypeChange', $scope.formSave = function() { var params; + console.log($scope); + params = { id: inventorySourceData.id, name: $scope.name, description: $scope.description, inventory: inventoryData.id, - instance_filters: $scope.instance_filters, source_script: $scope.inventory_script, credential: $scope.credential, overwrite: $scope.overwrite, @@ -355,9 +248,9 @@ export default ['$state', '$scope', 'ParseVariableString', 'ParseTypeChange', update_cache_timeout: $scope.update_cache_timeout || 0, verbosity: $scope.verbosity.value, custom_virtualenv: $scope.custom_virtualenv || null, - // comma-delimited strings - group_by: SourcesService.encodeGroupBy($scope.source, $scope.group_by), - source_regions: _.map($scope.source_regions, 'value').join(',') + enabled_var: $scope.enabled_var, + enabled_value: $scope.enabled_value, + host_filter: $scope.host_filter }; if ($scope.source) { @@ -417,20 +310,10 @@ export default ['$state', '$scope', 'ParseVariableString', 'ParseTypeChange', }); } - // reset fields - $scope.group_by_choices = source === 'ec2' ? $scope.ec2_group_by : null; - // azure_rm regions choices are keyed as "azure" in an OPTIONS request to the inventory_sources endpoint - $scope.source_region_choices = source === 'azure_rm' ? $scope.azure_regions : $scope[source + '_regions']; $scope.cloudCredentialRequired = source !== '' && source !== 'scm' && source !== 'custom' && source !== 'ec2' ? true : false; - $scope.source_regions = null; $scope.credential = null; $scope.credential_name = null; - $scope.group_by = null; - $scope.group_by_choices = []; $scope.overwrite_vars = false; - - initRegionSelect(); - }; } ]; diff --git a/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js b/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js index 3c76dd2e61..66b6693d58 100644 --- a/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js +++ b/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.form.js @@ -126,46 +126,6 @@ export default ['NotificationsList', 'i18n', function(NotificationsList, i18n){ includeInventoryFileNotFoundError: true, subForm: 'sourceSubForm' }, - source_regions: { - label: i18n._('Regions'), - type: 'select', - ngOptions: 'source.label for source in source_region_choices track by source.value', - multiSelect: true, - ngShow: "source && (source.value == 'rax' || source.value == 'ec2' || source.value == 'gce' || source.value == 'azure_rm')", - dataTitle: i18n._('Source Regions'), - dataPlacement: 'right', - awPopOver: "" + i18n._("Click on the regions field to see a list of regions for your cloud provider. You can select multiple regions, or choose") + - "" + i18n._("All") + " " + i18n._("to include all regions. Only Hosts associated with the selected regions will be updated.") + "
", - dataContainer: 'body', - ngDisabled: '!(inventory_source_obj.summary_fields.user_capabilities.edit || canAdd)', - subForm: 'sourceSubForm' - }, - instance_filters: { - label: i18n._("Instance Filters"), - type: 'text', - ngShow: "source && (source.value == 'ec2' || source.value == 'vmware' || source.value == 'tower')", - dataTitle: i18n._('Instance Filters'), - dataPlacement: 'right', - awPopOverWatch: 'instanceFilterPopOver', - awPopOver: '{{ instanceFilterPopOver }}', - dataContainer: 'body', - ngDisabled: '!(inventory_source_obj.summary_fields.user_capabilities.edit || canAdd)', - subForm: 'sourceSubForm' - }, - group_by: { - label: i18n._('Only Group By'), - type: 'select', - ngShow: "source && (source.value == 'ec2' || source.value == 'vmware')", - ngOptions: 'source.label for source in group_by_choices track by source.value', - multiSelect: true, - dataTitle: i18n._("Only Group By"), - dataPlacement: 'right', - awPopOverWatch: 'groupByPopOver', - awPopOver: '{{ groupByPopOver }}', - dataContainer: 'body', - ngDisabled: '!(inventory_source_obj.summary_fields.user_capabilities.edit || canAdd)', - subForm: 'sourceSubForm' - }, inventory_script: { label : i18n._("Custom Inventory Script"), type: 'lookup', @@ -340,6 +300,36 @@ export default ['NotificationsList', 'i18n', function(NotificationsList, i18n){ ngDisabled: '!(inventory_source_obj.summary_fields.user_capabilities.edit || canAdd)', subForm: 'sourceSubForm' }, + host_filter: { + label: i18n._("Host Filter"), + type: 'text', + dataTitle: i18n._('Host Filter'), + dataPlacement: 'right', + awPopOver: "" + i18n._("Regular expression where only matching host names will be imported. The filter is applied as a post-processing step after any inventory plugin filters are applied.") + "
", + dataContainer: 'body', + ngDisabled: '!(inventory_source_obj.summary_fields.user_capabilities.edit || canAdd)', + subForm: 'sourceSubForm' + }, + enabled_var: { + label: i18n._("Enabled Variable"), + type: 'text', + dataTitle: i18n._('Enabled Variable'), + dataPlacement: 'right', + awPopOver: "" + i18n._("Retrieve the enabled state from the given dict of host variables. The enabled variable may be specified using dot notation, e.g: 'foo.bar'") + "
", + dataContainer: 'body', + ngDisabled: '!(inventory_source_obj.summary_fields.user_capabilities.edit || canAdd)', + subForm: 'sourceSubForm' + }, + enabled_value: { + label: i18n._("Enabled Value"), + type: 'text', + dataTitle: i18n._('Enabled Value'), + dataPlacement: 'right', + awPopOver: "" + i18n._("This field is ignored unless an Enabled Variable is set. If the enabled variable matches this value, the host will be enabled on import.") + "
", + dataContainer: 'body', + ngDisabled: '!(inventory_source_obj.summary_fields.user_capabilities.edit || canAdd)', + subForm: 'sourceSubForm' + }, checkbox_group: { label: i18n._('Update Options'), type: 'checkbox_group', diff --git a/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.service.js b/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.service.js index ce65a7dae5..a42ac51097 100644 --- a/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.service.js +++ b/awx/ui/client/src/inventories-hosts/inventories/related/sources/sources.service.js @@ -116,24 +116,6 @@ export default .catch(this.error.bind(this)) .finally(Wait('stop')); }, - encodeGroupBy(source, group_by){ - source = source && source.value ? source.value : ''; - if(source === 'ec2'){ - return _.map(group_by, 'value').join(','); - } - - if(source === 'vmware'){ - group_by = _.map(group_by, (i) => {return i.value;}); - $("#inventory_source_group_by").siblings(".select2").first().find(".select2-selection__choice").each(function(optionIndex, option){ - group_by.push(option.title); - }); - group_by = (Array.isArray(group_by)) ? _.uniq(group_by).join() : ""; - return group_by; - } - else { - return; - } - }, deleteHosts(id) { this.url = GetBasePath('inventory_sources') + id + '/hosts/'; Rest.setUrl(this.url); diff --git a/awx/ui_next/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.jsx b/awx/ui_next/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.jsx index 42fad28421..a67c3c2064 100644 --- a/awx/ui_next/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.jsx +++ b/awx/ui_next/src/screens/Inventory/InventorySourceDetail/InventorySourceDetail.jsx @@ -3,10 +3,9 @@ import { Link, useHistory } from 'react-router-dom'; import { withI18n } from '@lingui/react'; import { t } from '@lingui/macro'; -import { Button, Chip, List, ListItem } from '@patternfly/react-core'; +import { Button, List, ListItem } from '@patternfly/react-core'; import AlertModal from '../../../components/AlertModal'; import { CardBody, CardActionsRow } from '../../../components/Card'; -import ChipGroup from '../../../components/ChipGroup'; import { VariablesDetail } from '../../../components/CodeMirrorInput'; import ContentError from '../../../components/ContentError'; import ContentLoading from '../../../components/ContentLoading'; @@ -28,21 +27,21 @@ function InventorySourceDetail({ inventorySource, i18n }) { created, custom_virtualenv, description, - group_by, id, - instance_filters, modified, name, overwrite, overwrite_vars, source, source_path, - source_regions, source_vars, update_cache_timeout, update_on_launch, update_on_project_update, verbosity, + enabled_var, + enabled_value, + host_filter, summary_fields: { created_by, credentials, @@ -224,6 +223,9 @@ function InventorySourceDetail({ inventorySource, i18n }) { label={i18n._(t`Cache timeout`)} value={`${update_cache_timeout} ${i18n._(t`seconds`)}`} /> +