diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index d5b2bcdcd3..e32c924538 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -857,7 +857,7 @@ class Command(NoArgsCommand): del_pks = del_host_pks[offset:(offset + self._batch_size)] for db_host in db_hosts.filter(pk__in=del_pks): group_host_count += 1 - if db_host not in db_group.hosts: + if db_host not in db_group.hosts.all(): continue db_group.hosts.remove(db_host) self.logger.info('Host "%s" removed from group "%s"', diff --git a/awx/plugins/inventory/ec2.py b/awx/plugins/inventory/ec2.py index cab0d2b85c..bf3ba28ea0 100755 --- a/awx/plugins/inventory/ec2.py +++ b/awx/plugins/inventory/ec2.py @@ -141,6 +141,7 @@ except ImportError: class Ec2Inventory(object): + def _empty_inventory(self): return {"_meta" : {"hostvars" : {}}} @@ -157,6 +158,9 @@ class Ec2Inventory(object): # Boto profile to use (if any) self.boto_profile = None + # AWS credentials. + self.credentials = {} + # Read settings and parse CLI arguments self.parse_cli_args() self.read_settings() @@ -224,7 +228,7 @@ class Ec2Inventory(object): configRegions_exclude = config.get('ec2', 'regions_exclude') if (configRegions == 'all'): if self.eucalyptus_host: - self.regions.append(boto.connect_euca(host=self.eucalyptus_host).region.name) + self.regions.append(boto.connect_euca(host=self.eucalyptus_host).region.name, **self.credentials) else: for regionInfo in ec2.regions(): if regionInfo.name not in configRegions_exclude: @@ -236,6 +240,19 @@ class Ec2Inventory(object): self.destination_variable = config.get('ec2', 'destination_variable') self.vpc_destination_variable = config.get('ec2', 'vpc_destination_variable') + if config.has_option('ec2', 'hostname_variable'): + self.hostname_variable = config.get('ec2', 'hostname_variable') + else: + self.hostname_variable = None + + if config.has_option('ec2', 'destination_format') and \ + config.has_option('ec2', 'destination_format_tags'): + self.destination_format = config.get('ec2', 'destination_format') + self.destination_format_tags = config.get('ec2', 'destination_format_tags').split(',') + else: + self.destination_format = None + self.destination_format_tags = None + # Route53 self.route53_enabled = config.getboolean('ec2', 'route53') self.route53_excluded_zones = [] @@ -310,6 +327,29 @@ class Ec2Inventory(object): if config.has_option('ec2', 'boto_profile') and not self.boto_profile: self.boto_profile = config.get('ec2', 'boto_profile') + # AWS credentials (prefer environment variables) + if not (self.boto_profile or os.environ.get('AWS_ACCESS_KEY_ID') or + os.environ.get('AWS_PROFILE')): + if config.has_option('credentials', 'aws_access_key_id'): + aws_access_key_id = config.get('credentials', 'aws_access_key_id') + else: + aws_access_key_id = None + if config.has_option('credentials', 'aws_secret_access_key'): + aws_secret_access_key = config.get('credentials', 'aws_secret_access_key') + else: + aws_secret_access_key = None + if config.has_option('credentials', 'aws_security_token'): + aws_security_token = config.get('credentials', 'aws_security_token') + else: + aws_security_token = None + if aws_access_key_id: + self.credentials = { + 'aws_access_key_id': aws_access_key_id, + 'aws_secret_access_key': aws_secret_access_key + } + if aws_security_token: + self.credentials['security_token'] = aws_security_token + # Cache related cache_dir = os.path.expanduser(config.get('ec2', 'cache_path')) if self.boto_profile: @@ -317,10 +357,22 @@ class Ec2Inventory(object): if not os.path.exists(cache_dir): os.makedirs(cache_dir) - self.cache_path_cache = cache_dir + "/ansible-ec2.cache" - self.cache_path_index = cache_dir + "/ansible-ec2.index" + cache_name = 'ansible-ec2' + aws_profile = lambda: (self.boto_profile or + os.environ.get('AWS_PROFILE') or + os.environ.get('AWS_ACCESS_KEY_ID') or + self.credentials.get('aws_access_key_id', None)) + if aws_profile(): + cache_name = '%s-%s' % (cache_name, aws_profile()) + self.cache_path_cache = cache_dir + "/%s.cache" % cache_name + self.cache_path_index = cache_dir + "/%s.index" % cache_name self.cache_max_age = config.getint('ec2', 'cache_max_age') + if config.has_option('ec2', 'expand_csv_tags'): + self.expand_csv_tags = config.getboolean('ec2', 'expand_csv_tags') + else: + self.expand_csv_tags = False + # Configure nested groups instead of flat namespace. if config.has_option('ec2', 'nested_groups'): self.nested_groups = config.getboolean('ec2', 'nested_groups') @@ -382,7 +434,10 @@ class Ec2Inventory(object): # Instance filters (see boto and EC2 API docs). Ignore invalid filters. self.ec2_instance_filters = defaultdict(list) if config.has_option('ec2', 'instance_filters'): - for instance_filter in config.get('ec2', 'instance_filters', '').split(','): + + filters = [f for f in config.get('ec2', 'instance_filters').split(',') if f] + + for instance_filter in filters: instance_filter = instance_filter.strip() if not instance_filter or '=' not in instance_filter: continue @@ -401,7 +456,7 @@ class Ec2Inventory(object): help='Get all the variables about a specific instance') parser.add_argument('--refresh-cache', action='store_true', default=False, help='Force refresh of cache by making API requests to EC2 (default: False - use cache files)') - parser.add_argument('--boto-profile', action='store', + parser.add_argument('--profile', '--boto-profile', action='store', dest='boto_profile', help='Use boto profile for connections to EC2') self.args = parser.parse_args() @@ -426,7 +481,7 @@ class Ec2Inventory(object): def connect(self, region): ''' create connection to api server''' if self.eucalyptus: - conn = boto.connect_euca(host=self.eucalyptus_host) + conn = boto.connect_euca(host=self.eucalyptus_host, **self.credentials) conn.APIVersion = '2010-08-31' else: conn = self.connect_to_aws(ec2, region) @@ -440,7 +495,7 @@ class Ec2Inventory(object): return connect_args def connect_to_aws(self, module, region): - connect_args = {} + connect_args = self.credentials # only pass the profile name if it's set (as it is not supported by older boto versions) if self.boto_profile: @@ -466,15 +521,27 @@ class Ec2Inventory(object): else: reservations = conn.get_all_instances() + # Pull the tags back in a second step + # AWS are on record as saying that the tags fetched in the first `get_all_instances` request are not + # reliable and may be missing, and the only way to guarantee they are there is by calling `get_all_tags` + instance_ids = [] + for reservation in reservations: + instance_ids.extend([instance.id for instance in reservation.instances]) + tags = conn.get_all_tags(filters={'resource-type': 'instance', 'resource-id': instance_ids}) + tags_by_instance_id = defaultdict(dict) + for tag in tags: + tags_by_instance_id[tag.res_id][tag.name] = tag.value + for reservation in reservations: for instance in reservation.instances: + instance.tags = tags_by_instance_id[instance.id] self.add_instance(instance, region) except boto.exception.BotoServerError as e: if e.error_code == 'AuthFailure': error = self.get_auth_error_message() else: - backend = 'Eucalyptus' if self.eucalyptus else 'AWS' + backend = 'Eucalyptus' if self.eucalyptus else 'AWS' error = "Error connecting to %s backend.\n%s" % (backend, e.message) self.fail_with_error(error, 'getting EC2 instances') @@ -485,9 +552,14 @@ class Ec2Inventory(object): try: conn = self.connect_to_aws(rds, region) if conn: - instances = conn.get_all_dbinstances() - for instance in instances: - self.add_rds_instance(instance, region) + marker = None + while True: + instances = conn.get_all_dbinstances(marker=marker) + marker = instances.marker + for instance in instances: + self.add_rds_instance(instance, region) + if not marker: + break except boto.exception.BotoServerError as e: error = e.reason @@ -609,7 +681,9 @@ class Ec2Inventory(object): return # Select the best destination address - if instance.subnet_id: + if self.destination_format and self.destination_format_tags: + dest = self.destination_format.format(*[ getattr(instance, 'tags').get(tag, '') for tag in self.destination_format_tags ]) + elif instance.subnet_id: dest = getattr(instance, self.vpc_destination_variable, None) if dest is None: dest = getattr(instance, 'tags').get(self.vpc_destination_variable, None) @@ -622,32 +696,46 @@ class Ec2Inventory(object): # Skip instances we cannot address (e.g. private VPC subnet) return + # Set the inventory name + hostname = None + if self.hostname_variable: + if self.hostname_variable.startswith('tag_'): + hostname = instance.tags.get(self.hostname_variable[4:], None) + else: + hostname = getattr(instance, self.hostname_variable) + + # If we can't get a nice hostname, use the destination address + if not hostname: + hostname = dest + else: + hostname = self.to_safe(hostname).lower() + # if we only want to include hosts that match a pattern, skip those that don't - if self.pattern_include and not self.pattern_include.match(dest): + if self.pattern_include and not self.pattern_include.match(hostname): return # if we need to exclude hosts that match a pattern, skip those - if self.pattern_exclude and self.pattern_exclude.match(dest): + if self.pattern_exclude and self.pattern_exclude.match(hostname): return # Add to index - self.index[dest] = [region, instance.id] + self.index[hostname] = [region, instance.id] # Inventory: Group by instance ID (always a group of 1) if self.group_by_instance_id: - self.inventory[instance.id] = [dest] + self.inventory[instance.id] = [hostname] if self.nested_groups: self.push_group(self.inventory, 'instances', instance.id) # Inventory: Group by region if self.group_by_region: - self.push(self.inventory, region, dest) + self.push(self.inventory, region, hostname) if self.nested_groups: self.push_group(self.inventory, 'regions', region) # Inventory: Group by availability zone if self.group_by_availability_zone: - self.push(self.inventory, instance.placement, dest) + self.push(self.inventory, instance.placement, hostname) if self.nested_groups: if self.group_by_region: self.push_group(self.inventory, region, instance.placement) @@ -656,28 +744,28 @@ class Ec2Inventory(object): # Inventory: Group by Amazon Machine Image (AMI) ID if self.group_by_ami_id: ami_id = self.to_safe(instance.image_id) - self.push(self.inventory, ami_id, dest) + self.push(self.inventory, ami_id, hostname) if self.nested_groups: self.push_group(self.inventory, 'images', ami_id) # Inventory: Group by instance type if self.group_by_instance_type: type_name = self.to_safe('type_' + instance.instance_type) - self.push(self.inventory, type_name, dest) + self.push(self.inventory, type_name, hostname) if self.nested_groups: self.push_group(self.inventory, 'types', type_name) # Inventory: Group by key pair if self.group_by_key_pair and instance.key_name: key_name = self.to_safe('key_' + instance.key_name) - self.push(self.inventory, key_name, dest) + self.push(self.inventory, key_name, hostname) if self.nested_groups: self.push_group(self.inventory, 'keys', key_name) # Inventory: Group by VPC if self.group_by_vpc_id and instance.vpc_id: vpc_id_name = self.to_safe('vpc_id_' + instance.vpc_id) - self.push(self.inventory, vpc_id_name, dest) + self.push(self.inventory, vpc_id_name, hostname) if self.nested_groups: self.push_group(self.inventory, 'vpcs', vpc_id_name) @@ -686,44 +774,51 @@ class Ec2Inventory(object): try: for group in instance.groups: key = self.to_safe("security_group_" + group.name) - self.push(self.inventory, key, dest) + self.push(self.inventory, key, hostname) if self.nested_groups: self.push_group(self.inventory, 'security_groups', key) except AttributeError: - self.fail_with_error('\n'.join(['Package boto seems a bit older.', + self.fail_with_error('\n'.join(['Package boto seems a bit older.', 'Please upgrade boto >= 2.3.0.'])) # Inventory: Group by tag keys if self.group_by_tag_keys: for k, v in instance.tags.items(): - if v: - key = self.to_safe("tag_" + k + "=" + v) + if self.expand_csv_tags and v and ',' in v: + values = map(lambda x: x.strip(), v.split(',')) else: - key = self.to_safe("tag_" + k) - self.push(self.inventory, key, dest) - if self.nested_groups: - self.push_group(self.inventory, 'tags', self.to_safe("tag_" + k)) + values = [v] + + for v in values: if v: - self.push_group(self.inventory, self.to_safe("tag_" + k), key) + key = self.to_safe("tag_" + k + "=" + v) + else: + key = self.to_safe("tag_" + k) + self.push(self.inventory, key, hostname) + if self.nested_groups: + self.push_group(self.inventory, 'tags', self.to_safe("tag_" + k)) + if v: + self.push_group(self.inventory, self.to_safe("tag_" + k), key) # Inventory: Group by Route53 domain names if enabled if self.route53_enabled and self.group_by_route53_names: route53_names = self.get_instance_route53_names(instance) for name in route53_names: - self.push(self.inventory, name, dest) + self.push(self.inventory, name, hostname) if self.nested_groups: self.push_group(self.inventory, 'route53', name) # Global Tag: instances without tags if self.group_by_tag_none and len(instance.tags) == 0: - self.push(self.inventory, 'tag_none', dest) + self.push(self.inventory, 'tag_none', hostname) if self.nested_groups: self.push_group(self.inventory, 'tags', 'tag_none') # Global Tag: tag all EC2 instances - self.push(self.inventory, 'ec2', dest) + self.push(self.inventory, 'ec2', hostname) - self.inventory["_meta"]["hostvars"][dest] = self.get_host_info_dict_from_instance(instance) + self.inventory["_meta"]["hostvars"][hostname] = self.get_host_info_dict_from_instance(instance) + self.inventory["_meta"]["hostvars"][hostname]['ansible_ssh_host'] = dest def add_rds_instance(self, instance, region): @@ -741,24 +836,38 @@ class Ec2Inventory(object): # Skip instances we cannot address (e.g. private VPC subnet) return + # Set the inventory name + hostname = None + if self.hostname_variable: + if self.hostname_variable.startswith('tag_'): + hostname = instance.tags.get(self.hostname_variable[4:], None) + else: + hostname = getattr(instance, self.hostname_variable) + + # If we can't get a nice hostname, use the destination address + if not hostname: + hostname = dest + + hostname = self.to_safe(hostname).lower() + # Add to index - self.index[dest] = [region, instance.id] + self.index[hostname] = [region, instance.id] # Inventory: Group by instance ID (always a group of 1) if self.group_by_instance_id: - self.inventory[instance.id] = [dest] + self.inventory[instance.id] = [hostname] if self.nested_groups: self.push_group(self.inventory, 'instances', instance.id) # Inventory: Group by region if self.group_by_region: - self.push(self.inventory, region, dest) + self.push(self.inventory, region, hostname) if self.nested_groups: self.push_group(self.inventory, 'regions', region) # Inventory: Group by availability zone if self.group_by_availability_zone: - self.push(self.inventory, instance.availability_zone, dest) + self.push(self.inventory, instance.availability_zone, hostname) if self.nested_groups: if self.group_by_region: self.push_group(self.inventory, region, instance.availability_zone) @@ -767,14 +876,14 @@ class Ec2Inventory(object): # Inventory: Group by instance type if self.group_by_instance_type: type_name = self.to_safe('type_' + instance.instance_class) - self.push(self.inventory, type_name, dest) + self.push(self.inventory, type_name, hostname) if self.nested_groups: self.push_group(self.inventory, 'types', type_name) # Inventory: Group by VPC if self.group_by_vpc_id and instance.subnet_group and instance.subnet_group.vpc_id: vpc_id_name = self.to_safe('vpc_id_' + instance.subnet_group.vpc_id) - self.push(self.inventory, vpc_id_name, dest) + self.push(self.inventory, vpc_id_name, hostname) if self.nested_groups: self.push_group(self.inventory, 'vpcs', vpc_id_name) @@ -783,31 +892,32 @@ class Ec2Inventory(object): try: if instance.security_group: key = self.to_safe("security_group_" + instance.security_group.name) - self.push(self.inventory, key, dest) + self.push(self.inventory, key, hostname) if self.nested_groups: self.push_group(self.inventory, 'security_groups', key) except AttributeError: - self.fail_with_error('\n'.join(['Package boto seems a bit older.', + self.fail_with_error('\n'.join(['Package boto seems a bit older.', 'Please upgrade boto >= 2.3.0.'])) # Inventory: Group by engine if self.group_by_rds_engine: - self.push(self.inventory, self.to_safe("rds_" + instance.engine), dest) + self.push(self.inventory, self.to_safe("rds_" + instance.engine), hostname) if self.nested_groups: self.push_group(self.inventory, 'rds_engines', self.to_safe("rds_" + instance.engine)) # Inventory: Group by parameter group if self.group_by_rds_parameter_group: - self.push(self.inventory, self.to_safe("rds_parameter_group_" + instance.parameter_group.name), dest) + self.push(self.inventory, self.to_safe("rds_parameter_group_" + instance.parameter_group.name), hostname) if self.nested_groups: self.push_group(self.inventory, 'rds_parameter_groups', self.to_safe("rds_parameter_group_" + instance.parameter_group.name)) # Global Tag: all RDS instances - self.push(self.inventory, 'rds', dest) + self.push(self.inventory, 'rds', hostname) - self.inventory["_meta"]["hostvars"][dest] = self.get_host_info_dict_from_instance(instance) + self.inventory["_meta"]["hostvars"][hostname] = self.get_host_info_dict_from_instance(instance) + self.inventory["_meta"]["hostvars"][hostname]['ansible_ssh_host'] = dest def add_elasticache_cluster(self, cluster, region): ''' Adds an ElastiCache cluster to the inventory and index, as long as @@ -1120,6 +1230,8 @@ class Ec2Inventory(object): instance_vars['ec2_placement'] = value.zone elif key == 'ec2_tags': for k, v in value.items(): + if self.expand_csv_tags and ',' in v: + v = map(lambda x: x.strip(), v.split(',')) key = self.to_safe('ec2_tag_' + k) instance_vars[key] = v elif key == 'ec2_groups': @@ -1294,7 +1406,7 @@ class Ec2Inventory(object): def to_safe(self, word): ''' Converts 'bad' characters in a string to underscores so they can be used as Ansible groups ''' regex = "[^A-Za-z0-9\_" - if self.replace_dash_in_groups: + if not self.replace_dash_in_groups: regex += "\-" return re.sub(regex + "]", "_", word) @@ -1310,4 +1422,3 @@ class Ec2Inventory(object): # Run the script Ec2Inventory() -