diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index ba989ea65f..46f8817511 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -46,8 +46,8 @@ class MemObject(object): ''' def __init__(self, name, source_dir): - assert name - assert source_dir + assert name, 'no name' + assert source_dir, 'no source dir' self.name = name self.source_dir = source_dir @@ -95,17 +95,18 @@ class MemGroup(MemObject): logger.debug('Looking for %s as child group of %s', name, self.name) # slight hack here, passing in 'self' for all_group but child=True won't use it group = loader.get_group(name, self, child=True) - # don't add to child groups if already there - for g in self.children: - if g.name == name: - return g - logger.debug('Adding child group %s to group %s', group.name, self.name) - self.children.append(group) + if group: + # don't add to child groups if already there + for g in self.children: + if g.name == name: + return g + logger.debug('Adding child group %s to group %s', group.name, self.name) + self.children.append(group) return group def add_child_group(self, group): - assert group.name is not 'all' - assert isinstance(group, MemGroup) + assert group.name is not 'all', 'group name is all' + assert isinstance(group, MemGroup), 'not MemGroup instance' logger.debug('Adding child group %s to parent %s', group.name, self.name) if group not in self.children: self.children.append(group) @@ -113,7 +114,7 @@ class MemGroup(MemObject): group.parents.append(self) def add_host(self, host): - assert isinstance(host, MemHost) + assert isinstance(host, MemHost), 'not MemHost instance' logger.debug('Adding host %s to group %s', host.name, self.name) if host not in self.hosts: self.hosts.append(host) @@ -156,10 +157,12 @@ class BaseLoader(object): Common functions for an inventory loader from a given source. ''' - def __init__(self, source, all_group=None): + def __init__(self, source, all_group=None, group_filter_re=None, host_filter_re=None): self.source = source self.source_dir = os.path.dirname(self.source) self.all_group = all_group or MemGroup('all', self.source_dir) + self.group_filter_re = group_filter_re + self.host_filter_re = host_filter_re def get_host(self, name): ''' @@ -168,6 +171,9 @@ class BaseLoader(object): if '[' in name or ']' in name: raise ValueError('host ranges like %s are not supported by this importer' % name) host_name = name.split(':')[0] + if self.host_filter_re and not self.host_filter_re.match(host_name): + logger.debug('Filtering host %s', host_name) + return None host = None if not host_name in self.all_group.all_hosts: host = MemHost(name, self.source_dir) @@ -219,6 +225,9 @@ class BaseLoader(object): all_group = all_group or self.all_group if name == 'all': return all_group + if self.group_filter_re and not self.group_filter_re.match(name): + logger.debug('Filtering group %s', name) + return None if not name in self.all_group.all_groups: group = MemGroup(name, self.source_dir) if not child: @@ -244,34 +253,38 @@ class IniLoader(BaseLoader): if not line: continue elif line.startswith('[') and line.endswith(']'): - # Mode change, possible new group name - line = line[1:-1].strip() - if line.endswith(':vars'): - input_mode = 'vars' - line = line[:-5] - elif line.endswith(':children'): - input_mode = 'children' - line = line[:-9] - else: - input_mode = 'host' - group = self.get_group(line) - else: - # Add hosts with inline variables, or variables/children to - # an existing group. - tokens = shlex.split(line) - if input_mode == 'host': - for host in self.get_hosts(tokens[0]): - if len(tokens) > 1: - for t in tokens[1:]: - k,v = t.split('=', 1) - host.variables[k] = v - group.add_host(host) - elif input_mode == 'children': - group.child_group_by_name(line, self) - elif input_mode == 'vars': - for t in tokens: - k, v = t.split('=', 1) - group.variables[k] = v + # Mode change, possible new group name + line = line[1:-1].strip() + if line.endswith(':vars'): + input_mode = 'vars' + line = line[:-5] + elif line.endswith(':children'): + input_mode = 'children' + line = line[:-9] + else: + input_mode = 'host' + group = self.get_group(line) + elif group: + # If group is None, we are skipping this group and shouldn't + # capture any children/variables/hosts under it. + # Add hosts with inline variables, or variables/children to + # an existing group. + tokens = shlex.split(line) + if input_mode == 'host': + for host in self.get_hosts(tokens[0]): + if not host: + continue + if len(tokens) > 1: + for t in tokens[1:]: + k,v = t.split('=', 1) + host.variables[k] = v + group.add_host(host) + elif input_mode == 'children': + group.child_group_by_name(line, self) + elif input_mode == 'vars': + for t in tokens: + k, v = t.split('=', 1) + group.variables[k] = v # TODO: expansion patterns are probably not going to be supported. YES THEY ARE! @@ -326,6 +339,8 @@ class ExecutableJsonLoader(BaseLoader): for k,v in data.iteritems(): group = self.get_group(k) + if not group: + continue # Load group hosts/vars/children from a dictionary. if isinstance(v, dict): @@ -334,6 +349,8 @@ class ExecutableJsonLoader(BaseLoader): if isinstance(hosts, dict): for hk, hv in hosts.iteritems(): host = self.get_host(hk) + if not host: + continue if isinstance(hv, dict): host.variables.update(hv) else: @@ -344,6 +361,8 @@ class ExecutableJsonLoader(BaseLoader): elif isinstance(hosts, (list, tuple)): for hk in hosts: host = self.get_host(hk) + if not host: + continue group.add_host(host) else: logger.warning('Expected dict or list of "hosts" for ' @@ -362,7 +381,8 @@ class ExecutableJsonLoader(BaseLoader): if isinstance(children, (list, tuple)): for c in children: child = self.get_group(c, self.all_group, child=True) - group.add_child_group(child) + if child: + group.add_child_group(child) else: self.logger.warning('Expected list of children for ' 'group "%s", got %s instead', @@ -372,6 +392,8 @@ class ExecutableJsonLoader(BaseLoader): elif isinstance(v, (list, tuple)): for h in v: host = self.get_host(h) + if not host: + continue group.add_host(host) else: logger.warning('') @@ -396,7 +418,8 @@ class ExecutableJsonLoader(BaseLoader): k, str(type(data))) -def load_inventory_source(source, all_group=None): +def load_inventory_source(source, all_group=None, group_filter_re=None, + host_filter_re=None, exclude_empty_groups=False): ''' Load inventory from given source directory or file. ''' @@ -409,15 +432,25 @@ def load_inventory_source(source, all_group=None): for filename in glob.glob(os.path.join(source, '*')): if filename.endswith(".ini") or os.path.isdir(filename): continue - load_inventory_source(filename, all_group) + load_inventory_source(filename, all_group, group_filter_re, + host_filter_re) else: all_group = all_group or MemGroup('all', os.path.dirname(source)) if os.access(source, os.X_OK): - ExecutableJsonLoader(source, all_group).load() + ExecutableJsonLoader(source, all_group, group_filter_re, host_filter_re).load() else: - IniLoader(source, all_group).load() + IniLoader(source, all_group, group_filter_re, host_filter_re).load() logger.debug('Finished loading from source: %s', source) + # Exclude groups that are completely empty. + if original_all_group is None and exclude_empty_groups: + for name, group in all_group.all_groups.items(): + if not group.children and not group.hosts and not group.variables: + logger.debug('Removing empty group %s', name) + for parent in group.parents: + if group in parent.children: + parent.children.remove(group) + del all_group.all_groups[name] if original_all_group is None: logger.info('Loaded %d groups, %d hosts', len(all_group.all_groups), len(all_group.all_hosts)) @@ -457,6 +490,16 @@ class Command(NoArgsCommand): default=None, metavar='v', help='value of host variable ' 'specified by --enabled-var that indicates host is ' 'enabled/online.'), + make_option('--group-filter', dest='group_filter', type='str', + default=None, metavar='regex', help='regular expression ' + 'to filter group name(s); only matches are imported.'), + make_option('--host-filter', dest='host_filter', type='str', + default=None, metavar='regex', help='regular expression ' + 'to filter host name(s); only matches are imported.'), + make_option('--exclude-empty-groups', dest='exclude_empty_groups', + action='store_true', default=False, help='when set, ' + 'exclude all groups that have no child groups, hosts, or ' + 'variables.'), ) def init_logging(self): @@ -768,6 +811,9 @@ class Command(NoArgsCommand): self.source = options.get('source', None) self.enabled_var = options.get('enabled_var', None) self.enabled_value = options.get('enabled_value', None) + self.group_filter = options.get('group_filter', None) or r'^.+$' + self.host_filter = options.get('host_filter', None) or r'^.+$' + self.exclude_empty_groups = bool(options.get('exclude_empty_groups', False)) # Load inventory and related objects from database. if self.inventory_name and self.inventory_id: @@ -778,6 +824,14 @@ class Command(NoArgsCommand): raise CommandError('--overwrite/--overwrite-vars and --keep-vars are mutually exclusive') if not self.source: raise CommandError('--source is required') + try: + self.group_filter_re = re.compile(self.group_filter) + except re.error: + raise CommandError('invalid regular expression for --group-filter') + try: + self.host_filter_re = re.compile(self.host_filter) + except re.error: + raise CommandError('invalid regular expression for --host-filter') self.check_license() begin = time.time() @@ -793,7 +847,10 @@ class Command(NoArgsCommand): transaction.commit() # Load inventory from source. - self.all_group = load_inventory_source(self.source) + self.all_group = load_inventory_source(self.source, None, + self.group_filter_re, + self.host_filter_re, + self.exclude_empty_groups) self.all_group.debug_tree() # Merge/overwrite inventory into database. diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 8b1d43d4e6..b9ee2eade6 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -823,15 +823,23 @@ class RunInventoryUpdate(BaseTask): if inventory_update.source == 'ec2': ec2_path = self.get_path_to('..', 'plugins', 'inventory', 'ec2.py') args.append(ec2_path) - args.extend(['--enabled-var', 'ec2_state']) - args.extend(['--enabled-value', 'running']) - #args.extend(['--instance-id', 'ec2_id']) + args.extend(['--enabled-var', settings.EC2_ENABLED_VAR]) + args.extend(['--enabled-value', settings.EC2_ENABLED_VALUE]) + args.extend(['--group-filter', settings.EC2_GROUP_FILTER]) + args.extend(['--host-filter', settings.EC2_HOST_FILTER]) + if settings.EC2_EXCLUDE_EMPTY_GROUPS: + args.append('--exclude-empty-groups') + #args.extend(['--instance-id', settings.EC2_INSTANCE_ID_VAR]) elif inventory_update.source == 'rax': rax_path = self.get_path_to('..', 'plugins', 'inventory', 'rax.py') args.append(rax_path) - args.extend(['--enabled-var', 'rax_status']) - args.extend(['--enabled-value', 'ACTIVE']) - #args.extend(['--instance-id', 'rax_id']) + args.extend(['--enabled-var', settings.RAX_ENABLED_VAR]) + args.extend(['--enabled-value', settings.RAX_ENABLED_VALUE]) + args.extend(['--group-filter', settings.RAX_GROUP_FILTER]) + args.extend(['--host-filter', settings.RAX_HOST_FILTER]) + if settings.RAX_EXCLUDE_EMPTY_GROUPS: + args.append('--exclude-empty-groups') + #args.extend(['--instance-id', settings.RAX_INSTANCE_ID_VAR]) elif inventory_update.source == 'file': args.append(inventory_update.source_path) verbosity = getattr(settings, 'INVENTORY_UPDATE_VERBOSITY', 1) diff --git a/awx/main/tests/inventory.py b/awx/main/tests/inventory.py index d73065e550..e3e66f35b4 100644 --- a/awx/main/tests/inventory.py +++ b/awx/main/tests/inventory.py @@ -1066,6 +1066,10 @@ class InventoryUpdatesTest(BaseTransactionTest): self.assertTrue(inventory_source.pk in source_pks) self.assertTrue(host.has_inventory_sources) self.assertTrue(host.enabled) + # Make sure EC2 RDS hosts are excluded. + if inventory_source.source == 'ec2': + self.assertFalse(re.match(r'^.+\.rds\.amazonaws\.com$', host.name, re.I), + host.name) with self.current_user(self.super_django_user): url = reverse('api:host_inventory_sources_list', args=(host.pk,)) response = self.get(url, expect=200) @@ -1074,9 +1078,18 @@ class InventoryUpdatesTest(BaseTransactionTest): source_pks = group.inventory_sources.values_list('pk', flat=True) self.assertTrue(inventory_source.pk in source_pks) self.assertTrue(group.has_inventory_sources) - # Make sure EC2 instance ID groups are excluded. - self.assertFalse(re.match(r'^i-[0-9a-f]{8}$', group.name, re.I), - group.name) + self.assertTrue(group.children.filter(active=True).exists() or + group.hosts.filter(active=True).exists()) + # Make sure EC2 instance ID groups and RDS groups are excluded. + if inventory_source.source == 'ec2': + self.assertFalse(re.match(r'^i-[0-9a-f]{8}$', group.name, re.I), + group.name) + self.assertFalse(re.match(r'^rds|rds_.+|type_db_.+$', group.name, re.I), + group.name) + # Make sure Rackspace instance ID groups are excluded. + if inventory_source.source == 'rax': + self.assertFalse(re.match(r'^instance-.+$', group.name, re.I), + group.name) with self.current_user(self.super_django_user): url = reverse('api:group_inventory_sources_list', args=(group.pk,)) response = self.get(url, expect=200) diff --git a/awx/plugins/inventory/ec2.py b/awx/plugins/inventory/ec2.py index ad2132be63..84841d3f09 100755 --- a/awx/plugins/inventory/ec2.py +++ b/awx/plugins/inventory/ec2.py @@ -254,8 +254,7 @@ class Ec2Inventory(object): for region in self.regions: self.get_instances_by_region(region) - # Don't return RDS instances for Ansible Tower! - #self.get_rds_instances_by_region(region) + self.get_rds_instances_by_region(region) self.write_to_cache(self.inventory, self.cache_path_cache) self.write_to_cache(self.index, self.cache_path_index) @@ -344,9 +343,8 @@ class Ec2Inventory(object): # Add to index self.index[dest] = [region, instance.id] - # Do not output group based on instance ID for Ansible Tower! # Inventory: Group by instance ID (always a group of 1) - #self.inventory[instance.id] = [dest] + self.inventory[instance.id] = [dest] # Inventory: Group by region self.push(self.inventory, region, dest) diff --git a/awx/plugins/inventory/rax.py b/awx/plugins/inventory/rax.py index 6836db61f6..039233005d 100755 --- a/awx/plugins/inventory/rax.py +++ b/awx/plugins/inventory/rax.py @@ -22,9 +22,11 @@ DOCUMENTATION = ''' inventory: rax short_description: Rackspace Public Cloud external inventory script description: - - Generates inventory that Ansible can understand by making API request to Rackspace Public Cloud API + - Generates inventory that Ansible can understand by making API request to + Rackspace Public Cloud API - | - When run against a specific host, this script returns the following variables: + When run against a specific host, this script returns the following + variables: rax_os-ext-sts_task_state rax_addresses rax_links @@ -65,12 +67,23 @@ options: authors: - Jesse Keating - Paul Durivage + - Matt Martz notes: - - RAX_CREDS_FILE is an optional environment variable that points to a pyrax-compatible credentials file. - - If RAX_CREDS_FILE is not supplied, rax.py will look for a credentials file at ~/.rackspace_cloud_credentials. + - RAX_CREDS_FILE is an optional environment variable that points to a + pyrax-compatible credentials file. + - If RAX_CREDS_FILE is not supplied, rax.py will look for a credentials file + at ~/.rackspace_cloud_credentials. - See https://github.com/rackspace/pyrax/blob/master/docs/getting_started.md#authenticating - - RAX_REGION is an optional environment variable to narrow inventory search scope - - RAX_REGION, if used, needs a value like ORD, DFW, SYD (a Rackspace datacenter) and optionally accepts a comma-separated list + - RAX_REGION is an optional environment variable to narrow inventory search + scope + - RAX_REGION, if used, needs a value like ORD, DFW, SYD (a Rackspace + datacenter) and optionally accepts a comma-separated list + - RAX_ENV is an environment variable that will use an environment as + configured in ~/.pyrax.cfg, see + https://github.com/rackspace/pyrax/blob/master/docs/getting_started.md#pyrax-configuration + - RAX_META_PREFIX is an environment variable that changes the prefix used + for meta key/value groups. For compatibility with ec2.py set to + RAX_META_PREFIX=tag requirements: [ "pyrax" ] examples: - description: List server instances @@ -83,13 +96,14 @@ examples: code: RAX_CREDS_FILE=~/.raxpub rax.py --host server.example.com ''' -import sys -import re import os - +import re +import sys import argparse import collections +from types import NoneType + try: import json except: @@ -98,9 +112,26 @@ except: try: import pyrax except ImportError: - print('pyrax required for this module') + print('pyrax is required for this module') sys.exit(1) +NON_CALLABLES = (basestring, bool, dict, int, list, NoneType) + + +def rax_slugify(value): + return 'rax_%s' % (re.sub('[^\w-]', '_', value).lower().lstrip('_')) + + +def to_dict(obj): + instance = {} + for key in dir(obj): + value = getattr(obj, key) + if (isinstance(value, NON_CALLABLES) and not key.startswith('_')): + key = rax_slugify(key) + instance[key] = value + + return instance + def host(regions, hostname): hostvars = {} @@ -110,15 +141,7 @@ def host(regions, hostname): cs = pyrax.connect_to_cloudservers(region=region) for server in cs.servers.list(): if server.name == hostname: - keys = [key for key in vars(server) if key not in ('manager', '_info')] - for key in keys: - # Extract value - value = getattr(server, key) - - # Generate sanitized key - key = 'rax_' + (re.sub("[^A-Za-z0-9\-]", "_", key) - .lower() - .lstrip("_")) + for key, value in to_dict(server).items(): hostvars[key] = value # And finally, add an IP address @@ -129,6 +152,7 @@ def host(regions, hostname): def _list(regions): groups = collections.defaultdict(list) hostvars = collections.defaultdict(dict) + images = {} # Go through all the regions looking for servers for region in regions: @@ -139,26 +163,39 @@ def _list(regions): groups[region].append(server.name) # Check if group metadata key in servers' metadata - try: - group = server.metadata['group'] - except KeyError: - pass - else: - # Create group if not exist and add the server + group = server.metadata.get('group') + if group: groups[group].append(server.name) - # Add host metadata - keys = [key for key in vars(server) if key not in ('manager', '_info')] - for key in keys: - # Extract value - value = getattr(server, key) + for extra_group in server.metadata.get('groups', '').split(','): + groups[extra_group].append(server.name) - # Generate sanitized key - key = 'rax_' + (re.sub("[^A-Za-z0-9\-]", "_", key) - .lower() - .lstrip('_')) + # Add host metadata + for key, value in to_dict(server).items(): hostvars[server.name][key] = value + hostvars[server.name]['rax_region'] = region + + for key, value in server.metadata.iteritems(): + prefix = os.getenv('RAX_META_PREFIX', 'meta') + groups['%s_%s_%s' % (prefix, key, value)].append(server.name) + + groups['instance-%s' % server.id].append(server.name) + groups['flavor-%s' % server.flavor['id']].append(server.name) + try: + imagegroup = 'image-%s' % images[server.image['id']] + groups[imagegroup].append(server.name) + groups['image-%s' % server.image['id']].append(server.name) + except KeyError: + try: + image = cs.images.get(server.image['id']) + except cs.exceptions.NotFound: + groups['image-%s' % server.image['id']].append(server.name) + else: + images[image.id] = image.human_id + groups['image-%s' % image.human_id].append(server.name) + groups['image-%s' % server.image['id']].append(server.name) + # And finally, add an IP address hostvars[server.name]['ansible_ssh_host'] = server.accessIPv4 @@ -172,7 +209,7 @@ def parse_args(): 'inventory module') group = parser.add_mutually_exclusive_group(required=True) group.add_argument('--list', action='store_true', - help='List active servers') + help='List active servers') group.add_argument('--host', help='List details about the specific host') return parser.parse_args() @@ -180,38 +217,54 @@ def parse_args(): def setup(): default_creds_file = os.path.expanduser('~/.rackspace_cloud_credentials') + env = os.getenv('RAX_ENV', None) + if env: + pyrax.set_environment(env) + + keyring_username = pyrax.get_setting('keyring_username') + # Attempt to grab credentials from environment first try: - creds_file = os.environ['RAX_CREDS_FILE'] + creds_file = os.path.expanduser(os.environ['RAX_CREDS_FILE']) except KeyError, e: - # But if that fails, use the default location of ~/.rackspace_cloud_credentials + # But if that fails, use the default location of + # ~/.rackspace_cloud_credentials if os.path.isfile(default_creds_file): creds_file = default_creds_file - else: + elif not keyring_username: sys.stderr.write('No value in environment variable %s and/or no ' 'credentials file at %s\n' % (e.message, default_creds_file)) sys.exit(1) - pyrax.set_setting('identity_type', 'rackspace') + identity_type = pyrax.get_setting('identity_type') + pyrax.set_setting('identity_type', identity_type or 'rackspace') + + region = pyrax.get_setting('region') try: - pyrax.set_credential_file(os.path.expanduser(creds_file)) + if keyring_username: + pyrax.keyring_auth(keyring_username, region=region) + else: + pyrax.set_credential_file(creds_file, region=region) except Exception, e: sys.stderr.write("%s: %s\n" % (e, e.message)) sys.exit(1) regions = [] - for region in os.getenv('RAX_REGION', 'all').split(','): - region = region.strip().upper() - if region == 'ALL': - regions = pyrax.regions - break - elif region not in pyrax.regions: - sys.stderr.write('Unsupported region %s' % region) - sys.exit(1) - elif region not in regions: - regions.append(region) + if region: + regions.append(region) + else: + for region in os.getenv('RAX_REGION', 'all').split(','): + region = region.strip().upper() + if region == 'ALL': + regions = pyrax.regions + break + elif region not in pyrax.regions: + sys.stderr.write('Unsupported region %s' % region) + sys.exit(1) + elif region not in regions: + regions.append(region) return regions @@ -225,5 +278,6 @@ def main(): host(regions, args.host) sys.exit(0) + if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 80162c4a69..32cc9a659b 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -323,6 +323,20 @@ RAX_REGION_CHOICES = [ ('HKG', 'Hong Kong'), ] +# Inventory variable name/values for determining if host is active/enabled. +RAX_ENABLED_VAR = 'rax_status' +RAX_ENABLED_VALUE = 'ACTIVE' + +# Inventory variable name containing unique instance ID. +RAX_INSTANCE_ID_VAR = 'rax_id' + +# Filter for allowed group/host names when importing inventory from Rackspace. +# By default, filter group of one created for each instance and exclude all +# groups without children, hosts and variables. +RAX_GROUP_FILTER = r'^(?!instance-.+).+$' +RAX_HOST_FILTER = r'^.+$' +RAX_EXCLUDE_EMPTY_GROUPS = True + # AWS does not appear to provide pretty region names via any API, so store the # list of names here. The available region IDs will be pulled from boto. # http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region @@ -343,6 +357,20 @@ EC2_REGIONS_BLACKLIST = [ 'cn-north-1', ] +# Inventory variable name/values for determining if host is active/enabled. +EC2_ENABLED_VAR = 'ec2_state' +EC2_ENABLED_VALUE = 'running' + +# Inventory variable name containing unique instance ID. +EC2_INSTANCE_ID_VAR = 'ec2_id' + +# Filter for allowed group/host names when importing inventory from EC2. +# By default, filter group of one created for each instance, filter all RDS +# hosts, and exclude all groups without children, hosts and variables. +EC2_GROUP_FILTER = r'^(?!i-[a-f0-9]{8,}).+$' +EC2_HOST_FILTER = r'^.+(?