AC-1134 Updated rax/ec2 inventory scripts from ansible. Added ability to filter groups/hosts by regular expression and exclude empty groups, so that Tower can exclude RAX/EC2 instance ID groups and EC2 RDS hosts/groups.

This commit is contained in:
Chris Church 2014-04-08 23:50:57 -04:00
parent 59dc2a2cb8
commit 1283a78e09
6 changed files with 268 additions and 110 deletions

View File

@ -46,8 +46,8 @@ class MemObject(object):
'''
def __init__(self, name, source_dir):
assert name
assert source_dir
assert name, 'no name'
assert source_dir, 'no source dir'
self.name = name
self.source_dir = source_dir
@ -95,17 +95,18 @@ class MemGroup(MemObject):
logger.debug('Looking for %s as child group of %s', name, self.name)
# slight hack here, passing in 'self' for all_group but child=True won't use it
group = loader.get_group(name, self, child=True)
# don't add to child groups if already there
for g in self.children:
if g.name == name:
return g
logger.debug('Adding child group %s to group %s', group.name, self.name)
self.children.append(group)
if group:
# don't add to child groups if already there
for g in self.children:
if g.name == name:
return g
logger.debug('Adding child group %s to group %s', group.name, self.name)
self.children.append(group)
return group
def add_child_group(self, group):
assert group.name is not 'all'
assert isinstance(group, MemGroup)
assert group.name is not 'all', 'group name is all'
assert isinstance(group, MemGroup), 'not MemGroup instance'
logger.debug('Adding child group %s to parent %s', group.name, self.name)
if group not in self.children:
self.children.append(group)
@ -113,7 +114,7 @@ class MemGroup(MemObject):
group.parents.append(self)
def add_host(self, host):
assert isinstance(host, MemHost)
assert isinstance(host, MemHost), 'not MemHost instance'
logger.debug('Adding host %s to group %s', host.name, self.name)
if host not in self.hosts:
self.hosts.append(host)
@ -156,10 +157,12 @@ class BaseLoader(object):
Common functions for an inventory loader from a given source.
'''
def __init__(self, source, all_group=None):
def __init__(self, source, all_group=None, group_filter_re=None, host_filter_re=None):
self.source = source
self.source_dir = os.path.dirname(self.source)
self.all_group = all_group or MemGroup('all', self.source_dir)
self.group_filter_re = group_filter_re
self.host_filter_re = host_filter_re
def get_host(self, name):
'''
@ -168,6 +171,9 @@ class BaseLoader(object):
if '[' in name or ']' in name:
raise ValueError('host ranges like %s are not supported by this importer' % name)
host_name = name.split(':')[0]
if self.host_filter_re and not self.host_filter_re.match(host_name):
logger.debug('Filtering host %s', host_name)
return None
host = None
if not host_name in self.all_group.all_hosts:
host = MemHost(name, self.source_dir)
@ -219,6 +225,9 @@ class BaseLoader(object):
all_group = all_group or self.all_group
if name == 'all':
return all_group
if self.group_filter_re and not self.group_filter_re.match(name):
logger.debug('Filtering group %s', name)
return None
if not name in self.all_group.all_groups:
group = MemGroup(name, self.source_dir)
if not child:
@ -244,34 +253,38 @@ class IniLoader(BaseLoader):
if not line:
continue
elif line.startswith('[') and line.endswith(']'):
# Mode change, possible new group name
line = line[1:-1].strip()
if line.endswith(':vars'):
input_mode = 'vars'
line = line[:-5]
elif line.endswith(':children'):
input_mode = 'children'
line = line[:-9]
else:
input_mode = 'host'
group = self.get_group(line)
else:
# Add hosts with inline variables, or variables/children to
# an existing group.
tokens = shlex.split(line)
if input_mode == 'host':
for host in self.get_hosts(tokens[0]):
if len(tokens) > 1:
for t in tokens[1:]:
k,v = t.split('=', 1)
host.variables[k] = v
group.add_host(host)
elif input_mode == 'children':
group.child_group_by_name(line, self)
elif input_mode == 'vars':
for t in tokens:
k, v = t.split('=', 1)
group.variables[k] = v
# Mode change, possible new group name
line = line[1:-1].strip()
if line.endswith(':vars'):
input_mode = 'vars'
line = line[:-5]
elif line.endswith(':children'):
input_mode = 'children'
line = line[:-9]
else:
input_mode = 'host'
group = self.get_group(line)
elif group:
# If group is None, we are skipping this group and shouldn't
# capture any children/variables/hosts under it.
# Add hosts with inline variables, or variables/children to
# an existing group.
tokens = shlex.split(line)
if input_mode == 'host':
for host in self.get_hosts(tokens[0]):
if not host:
continue
if len(tokens) > 1:
for t in tokens[1:]:
k,v = t.split('=', 1)
host.variables[k] = v
group.add_host(host)
elif input_mode == 'children':
group.child_group_by_name(line, self)
elif input_mode == 'vars':
for t in tokens:
k, v = t.split('=', 1)
group.variables[k] = v
# TODO: expansion patterns are probably not going to be supported. YES THEY ARE!
@ -326,6 +339,8 @@ class ExecutableJsonLoader(BaseLoader):
for k,v in data.iteritems():
group = self.get_group(k)
if not group:
continue
# Load group hosts/vars/children from a dictionary.
if isinstance(v, dict):
@ -334,6 +349,8 @@ class ExecutableJsonLoader(BaseLoader):
if isinstance(hosts, dict):
for hk, hv in hosts.iteritems():
host = self.get_host(hk)
if not host:
continue
if isinstance(hv, dict):
host.variables.update(hv)
else:
@ -344,6 +361,8 @@ class ExecutableJsonLoader(BaseLoader):
elif isinstance(hosts, (list, tuple)):
for hk in hosts:
host = self.get_host(hk)
if not host:
continue
group.add_host(host)
else:
logger.warning('Expected dict or list of "hosts" for '
@ -362,7 +381,8 @@ class ExecutableJsonLoader(BaseLoader):
if isinstance(children, (list, tuple)):
for c in children:
child = self.get_group(c, self.all_group, child=True)
group.add_child_group(child)
if child:
group.add_child_group(child)
else:
self.logger.warning('Expected list of children for '
'group "%s", got %s instead',
@ -372,6 +392,8 @@ class ExecutableJsonLoader(BaseLoader):
elif isinstance(v, (list, tuple)):
for h in v:
host = self.get_host(h)
if not host:
continue
group.add_host(host)
else:
logger.warning('')
@ -396,7 +418,8 @@ class ExecutableJsonLoader(BaseLoader):
k, str(type(data)))
def load_inventory_source(source, all_group=None):
def load_inventory_source(source, all_group=None, group_filter_re=None,
host_filter_re=None, exclude_empty_groups=False):
'''
Load inventory from given source directory or file.
'''
@ -409,15 +432,25 @@ def load_inventory_source(source, all_group=None):
for filename in glob.glob(os.path.join(source, '*')):
if filename.endswith(".ini") or os.path.isdir(filename):
continue
load_inventory_source(filename, all_group)
load_inventory_source(filename, all_group, group_filter_re,
host_filter_re)
else:
all_group = all_group or MemGroup('all', os.path.dirname(source))
if os.access(source, os.X_OK):
ExecutableJsonLoader(source, all_group).load()
ExecutableJsonLoader(source, all_group, group_filter_re, host_filter_re).load()
else:
IniLoader(source, all_group).load()
IniLoader(source, all_group, group_filter_re, host_filter_re).load()
logger.debug('Finished loading from source: %s', source)
# Exclude groups that are completely empty.
if original_all_group is None and exclude_empty_groups:
for name, group in all_group.all_groups.items():
if not group.children and not group.hosts and not group.variables:
logger.debug('Removing empty group %s', name)
for parent in group.parents:
if group in parent.children:
parent.children.remove(group)
del all_group.all_groups[name]
if original_all_group is None:
logger.info('Loaded %d groups, %d hosts', len(all_group.all_groups),
len(all_group.all_hosts))
@ -457,6 +490,16 @@ class Command(NoArgsCommand):
default=None, metavar='v', help='value of host variable '
'specified by --enabled-var that indicates host is '
'enabled/online.'),
make_option('--group-filter', dest='group_filter', type='str',
default=None, metavar='regex', help='regular expression '
'to filter group name(s); only matches are imported.'),
make_option('--host-filter', dest='host_filter', type='str',
default=None, metavar='regex', help='regular expression '
'to filter host name(s); only matches are imported.'),
make_option('--exclude-empty-groups', dest='exclude_empty_groups',
action='store_true', default=False, help='when set, '
'exclude all groups that have no child groups, hosts, or '
'variables.'),
)
def init_logging(self):
@ -768,6 +811,9 @@ class Command(NoArgsCommand):
self.source = options.get('source', None)
self.enabled_var = options.get('enabled_var', None)
self.enabled_value = options.get('enabled_value', None)
self.group_filter = options.get('group_filter', None) or r'^.+$'
self.host_filter = options.get('host_filter', None) or r'^.+$'
self.exclude_empty_groups = bool(options.get('exclude_empty_groups', False))
# Load inventory and related objects from database.
if self.inventory_name and self.inventory_id:
@ -778,6 +824,14 @@ class Command(NoArgsCommand):
raise CommandError('--overwrite/--overwrite-vars and --keep-vars are mutually exclusive')
if not self.source:
raise CommandError('--source is required')
try:
self.group_filter_re = re.compile(self.group_filter)
except re.error:
raise CommandError('invalid regular expression for --group-filter')
try:
self.host_filter_re = re.compile(self.host_filter)
except re.error:
raise CommandError('invalid regular expression for --host-filter')
self.check_license()
begin = time.time()
@ -793,7 +847,10 @@ class Command(NoArgsCommand):
transaction.commit()
# Load inventory from source.
self.all_group = load_inventory_source(self.source)
self.all_group = load_inventory_source(self.source, None,
self.group_filter_re,
self.host_filter_re,
self.exclude_empty_groups)
self.all_group.debug_tree()
# Merge/overwrite inventory into database.

View File

@ -823,15 +823,23 @@ class RunInventoryUpdate(BaseTask):
if inventory_update.source == 'ec2':
ec2_path = self.get_path_to('..', 'plugins', 'inventory', 'ec2.py')
args.append(ec2_path)
args.extend(['--enabled-var', 'ec2_state'])
args.extend(['--enabled-value', 'running'])
#args.extend(['--instance-id', 'ec2_id'])
args.extend(['--enabled-var', settings.EC2_ENABLED_VAR])
args.extend(['--enabled-value', settings.EC2_ENABLED_VALUE])
args.extend(['--group-filter', settings.EC2_GROUP_FILTER])
args.extend(['--host-filter', settings.EC2_HOST_FILTER])
if settings.EC2_EXCLUDE_EMPTY_GROUPS:
args.append('--exclude-empty-groups')
#args.extend(['--instance-id', settings.EC2_INSTANCE_ID_VAR])
elif inventory_update.source == 'rax':
rax_path = self.get_path_to('..', 'plugins', 'inventory', 'rax.py')
args.append(rax_path)
args.extend(['--enabled-var', 'rax_status'])
args.extend(['--enabled-value', 'ACTIVE'])
#args.extend(['--instance-id', 'rax_id'])
args.extend(['--enabled-var', settings.RAX_ENABLED_VAR])
args.extend(['--enabled-value', settings.RAX_ENABLED_VALUE])
args.extend(['--group-filter', settings.RAX_GROUP_FILTER])
args.extend(['--host-filter', settings.RAX_HOST_FILTER])
if settings.RAX_EXCLUDE_EMPTY_GROUPS:
args.append('--exclude-empty-groups')
#args.extend(['--instance-id', settings.RAX_INSTANCE_ID_VAR])
elif inventory_update.source == 'file':
args.append(inventory_update.source_path)
verbosity = getattr(settings, 'INVENTORY_UPDATE_VERBOSITY', 1)

View File

@ -1066,6 +1066,10 @@ class InventoryUpdatesTest(BaseTransactionTest):
self.assertTrue(inventory_source.pk in source_pks)
self.assertTrue(host.has_inventory_sources)
self.assertTrue(host.enabled)
# Make sure EC2 RDS hosts are excluded.
if inventory_source.source == 'ec2':
self.assertFalse(re.match(r'^.+\.rds\.amazonaws\.com$', host.name, re.I),
host.name)
with self.current_user(self.super_django_user):
url = reverse('api:host_inventory_sources_list', args=(host.pk,))
response = self.get(url, expect=200)
@ -1074,9 +1078,18 @@ class InventoryUpdatesTest(BaseTransactionTest):
source_pks = group.inventory_sources.values_list('pk', flat=True)
self.assertTrue(inventory_source.pk in source_pks)
self.assertTrue(group.has_inventory_sources)
# Make sure EC2 instance ID groups are excluded.
self.assertFalse(re.match(r'^i-[0-9a-f]{8}$', group.name, re.I),
group.name)
self.assertTrue(group.children.filter(active=True).exists() or
group.hosts.filter(active=True).exists())
# Make sure EC2 instance ID groups and RDS groups are excluded.
if inventory_source.source == 'ec2':
self.assertFalse(re.match(r'^i-[0-9a-f]{8}$', group.name, re.I),
group.name)
self.assertFalse(re.match(r'^rds|rds_.+|type_db_.+$', group.name, re.I),
group.name)
# Make sure Rackspace instance ID groups are excluded.
if inventory_source.source == 'rax':
self.assertFalse(re.match(r'^instance-.+$', group.name, re.I),
group.name)
with self.current_user(self.super_django_user):
url = reverse('api:group_inventory_sources_list', args=(group.pk,))
response = self.get(url, expect=200)

View File

@ -254,8 +254,7 @@ class Ec2Inventory(object):
for region in self.regions:
self.get_instances_by_region(region)
# Don't return RDS instances for Ansible Tower!
#self.get_rds_instances_by_region(region)
self.get_rds_instances_by_region(region)
self.write_to_cache(self.inventory, self.cache_path_cache)
self.write_to_cache(self.index, self.cache_path_index)
@ -344,9 +343,8 @@ class Ec2Inventory(object):
# Add to index
self.index[dest] = [region, instance.id]
# Do not output group based on instance ID for Ansible Tower!
# Inventory: Group by instance ID (always a group of 1)
#self.inventory[instance.id] = [dest]
self.inventory[instance.id] = [dest]
# Inventory: Group by region
self.push(self.inventory, region, dest)

View File

@ -22,9 +22,11 @@ DOCUMENTATION = '''
inventory: rax
short_description: Rackspace Public Cloud external inventory script
description:
- Generates inventory that Ansible can understand by making API request to Rackspace Public Cloud API
- Generates inventory that Ansible can understand by making API request to
Rackspace Public Cloud API
- |
When run against a specific host, this script returns the following variables:
When run against a specific host, this script returns the following
variables:
rax_os-ext-sts_task_state
rax_addresses
rax_links
@ -65,12 +67,23 @@ options:
authors:
- Jesse Keating <jesse.keating@rackspace.com>
- Paul Durivage <paul.durivage@rackspace.com>
- Matt Martz <matt@sivel.net>
notes:
- RAX_CREDS_FILE is an optional environment variable that points to a pyrax-compatible credentials file.
- If RAX_CREDS_FILE is not supplied, rax.py will look for a credentials file at ~/.rackspace_cloud_credentials.
- RAX_CREDS_FILE is an optional environment variable that points to a
pyrax-compatible credentials file.
- If RAX_CREDS_FILE is not supplied, rax.py will look for a credentials file
at ~/.rackspace_cloud_credentials.
- See https://github.com/rackspace/pyrax/blob/master/docs/getting_started.md#authenticating
- RAX_REGION is an optional environment variable to narrow inventory search scope
- RAX_REGION, if used, needs a value like ORD, DFW, SYD (a Rackspace datacenter) and optionally accepts a comma-separated list
- RAX_REGION is an optional environment variable to narrow inventory search
scope
- RAX_REGION, if used, needs a value like ORD, DFW, SYD (a Rackspace
datacenter) and optionally accepts a comma-separated list
- RAX_ENV is an environment variable that will use an environment as
configured in ~/.pyrax.cfg, see
https://github.com/rackspace/pyrax/blob/master/docs/getting_started.md#pyrax-configuration
- RAX_META_PREFIX is an environment variable that changes the prefix used
for meta key/value groups. For compatibility with ec2.py set to
RAX_META_PREFIX=tag
requirements: [ "pyrax" ]
examples:
- description: List server instances
@ -83,13 +96,14 @@ examples:
code: RAX_CREDS_FILE=~/.raxpub rax.py --host server.example.com
'''
import sys
import re
import os
import re
import sys
import argparse
import collections
from types import NoneType
try:
import json
except:
@ -98,9 +112,26 @@ except:
try:
import pyrax
except ImportError:
print('pyrax required for this module')
print('pyrax is required for this module')
sys.exit(1)
NON_CALLABLES = (basestring, bool, dict, int, list, NoneType)
def rax_slugify(value):
return 'rax_%s' % (re.sub('[^\w-]', '_', value).lower().lstrip('_'))
def to_dict(obj):
instance = {}
for key in dir(obj):
value = getattr(obj, key)
if (isinstance(value, NON_CALLABLES) and not key.startswith('_')):
key = rax_slugify(key)
instance[key] = value
return instance
def host(regions, hostname):
hostvars = {}
@ -110,15 +141,7 @@ def host(regions, hostname):
cs = pyrax.connect_to_cloudservers(region=region)
for server in cs.servers.list():
if server.name == hostname:
keys = [key for key in vars(server) if key not in ('manager', '_info')]
for key in keys:
# Extract value
value = getattr(server, key)
# Generate sanitized key
key = 'rax_' + (re.sub("[^A-Za-z0-9\-]", "_", key)
.lower()
.lstrip("_"))
for key, value in to_dict(server).items():
hostvars[key] = value
# And finally, add an IP address
@ -129,6 +152,7 @@ def host(regions, hostname):
def _list(regions):
groups = collections.defaultdict(list)
hostvars = collections.defaultdict(dict)
images = {}
# Go through all the regions looking for servers
for region in regions:
@ -139,26 +163,39 @@ def _list(regions):
groups[region].append(server.name)
# Check if group metadata key in servers' metadata
try:
group = server.metadata['group']
except KeyError:
pass
else:
# Create group if not exist and add the server
group = server.metadata.get('group')
if group:
groups[group].append(server.name)
# Add host metadata
keys = [key for key in vars(server) if key not in ('manager', '_info')]
for key in keys:
# Extract value
value = getattr(server, key)
for extra_group in server.metadata.get('groups', '').split(','):
groups[extra_group].append(server.name)
# Generate sanitized key
key = 'rax_' + (re.sub("[^A-Za-z0-9\-]", "_", key)
.lower()
.lstrip('_'))
# Add host metadata
for key, value in to_dict(server).items():
hostvars[server.name][key] = value
hostvars[server.name]['rax_region'] = region
for key, value in server.metadata.iteritems():
prefix = os.getenv('RAX_META_PREFIX', 'meta')
groups['%s_%s_%s' % (prefix, key, value)].append(server.name)
groups['instance-%s' % server.id].append(server.name)
groups['flavor-%s' % server.flavor['id']].append(server.name)
try:
imagegroup = 'image-%s' % images[server.image['id']]
groups[imagegroup].append(server.name)
groups['image-%s' % server.image['id']].append(server.name)
except KeyError:
try:
image = cs.images.get(server.image['id'])
except cs.exceptions.NotFound:
groups['image-%s' % server.image['id']].append(server.name)
else:
images[image.id] = image.human_id
groups['image-%s' % image.human_id].append(server.name)
groups['image-%s' % server.image['id']].append(server.name)
# And finally, add an IP address
hostvars[server.name]['ansible_ssh_host'] = server.accessIPv4
@ -172,7 +209,7 @@ def parse_args():
'inventory module')
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('--list', action='store_true',
help='List active servers')
help='List active servers')
group.add_argument('--host', help='List details about the specific host')
return parser.parse_args()
@ -180,38 +217,54 @@ def parse_args():
def setup():
default_creds_file = os.path.expanduser('~/.rackspace_cloud_credentials')
env = os.getenv('RAX_ENV', None)
if env:
pyrax.set_environment(env)
keyring_username = pyrax.get_setting('keyring_username')
# Attempt to grab credentials from environment first
try:
creds_file = os.environ['RAX_CREDS_FILE']
creds_file = os.path.expanduser(os.environ['RAX_CREDS_FILE'])
except KeyError, e:
# But if that fails, use the default location of ~/.rackspace_cloud_credentials
# But if that fails, use the default location of
# ~/.rackspace_cloud_credentials
if os.path.isfile(default_creds_file):
creds_file = default_creds_file
else:
elif not keyring_username:
sys.stderr.write('No value in environment variable %s and/or no '
'credentials file at %s\n'
% (e.message, default_creds_file))
sys.exit(1)
pyrax.set_setting('identity_type', 'rackspace')
identity_type = pyrax.get_setting('identity_type')
pyrax.set_setting('identity_type', identity_type or 'rackspace')
region = pyrax.get_setting('region')
try:
pyrax.set_credential_file(os.path.expanduser(creds_file))
if keyring_username:
pyrax.keyring_auth(keyring_username, region=region)
else:
pyrax.set_credential_file(creds_file, region=region)
except Exception, e:
sys.stderr.write("%s: %s\n" % (e, e.message))
sys.exit(1)
regions = []
for region in os.getenv('RAX_REGION', 'all').split(','):
region = region.strip().upper()
if region == 'ALL':
regions = pyrax.regions
break
elif region not in pyrax.regions:
sys.stderr.write('Unsupported region %s' % region)
sys.exit(1)
elif region not in regions:
regions.append(region)
if region:
regions.append(region)
else:
for region in os.getenv('RAX_REGION', 'all').split(','):
region = region.strip().upper()
if region == 'ALL':
regions = pyrax.regions
break
elif region not in pyrax.regions:
sys.stderr.write('Unsupported region %s' % region)
sys.exit(1)
elif region not in regions:
regions.append(region)
return regions
@ -225,5 +278,6 @@ def main():
host(regions, args.host)
sys.exit(0)
if __name__ == '__main__':
main()
main()

View File

@ -323,6 +323,20 @@ RAX_REGION_CHOICES = [
('HKG', 'Hong Kong'),
]
# Inventory variable name/values for determining if host is active/enabled.
RAX_ENABLED_VAR = 'rax_status'
RAX_ENABLED_VALUE = 'ACTIVE'
# Inventory variable name containing unique instance ID.
RAX_INSTANCE_ID_VAR = 'rax_id'
# Filter for allowed group/host names when importing inventory from Rackspace.
# By default, filter group of one created for each instance and exclude all
# groups without children, hosts and variables.
RAX_GROUP_FILTER = r'^(?!instance-.+).+$'
RAX_HOST_FILTER = r'^.+$'
RAX_EXCLUDE_EMPTY_GROUPS = True
# AWS does not appear to provide pretty region names via any API, so store the
# list of names here. The available region IDs will be pulled from boto.
# http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region
@ -343,6 +357,20 @@ EC2_REGIONS_BLACKLIST = [
'cn-north-1',
]
# Inventory variable name/values for determining if host is active/enabled.
EC2_ENABLED_VAR = 'ec2_state'
EC2_ENABLED_VALUE = 'running'
# Inventory variable name containing unique instance ID.
EC2_INSTANCE_ID_VAR = 'ec2_id'
# Filter for allowed group/host names when importing inventory from EC2.
# By default, filter group of one created for each instance, filter all RDS
# hosts, and exclude all groups without children, hosts and variables.
EC2_GROUP_FILTER = r'^(?!i-[a-f0-9]{8,}).+$'
EC2_HOST_FILTER = r'^.+(?<!rds\.amazonaws\.com)$'
EC2_EXCLUDE_EMPTY_GROUPS = True
# Defaults for enabling/disabling activity stream.
ACTIVITY_STREAM_ENABLED = True
ACTIVITY_STREAM_ENABLED_FOR_INVENTORY_SYNC = False