mirror of
https://github.com/ansible/awx.git
synced 2026-03-05 10:41:05 -03:30
AC-1134 Updated rax/ec2 inventory scripts from ansible. Added ability to filter groups/hosts by regular expression and exclude empty groups, so that Tower can exclude RAX/EC2 instance ID groups and EC2 RDS hosts/groups.
This commit is contained in:
@@ -46,8 +46,8 @@ class MemObject(object):
|
|||||||
'''
|
'''
|
||||||
|
|
||||||
def __init__(self, name, source_dir):
|
def __init__(self, name, source_dir):
|
||||||
assert name
|
assert name, 'no name'
|
||||||
assert source_dir
|
assert source_dir, 'no source dir'
|
||||||
self.name = name
|
self.name = name
|
||||||
self.source_dir = source_dir
|
self.source_dir = source_dir
|
||||||
|
|
||||||
@@ -95,17 +95,18 @@ class MemGroup(MemObject):
|
|||||||
logger.debug('Looking for %s as child group of %s', name, self.name)
|
logger.debug('Looking for %s as child group of %s', name, self.name)
|
||||||
# slight hack here, passing in 'self' for all_group but child=True won't use it
|
# slight hack here, passing in 'self' for all_group but child=True won't use it
|
||||||
group = loader.get_group(name, self, child=True)
|
group = loader.get_group(name, self, child=True)
|
||||||
# don't add to child groups if already there
|
if group:
|
||||||
for g in self.children:
|
# don't add to child groups if already there
|
||||||
if g.name == name:
|
for g in self.children:
|
||||||
return g
|
if g.name == name:
|
||||||
logger.debug('Adding child group %s to group %s', group.name, self.name)
|
return g
|
||||||
self.children.append(group)
|
logger.debug('Adding child group %s to group %s', group.name, self.name)
|
||||||
|
self.children.append(group)
|
||||||
return group
|
return group
|
||||||
|
|
||||||
def add_child_group(self, group):
|
def add_child_group(self, group):
|
||||||
assert group.name is not 'all'
|
assert group.name is not 'all', 'group name is all'
|
||||||
assert isinstance(group, MemGroup)
|
assert isinstance(group, MemGroup), 'not MemGroup instance'
|
||||||
logger.debug('Adding child group %s to parent %s', group.name, self.name)
|
logger.debug('Adding child group %s to parent %s', group.name, self.name)
|
||||||
if group not in self.children:
|
if group not in self.children:
|
||||||
self.children.append(group)
|
self.children.append(group)
|
||||||
@@ -113,7 +114,7 @@ class MemGroup(MemObject):
|
|||||||
group.parents.append(self)
|
group.parents.append(self)
|
||||||
|
|
||||||
def add_host(self, host):
|
def add_host(self, host):
|
||||||
assert isinstance(host, MemHost)
|
assert isinstance(host, MemHost), 'not MemHost instance'
|
||||||
logger.debug('Adding host %s to group %s', host.name, self.name)
|
logger.debug('Adding host %s to group %s', host.name, self.name)
|
||||||
if host not in self.hosts:
|
if host not in self.hosts:
|
||||||
self.hosts.append(host)
|
self.hosts.append(host)
|
||||||
@@ -156,10 +157,12 @@ class BaseLoader(object):
|
|||||||
Common functions for an inventory loader from a given source.
|
Common functions for an inventory loader from a given source.
|
||||||
'''
|
'''
|
||||||
|
|
||||||
def __init__(self, source, all_group=None):
|
def __init__(self, source, all_group=None, group_filter_re=None, host_filter_re=None):
|
||||||
self.source = source
|
self.source = source
|
||||||
self.source_dir = os.path.dirname(self.source)
|
self.source_dir = os.path.dirname(self.source)
|
||||||
self.all_group = all_group or MemGroup('all', self.source_dir)
|
self.all_group = all_group or MemGroup('all', self.source_dir)
|
||||||
|
self.group_filter_re = group_filter_re
|
||||||
|
self.host_filter_re = host_filter_re
|
||||||
|
|
||||||
def get_host(self, name):
|
def get_host(self, name):
|
||||||
'''
|
'''
|
||||||
@@ -168,6 +171,9 @@ class BaseLoader(object):
|
|||||||
if '[' in name or ']' in name:
|
if '[' in name or ']' in name:
|
||||||
raise ValueError('host ranges like %s are not supported by this importer' % name)
|
raise ValueError('host ranges like %s are not supported by this importer' % name)
|
||||||
host_name = name.split(':')[0]
|
host_name = name.split(':')[0]
|
||||||
|
if self.host_filter_re and not self.host_filter_re.match(host_name):
|
||||||
|
logger.debug('Filtering host %s', host_name)
|
||||||
|
return None
|
||||||
host = None
|
host = None
|
||||||
if not host_name in self.all_group.all_hosts:
|
if not host_name in self.all_group.all_hosts:
|
||||||
host = MemHost(name, self.source_dir)
|
host = MemHost(name, self.source_dir)
|
||||||
@@ -219,6 +225,9 @@ class BaseLoader(object):
|
|||||||
all_group = all_group or self.all_group
|
all_group = all_group or self.all_group
|
||||||
if name == 'all':
|
if name == 'all':
|
||||||
return all_group
|
return all_group
|
||||||
|
if self.group_filter_re and not self.group_filter_re.match(name):
|
||||||
|
logger.debug('Filtering group %s', name)
|
||||||
|
return None
|
||||||
if not name in self.all_group.all_groups:
|
if not name in self.all_group.all_groups:
|
||||||
group = MemGroup(name, self.source_dir)
|
group = MemGroup(name, self.source_dir)
|
||||||
if not child:
|
if not child:
|
||||||
@@ -244,34 +253,38 @@ class IniLoader(BaseLoader):
|
|||||||
if not line:
|
if not line:
|
||||||
continue
|
continue
|
||||||
elif line.startswith('[') and line.endswith(']'):
|
elif line.startswith('[') and line.endswith(']'):
|
||||||
# Mode change, possible new group name
|
# Mode change, possible new group name
|
||||||
line = line[1:-1].strip()
|
line = line[1:-1].strip()
|
||||||
if line.endswith(':vars'):
|
if line.endswith(':vars'):
|
||||||
input_mode = 'vars'
|
input_mode = 'vars'
|
||||||
line = line[:-5]
|
line = line[:-5]
|
||||||
elif line.endswith(':children'):
|
elif line.endswith(':children'):
|
||||||
input_mode = 'children'
|
input_mode = 'children'
|
||||||
line = line[:-9]
|
line = line[:-9]
|
||||||
else:
|
else:
|
||||||
input_mode = 'host'
|
input_mode = 'host'
|
||||||
group = self.get_group(line)
|
group = self.get_group(line)
|
||||||
else:
|
elif group:
|
||||||
# Add hosts with inline variables, or variables/children to
|
# If group is None, we are skipping this group and shouldn't
|
||||||
# an existing group.
|
# capture any children/variables/hosts under it.
|
||||||
tokens = shlex.split(line)
|
# Add hosts with inline variables, or variables/children to
|
||||||
if input_mode == 'host':
|
# an existing group.
|
||||||
for host in self.get_hosts(tokens[0]):
|
tokens = shlex.split(line)
|
||||||
if len(tokens) > 1:
|
if input_mode == 'host':
|
||||||
for t in tokens[1:]:
|
for host in self.get_hosts(tokens[0]):
|
||||||
k,v = t.split('=', 1)
|
if not host:
|
||||||
host.variables[k] = v
|
continue
|
||||||
group.add_host(host)
|
if len(tokens) > 1:
|
||||||
elif input_mode == 'children':
|
for t in tokens[1:]:
|
||||||
group.child_group_by_name(line, self)
|
k,v = t.split('=', 1)
|
||||||
elif input_mode == 'vars':
|
host.variables[k] = v
|
||||||
for t in tokens:
|
group.add_host(host)
|
||||||
k, v = t.split('=', 1)
|
elif input_mode == 'children':
|
||||||
group.variables[k] = v
|
group.child_group_by_name(line, self)
|
||||||
|
elif input_mode == 'vars':
|
||||||
|
for t in tokens:
|
||||||
|
k, v = t.split('=', 1)
|
||||||
|
group.variables[k] = v
|
||||||
# TODO: expansion patterns are probably not going to be supported. YES THEY ARE!
|
# TODO: expansion patterns are probably not going to be supported. YES THEY ARE!
|
||||||
|
|
||||||
|
|
||||||
@@ -326,6 +339,8 @@ class ExecutableJsonLoader(BaseLoader):
|
|||||||
|
|
||||||
for k,v in data.iteritems():
|
for k,v in data.iteritems():
|
||||||
group = self.get_group(k)
|
group = self.get_group(k)
|
||||||
|
if not group:
|
||||||
|
continue
|
||||||
|
|
||||||
# Load group hosts/vars/children from a dictionary.
|
# Load group hosts/vars/children from a dictionary.
|
||||||
if isinstance(v, dict):
|
if isinstance(v, dict):
|
||||||
@@ -334,6 +349,8 @@ class ExecutableJsonLoader(BaseLoader):
|
|||||||
if isinstance(hosts, dict):
|
if isinstance(hosts, dict):
|
||||||
for hk, hv in hosts.iteritems():
|
for hk, hv in hosts.iteritems():
|
||||||
host = self.get_host(hk)
|
host = self.get_host(hk)
|
||||||
|
if not host:
|
||||||
|
continue
|
||||||
if isinstance(hv, dict):
|
if isinstance(hv, dict):
|
||||||
host.variables.update(hv)
|
host.variables.update(hv)
|
||||||
else:
|
else:
|
||||||
@@ -344,6 +361,8 @@ class ExecutableJsonLoader(BaseLoader):
|
|||||||
elif isinstance(hosts, (list, tuple)):
|
elif isinstance(hosts, (list, tuple)):
|
||||||
for hk in hosts:
|
for hk in hosts:
|
||||||
host = self.get_host(hk)
|
host = self.get_host(hk)
|
||||||
|
if not host:
|
||||||
|
continue
|
||||||
group.add_host(host)
|
group.add_host(host)
|
||||||
else:
|
else:
|
||||||
logger.warning('Expected dict or list of "hosts" for '
|
logger.warning('Expected dict or list of "hosts" for '
|
||||||
@@ -362,7 +381,8 @@ class ExecutableJsonLoader(BaseLoader):
|
|||||||
if isinstance(children, (list, tuple)):
|
if isinstance(children, (list, tuple)):
|
||||||
for c in children:
|
for c in children:
|
||||||
child = self.get_group(c, self.all_group, child=True)
|
child = self.get_group(c, self.all_group, child=True)
|
||||||
group.add_child_group(child)
|
if child:
|
||||||
|
group.add_child_group(child)
|
||||||
else:
|
else:
|
||||||
self.logger.warning('Expected list of children for '
|
self.logger.warning('Expected list of children for '
|
||||||
'group "%s", got %s instead',
|
'group "%s", got %s instead',
|
||||||
@@ -372,6 +392,8 @@ class ExecutableJsonLoader(BaseLoader):
|
|||||||
elif isinstance(v, (list, tuple)):
|
elif isinstance(v, (list, tuple)):
|
||||||
for h in v:
|
for h in v:
|
||||||
host = self.get_host(h)
|
host = self.get_host(h)
|
||||||
|
if not host:
|
||||||
|
continue
|
||||||
group.add_host(host)
|
group.add_host(host)
|
||||||
else:
|
else:
|
||||||
logger.warning('')
|
logger.warning('')
|
||||||
@@ -396,7 +418,8 @@ class ExecutableJsonLoader(BaseLoader):
|
|||||||
k, str(type(data)))
|
k, str(type(data)))
|
||||||
|
|
||||||
|
|
||||||
def load_inventory_source(source, all_group=None):
|
def load_inventory_source(source, all_group=None, group_filter_re=None,
|
||||||
|
host_filter_re=None, exclude_empty_groups=False):
|
||||||
'''
|
'''
|
||||||
Load inventory from given source directory or file.
|
Load inventory from given source directory or file.
|
||||||
'''
|
'''
|
||||||
@@ -409,15 +432,25 @@ def load_inventory_source(source, all_group=None):
|
|||||||
for filename in glob.glob(os.path.join(source, '*')):
|
for filename in glob.glob(os.path.join(source, '*')):
|
||||||
if filename.endswith(".ini") or os.path.isdir(filename):
|
if filename.endswith(".ini") or os.path.isdir(filename):
|
||||||
continue
|
continue
|
||||||
load_inventory_source(filename, all_group)
|
load_inventory_source(filename, all_group, group_filter_re,
|
||||||
|
host_filter_re)
|
||||||
else:
|
else:
|
||||||
all_group = all_group or MemGroup('all', os.path.dirname(source))
|
all_group = all_group or MemGroup('all', os.path.dirname(source))
|
||||||
if os.access(source, os.X_OK):
|
if os.access(source, os.X_OK):
|
||||||
ExecutableJsonLoader(source, all_group).load()
|
ExecutableJsonLoader(source, all_group, group_filter_re, host_filter_re).load()
|
||||||
else:
|
else:
|
||||||
IniLoader(source, all_group).load()
|
IniLoader(source, all_group, group_filter_re, host_filter_re).load()
|
||||||
|
|
||||||
logger.debug('Finished loading from source: %s', source)
|
logger.debug('Finished loading from source: %s', source)
|
||||||
|
# Exclude groups that are completely empty.
|
||||||
|
if original_all_group is None and exclude_empty_groups:
|
||||||
|
for name, group in all_group.all_groups.items():
|
||||||
|
if not group.children and not group.hosts and not group.variables:
|
||||||
|
logger.debug('Removing empty group %s', name)
|
||||||
|
for parent in group.parents:
|
||||||
|
if group in parent.children:
|
||||||
|
parent.children.remove(group)
|
||||||
|
del all_group.all_groups[name]
|
||||||
if original_all_group is None:
|
if original_all_group is None:
|
||||||
logger.info('Loaded %d groups, %d hosts', len(all_group.all_groups),
|
logger.info('Loaded %d groups, %d hosts', len(all_group.all_groups),
|
||||||
len(all_group.all_hosts))
|
len(all_group.all_hosts))
|
||||||
@@ -457,6 +490,16 @@ class Command(NoArgsCommand):
|
|||||||
default=None, metavar='v', help='value of host variable '
|
default=None, metavar='v', help='value of host variable '
|
||||||
'specified by --enabled-var that indicates host is '
|
'specified by --enabled-var that indicates host is '
|
||||||
'enabled/online.'),
|
'enabled/online.'),
|
||||||
|
make_option('--group-filter', dest='group_filter', type='str',
|
||||||
|
default=None, metavar='regex', help='regular expression '
|
||||||
|
'to filter group name(s); only matches are imported.'),
|
||||||
|
make_option('--host-filter', dest='host_filter', type='str',
|
||||||
|
default=None, metavar='regex', help='regular expression '
|
||||||
|
'to filter host name(s); only matches are imported.'),
|
||||||
|
make_option('--exclude-empty-groups', dest='exclude_empty_groups',
|
||||||
|
action='store_true', default=False, help='when set, '
|
||||||
|
'exclude all groups that have no child groups, hosts, or '
|
||||||
|
'variables.'),
|
||||||
)
|
)
|
||||||
|
|
||||||
def init_logging(self):
|
def init_logging(self):
|
||||||
@@ -768,6 +811,9 @@ class Command(NoArgsCommand):
|
|||||||
self.source = options.get('source', None)
|
self.source = options.get('source', None)
|
||||||
self.enabled_var = options.get('enabled_var', None)
|
self.enabled_var = options.get('enabled_var', None)
|
||||||
self.enabled_value = options.get('enabled_value', None)
|
self.enabled_value = options.get('enabled_value', None)
|
||||||
|
self.group_filter = options.get('group_filter', None) or r'^.+$'
|
||||||
|
self.host_filter = options.get('host_filter', None) or r'^.+$'
|
||||||
|
self.exclude_empty_groups = bool(options.get('exclude_empty_groups', False))
|
||||||
|
|
||||||
# Load inventory and related objects from database.
|
# Load inventory and related objects from database.
|
||||||
if self.inventory_name and self.inventory_id:
|
if self.inventory_name and self.inventory_id:
|
||||||
@@ -778,6 +824,14 @@ class Command(NoArgsCommand):
|
|||||||
raise CommandError('--overwrite/--overwrite-vars and --keep-vars are mutually exclusive')
|
raise CommandError('--overwrite/--overwrite-vars and --keep-vars are mutually exclusive')
|
||||||
if not self.source:
|
if not self.source:
|
||||||
raise CommandError('--source is required')
|
raise CommandError('--source is required')
|
||||||
|
try:
|
||||||
|
self.group_filter_re = re.compile(self.group_filter)
|
||||||
|
except re.error:
|
||||||
|
raise CommandError('invalid regular expression for --group-filter')
|
||||||
|
try:
|
||||||
|
self.host_filter_re = re.compile(self.host_filter)
|
||||||
|
except re.error:
|
||||||
|
raise CommandError('invalid regular expression for --host-filter')
|
||||||
|
|
||||||
self.check_license()
|
self.check_license()
|
||||||
begin = time.time()
|
begin = time.time()
|
||||||
@@ -793,7 +847,10 @@ class Command(NoArgsCommand):
|
|||||||
transaction.commit()
|
transaction.commit()
|
||||||
|
|
||||||
# Load inventory from source.
|
# Load inventory from source.
|
||||||
self.all_group = load_inventory_source(self.source)
|
self.all_group = load_inventory_source(self.source, None,
|
||||||
|
self.group_filter_re,
|
||||||
|
self.host_filter_re,
|
||||||
|
self.exclude_empty_groups)
|
||||||
self.all_group.debug_tree()
|
self.all_group.debug_tree()
|
||||||
|
|
||||||
# Merge/overwrite inventory into database.
|
# Merge/overwrite inventory into database.
|
||||||
|
|||||||
@@ -823,15 +823,23 @@ class RunInventoryUpdate(BaseTask):
|
|||||||
if inventory_update.source == 'ec2':
|
if inventory_update.source == 'ec2':
|
||||||
ec2_path = self.get_path_to('..', 'plugins', 'inventory', 'ec2.py')
|
ec2_path = self.get_path_to('..', 'plugins', 'inventory', 'ec2.py')
|
||||||
args.append(ec2_path)
|
args.append(ec2_path)
|
||||||
args.extend(['--enabled-var', 'ec2_state'])
|
args.extend(['--enabled-var', settings.EC2_ENABLED_VAR])
|
||||||
args.extend(['--enabled-value', 'running'])
|
args.extend(['--enabled-value', settings.EC2_ENABLED_VALUE])
|
||||||
#args.extend(['--instance-id', 'ec2_id'])
|
args.extend(['--group-filter', settings.EC2_GROUP_FILTER])
|
||||||
|
args.extend(['--host-filter', settings.EC2_HOST_FILTER])
|
||||||
|
if settings.EC2_EXCLUDE_EMPTY_GROUPS:
|
||||||
|
args.append('--exclude-empty-groups')
|
||||||
|
#args.extend(['--instance-id', settings.EC2_INSTANCE_ID_VAR])
|
||||||
elif inventory_update.source == 'rax':
|
elif inventory_update.source == 'rax':
|
||||||
rax_path = self.get_path_to('..', 'plugins', 'inventory', 'rax.py')
|
rax_path = self.get_path_to('..', 'plugins', 'inventory', 'rax.py')
|
||||||
args.append(rax_path)
|
args.append(rax_path)
|
||||||
args.extend(['--enabled-var', 'rax_status'])
|
args.extend(['--enabled-var', settings.RAX_ENABLED_VAR])
|
||||||
args.extend(['--enabled-value', 'ACTIVE'])
|
args.extend(['--enabled-value', settings.RAX_ENABLED_VALUE])
|
||||||
#args.extend(['--instance-id', 'rax_id'])
|
args.extend(['--group-filter', settings.RAX_GROUP_FILTER])
|
||||||
|
args.extend(['--host-filter', settings.RAX_HOST_FILTER])
|
||||||
|
if settings.RAX_EXCLUDE_EMPTY_GROUPS:
|
||||||
|
args.append('--exclude-empty-groups')
|
||||||
|
#args.extend(['--instance-id', settings.RAX_INSTANCE_ID_VAR])
|
||||||
elif inventory_update.source == 'file':
|
elif inventory_update.source == 'file':
|
||||||
args.append(inventory_update.source_path)
|
args.append(inventory_update.source_path)
|
||||||
verbosity = getattr(settings, 'INVENTORY_UPDATE_VERBOSITY', 1)
|
verbosity = getattr(settings, 'INVENTORY_UPDATE_VERBOSITY', 1)
|
||||||
|
|||||||
@@ -1066,6 +1066,10 @@ class InventoryUpdatesTest(BaseTransactionTest):
|
|||||||
self.assertTrue(inventory_source.pk in source_pks)
|
self.assertTrue(inventory_source.pk in source_pks)
|
||||||
self.assertTrue(host.has_inventory_sources)
|
self.assertTrue(host.has_inventory_sources)
|
||||||
self.assertTrue(host.enabled)
|
self.assertTrue(host.enabled)
|
||||||
|
# Make sure EC2 RDS hosts are excluded.
|
||||||
|
if inventory_source.source == 'ec2':
|
||||||
|
self.assertFalse(re.match(r'^.+\.rds\.amazonaws\.com$', host.name, re.I),
|
||||||
|
host.name)
|
||||||
with self.current_user(self.super_django_user):
|
with self.current_user(self.super_django_user):
|
||||||
url = reverse('api:host_inventory_sources_list', args=(host.pk,))
|
url = reverse('api:host_inventory_sources_list', args=(host.pk,))
|
||||||
response = self.get(url, expect=200)
|
response = self.get(url, expect=200)
|
||||||
@@ -1074,9 +1078,18 @@ class InventoryUpdatesTest(BaseTransactionTest):
|
|||||||
source_pks = group.inventory_sources.values_list('pk', flat=True)
|
source_pks = group.inventory_sources.values_list('pk', flat=True)
|
||||||
self.assertTrue(inventory_source.pk in source_pks)
|
self.assertTrue(inventory_source.pk in source_pks)
|
||||||
self.assertTrue(group.has_inventory_sources)
|
self.assertTrue(group.has_inventory_sources)
|
||||||
# Make sure EC2 instance ID groups are excluded.
|
self.assertTrue(group.children.filter(active=True).exists() or
|
||||||
self.assertFalse(re.match(r'^i-[0-9a-f]{8}$', group.name, re.I),
|
group.hosts.filter(active=True).exists())
|
||||||
group.name)
|
# Make sure EC2 instance ID groups and RDS groups are excluded.
|
||||||
|
if inventory_source.source == 'ec2':
|
||||||
|
self.assertFalse(re.match(r'^i-[0-9a-f]{8}$', group.name, re.I),
|
||||||
|
group.name)
|
||||||
|
self.assertFalse(re.match(r'^rds|rds_.+|type_db_.+$', group.name, re.I),
|
||||||
|
group.name)
|
||||||
|
# Make sure Rackspace instance ID groups are excluded.
|
||||||
|
if inventory_source.source == 'rax':
|
||||||
|
self.assertFalse(re.match(r'^instance-.+$', group.name, re.I),
|
||||||
|
group.name)
|
||||||
with self.current_user(self.super_django_user):
|
with self.current_user(self.super_django_user):
|
||||||
url = reverse('api:group_inventory_sources_list', args=(group.pk,))
|
url = reverse('api:group_inventory_sources_list', args=(group.pk,))
|
||||||
response = self.get(url, expect=200)
|
response = self.get(url, expect=200)
|
||||||
|
|||||||
@@ -254,8 +254,7 @@ class Ec2Inventory(object):
|
|||||||
|
|
||||||
for region in self.regions:
|
for region in self.regions:
|
||||||
self.get_instances_by_region(region)
|
self.get_instances_by_region(region)
|
||||||
# Don't return RDS instances for Ansible Tower!
|
self.get_rds_instances_by_region(region)
|
||||||
#self.get_rds_instances_by_region(region)
|
|
||||||
|
|
||||||
self.write_to_cache(self.inventory, self.cache_path_cache)
|
self.write_to_cache(self.inventory, self.cache_path_cache)
|
||||||
self.write_to_cache(self.index, self.cache_path_index)
|
self.write_to_cache(self.index, self.cache_path_index)
|
||||||
@@ -344,9 +343,8 @@ class Ec2Inventory(object):
|
|||||||
# Add to index
|
# Add to index
|
||||||
self.index[dest] = [region, instance.id]
|
self.index[dest] = [region, instance.id]
|
||||||
|
|
||||||
# Do not output group based on instance ID for Ansible Tower!
|
|
||||||
# Inventory: Group by instance ID (always a group of 1)
|
# Inventory: Group by instance ID (always a group of 1)
|
||||||
#self.inventory[instance.id] = [dest]
|
self.inventory[instance.id] = [dest]
|
||||||
|
|
||||||
# Inventory: Group by region
|
# Inventory: Group by region
|
||||||
self.push(self.inventory, region, dest)
|
self.push(self.inventory, region, dest)
|
||||||
|
|||||||
@@ -22,9 +22,11 @@ DOCUMENTATION = '''
|
|||||||
inventory: rax
|
inventory: rax
|
||||||
short_description: Rackspace Public Cloud external inventory script
|
short_description: Rackspace Public Cloud external inventory script
|
||||||
description:
|
description:
|
||||||
- Generates inventory that Ansible can understand by making API request to Rackspace Public Cloud API
|
- Generates inventory that Ansible can understand by making API request to
|
||||||
|
Rackspace Public Cloud API
|
||||||
- |
|
- |
|
||||||
When run against a specific host, this script returns the following variables:
|
When run against a specific host, this script returns the following
|
||||||
|
variables:
|
||||||
rax_os-ext-sts_task_state
|
rax_os-ext-sts_task_state
|
||||||
rax_addresses
|
rax_addresses
|
||||||
rax_links
|
rax_links
|
||||||
@@ -65,12 +67,23 @@ options:
|
|||||||
authors:
|
authors:
|
||||||
- Jesse Keating <jesse.keating@rackspace.com>
|
- Jesse Keating <jesse.keating@rackspace.com>
|
||||||
- Paul Durivage <paul.durivage@rackspace.com>
|
- Paul Durivage <paul.durivage@rackspace.com>
|
||||||
|
- Matt Martz <matt@sivel.net>
|
||||||
notes:
|
notes:
|
||||||
- RAX_CREDS_FILE is an optional environment variable that points to a pyrax-compatible credentials file.
|
- RAX_CREDS_FILE is an optional environment variable that points to a
|
||||||
- If RAX_CREDS_FILE is not supplied, rax.py will look for a credentials file at ~/.rackspace_cloud_credentials.
|
pyrax-compatible credentials file.
|
||||||
|
- If RAX_CREDS_FILE is not supplied, rax.py will look for a credentials file
|
||||||
|
at ~/.rackspace_cloud_credentials.
|
||||||
- See https://github.com/rackspace/pyrax/blob/master/docs/getting_started.md#authenticating
|
- See https://github.com/rackspace/pyrax/blob/master/docs/getting_started.md#authenticating
|
||||||
- RAX_REGION is an optional environment variable to narrow inventory search scope
|
- RAX_REGION is an optional environment variable to narrow inventory search
|
||||||
- RAX_REGION, if used, needs a value like ORD, DFW, SYD (a Rackspace datacenter) and optionally accepts a comma-separated list
|
scope
|
||||||
|
- RAX_REGION, if used, needs a value like ORD, DFW, SYD (a Rackspace
|
||||||
|
datacenter) and optionally accepts a comma-separated list
|
||||||
|
- RAX_ENV is an environment variable that will use an environment as
|
||||||
|
configured in ~/.pyrax.cfg, see
|
||||||
|
https://github.com/rackspace/pyrax/blob/master/docs/getting_started.md#pyrax-configuration
|
||||||
|
- RAX_META_PREFIX is an environment variable that changes the prefix used
|
||||||
|
for meta key/value groups. For compatibility with ec2.py set to
|
||||||
|
RAX_META_PREFIX=tag
|
||||||
requirements: [ "pyrax" ]
|
requirements: [ "pyrax" ]
|
||||||
examples:
|
examples:
|
||||||
- description: List server instances
|
- description: List server instances
|
||||||
@@ -83,13 +96,14 @@ examples:
|
|||||||
code: RAX_CREDS_FILE=~/.raxpub rax.py --host server.example.com
|
code: RAX_CREDS_FILE=~/.raxpub rax.py --host server.example.com
|
||||||
'''
|
'''
|
||||||
|
|
||||||
import sys
|
|
||||||
import re
|
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
import argparse
|
import argparse
|
||||||
import collections
|
import collections
|
||||||
|
|
||||||
|
from types import NoneType
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import json
|
import json
|
||||||
except:
|
except:
|
||||||
@@ -98,9 +112,26 @@ except:
|
|||||||
try:
|
try:
|
||||||
import pyrax
|
import pyrax
|
||||||
except ImportError:
|
except ImportError:
|
||||||
print('pyrax required for this module')
|
print('pyrax is required for this module')
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
NON_CALLABLES = (basestring, bool, dict, int, list, NoneType)
|
||||||
|
|
||||||
|
|
||||||
|
def rax_slugify(value):
|
||||||
|
return 'rax_%s' % (re.sub('[^\w-]', '_', value).lower().lstrip('_'))
|
||||||
|
|
||||||
|
|
||||||
|
def to_dict(obj):
|
||||||
|
instance = {}
|
||||||
|
for key in dir(obj):
|
||||||
|
value = getattr(obj, key)
|
||||||
|
if (isinstance(value, NON_CALLABLES) and not key.startswith('_')):
|
||||||
|
key = rax_slugify(key)
|
||||||
|
instance[key] = value
|
||||||
|
|
||||||
|
return instance
|
||||||
|
|
||||||
|
|
||||||
def host(regions, hostname):
|
def host(regions, hostname):
|
||||||
hostvars = {}
|
hostvars = {}
|
||||||
@@ -110,15 +141,7 @@ def host(regions, hostname):
|
|||||||
cs = pyrax.connect_to_cloudservers(region=region)
|
cs = pyrax.connect_to_cloudservers(region=region)
|
||||||
for server in cs.servers.list():
|
for server in cs.servers.list():
|
||||||
if server.name == hostname:
|
if server.name == hostname:
|
||||||
keys = [key for key in vars(server) if key not in ('manager', '_info')]
|
for key, value in to_dict(server).items():
|
||||||
for key in keys:
|
|
||||||
# Extract value
|
|
||||||
value = getattr(server, key)
|
|
||||||
|
|
||||||
# Generate sanitized key
|
|
||||||
key = 'rax_' + (re.sub("[^A-Za-z0-9\-]", "_", key)
|
|
||||||
.lower()
|
|
||||||
.lstrip("_"))
|
|
||||||
hostvars[key] = value
|
hostvars[key] = value
|
||||||
|
|
||||||
# And finally, add an IP address
|
# And finally, add an IP address
|
||||||
@@ -129,6 +152,7 @@ def host(regions, hostname):
|
|||||||
def _list(regions):
|
def _list(regions):
|
||||||
groups = collections.defaultdict(list)
|
groups = collections.defaultdict(list)
|
||||||
hostvars = collections.defaultdict(dict)
|
hostvars = collections.defaultdict(dict)
|
||||||
|
images = {}
|
||||||
|
|
||||||
# Go through all the regions looking for servers
|
# Go through all the regions looking for servers
|
||||||
for region in regions:
|
for region in regions:
|
||||||
@@ -139,26 +163,39 @@ def _list(regions):
|
|||||||
groups[region].append(server.name)
|
groups[region].append(server.name)
|
||||||
|
|
||||||
# Check if group metadata key in servers' metadata
|
# Check if group metadata key in servers' metadata
|
||||||
try:
|
group = server.metadata.get('group')
|
||||||
group = server.metadata['group']
|
if group:
|
||||||
except KeyError:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
# Create group if not exist and add the server
|
|
||||||
groups[group].append(server.name)
|
groups[group].append(server.name)
|
||||||
|
|
||||||
# Add host metadata
|
for extra_group in server.metadata.get('groups', '').split(','):
|
||||||
keys = [key for key in vars(server) if key not in ('manager', '_info')]
|
groups[extra_group].append(server.name)
|
||||||
for key in keys:
|
|
||||||
# Extract value
|
|
||||||
value = getattr(server, key)
|
|
||||||
|
|
||||||
# Generate sanitized key
|
# Add host metadata
|
||||||
key = 'rax_' + (re.sub("[^A-Za-z0-9\-]", "_", key)
|
for key, value in to_dict(server).items():
|
||||||
.lower()
|
|
||||||
.lstrip('_'))
|
|
||||||
hostvars[server.name][key] = value
|
hostvars[server.name][key] = value
|
||||||
|
|
||||||
|
hostvars[server.name]['rax_region'] = region
|
||||||
|
|
||||||
|
for key, value in server.metadata.iteritems():
|
||||||
|
prefix = os.getenv('RAX_META_PREFIX', 'meta')
|
||||||
|
groups['%s_%s_%s' % (prefix, key, value)].append(server.name)
|
||||||
|
|
||||||
|
groups['instance-%s' % server.id].append(server.name)
|
||||||
|
groups['flavor-%s' % server.flavor['id']].append(server.name)
|
||||||
|
try:
|
||||||
|
imagegroup = 'image-%s' % images[server.image['id']]
|
||||||
|
groups[imagegroup].append(server.name)
|
||||||
|
groups['image-%s' % server.image['id']].append(server.name)
|
||||||
|
except KeyError:
|
||||||
|
try:
|
||||||
|
image = cs.images.get(server.image['id'])
|
||||||
|
except cs.exceptions.NotFound:
|
||||||
|
groups['image-%s' % server.image['id']].append(server.name)
|
||||||
|
else:
|
||||||
|
images[image.id] = image.human_id
|
||||||
|
groups['image-%s' % image.human_id].append(server.name)
|
||||||
|
groups['image-%s' % server.image['id']].append(server.name)
|
||||||
|
|
||||||
# And finally, add an IP address
|
# And finally, add an IP address
|
||||||
hostvars[server.name]['ansible_ssh_host'] = server.accessIPv4
|
hostvars[server.name]['ansible_ssh_host'] = server.accessIPv4
|
||||||
|
|
||||||
@@ -172,7 +209,7 @@ def parse_args():
|
|||||||
'inventory module')
|
'inventory module')
|
||||||
group = parser.add_mutually_exclusive_group(required=True)
|
group = parser.add_mutually_exclusive_group(required=True)
|
||||||
group.add_argument('--list', action='store_true',
|
group.add_argument('--list', action='store_true',
|
||||||
help='List active servers')
|
help='List active servers')
|
||||||
group.add_argument('--host', help='List details about the specific host')
|
group.add_argument('--host', help='List details about the specific host')
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
@@ -180,38 +217,54 @@ def parse_args():
|
|||||||
def setup():
|
def setup():
|
||||||
default_creds_file = os.path.expanduser('~/.rackspace_cloud_credentials')
|
default_creds_file = os.path.expanduser('~/.rackspace_cloud_credentials')
|
||||||
|
|
||||||
|
env = os.getenv('RAX_ENV', None)
|
||||||
|
if env:
|
||||||
|
pyrax.set_environment(env)
|
||||||
|
|
||||||
|
keyring_username = pyrax.get_setting('keyring_username')
|
||||||
|
|
||||||
# Attempt to grab credentials from environment first
|
# Attempt to grab credentials from environment first
|
||||||
try:
|
try:
|
||||||
creds_file = os.environ['RAX_CREDS_FILE']
|
creds_file = os.path.expanduser(os.environ['RAX_CREDS_FILE'])
|
||||||
except KeyError, e:
|
except KeyError, e:
|
||||||
# But if that fails, use the default location of ~/.rackspace_cloud_credentials
|
# But if that fails, use the default location of
|
||||||
|
# ~/.rackspace_cloud_credentials
|
||||||
if os.path.isfile(default_creds_file):
|
if os.path.isfile(default_creds_file):
|
||||||
creds_file = default_creds_file
|
creds_file = default_creds_file
|
||||||
else:
|
elif not keyring_username:
|
||||||
sys.stderr.write('No value in environment variable %s and/or no '
|
sys.stderr.write('No value in environment variable %s and/or no '
|
||||||
'credentials file at %s\n'
|
'credentials file at %s\n'
|
||||||
% (e.message, default_creds_file))
|
% (e.message, default_creds_file))
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
pyrax.set_setting('identity_type', 'rackspace')
|
identity_type = pyrax.get_setting('identity_type')
|
||||||
|
pyrax.set_setting('identity_type', identity_type or 'rackspace')
|
||||||
|
|
||||||
|
region = pyrax.get_setting('region')
|
||||||
|
|
||||||
try:
|
try:
|
||||||
pyrax.set_credential_file(os.path.expanduser(creds_file))
|
if keyring_username:
|
||||||
|
pyrax.keyring_auth(keyring_username, region=region)
|
||||||
|
else:
|
||||||
|
pyrax.set_credential_file(creds_file, region=region)
|
||||||
except Exception, e:
|
except Exception, e:
|
||||||
sys.stderr.write("%s: %s\n" % (e, e.message))
|
sys.stderr.write("%s: %s\n" % (e, e.message))
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
regions = []
|
regions = []
|
||||||
for region in os.getenv('RAX_REGION', 'all').split(','):
|
if region:
|
||||||
region = region.strip().upper()
|
regions.append(region)
|
||||||
if region == 'ALL':
|
else:
|
||||||
regions = pyrax.regions
|
for region in os.getenv('RAX_REGION', 'all').split(','):
|
||||||
break
|
region = region.strip().upper()
|
||||||
elif region not in pyrax.regions:
|
if region == 'ALL':
|
||||||
sys.stderr.write('Unsupported region %s' % region)
|
regions = pyrax.regions
|
||||||
sys.exit(1)
|
break
|
||||||
elif region not in regions:
|
elif region not in pyrax.regions:
|
||||||
regions.append(region)
|
sys.stderr.write('Unsupported region %s' % region)
|
||||||
|
sys.exit(1)
|
||||||
|
elif region not in regions:
|
||||||
|
regions.append(region)
|
||||||
|
|
||||||
return regions
|
return regions
|
||||||
|
|
||||||
@@ -225,5 +278,6 @@ def main():
|
|||||||
host(regions, args.host)
|
host(regions, args.host)
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
main()
|
main()
|
||||||
@@ -323,6 +323,20 @@ RAX_REGION_CHOICES = [
|
|||||||
('HKG', 'Hong Kong'),
|
('HKG', 'Hong Kong'),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# Inventory variable name/values for determining if host is active/enabled.
|
||||||
|
RAX_ENABLED_VAR = 'rax_status'
|
||||||
|
RAX_ENABLED_VALUE = 'ACTIVE'
|
||||||
|
|
||||||
|
# Inventory variable name containing unique instance ID.
|
||||||
|
RAX_INSTANCE_ID_VAR = 'rax_id'
|
||||||
|
|
||||||
|
# Filter for allowed group/host names when importing inventory from Rackspace.
|
||||||
|
# By default, filter group of one created for each instance and exclude all
|
||||||
|
# groups without children, hosts and variables.
|
||||||
|
RAX_GROUP_FILTER = r'^(?!instance-.+).+$'
|
||||||
|
RAX_HOST_FILTER = r'^.+$'
|
||||||
|
RAX_EXCLUDE_EMPTY_GROUPS = True
|
||||||
|
|
||||||
# AWS does not appear to provide pretty region names via any API, so store the
|
# AWS does not appear to provide pretty region names via any API, so store the
|
||||||
# list of names here. The available region IDs will be pulled from boto.
|
# list of names here. The available region IDs will be pulled from boto.
|
||||||
# http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region
|
# http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region
|
||||||
@@ -343,6 +357,20 @@ EC2_REGIONS_BLACKLIST = [
|
|||||||
'cn-north-1',
|
'cn-north-1',
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# Inventory variable name/values for determining if host is active/enabled.
|
||||||
|
EC2_ENABLED_VAR = 'ec2_state'
|
||||||
|
EC2_ENABLED_VALUE = 'running'
|
||||||
|
|
||||||
|
# Inventory variable name containing unique instance ID.
|
||||||
|
EC2_INSTANCE_ID_VAR = 'ec2_id'
|
||||||
|
|
||||||
|
# Filter for allowed group/host names when importing inventory from EC2.
|
||||||
|
# By default, filter group of one created for each instance, filter all RDS
|
||||||
|
# hosts, and exclude all groups without children, hosts and variables.
|
||||||
|
EC2_GROUP_FILTER = r'^(?!i-[a-f0-9]{8,}).+$'
|
||||||
|
EC2_HOST_FILTER = r'^.+(?<!rds\.amazonaws\.com)$'
|
||||||
|
EC2_EXCLUDE_EMPTY_GROUPS = True
|
||||||
|
|
||||||
# Defaults for enabling/disabling activity stream.
|
# Defaults for enabling/disabling activity stream.
|
||||||
ACTIVITY_STREAM_ENABLED = True
|
ACTIVITY_STREAM_ENABLED = True
|
||||||
ACTIVITY_STREAM_ENABLED_FOR_INVENTORY_SYNC = False
|
ACTIVITY_STREAM_ENABLED_FOR_INVENTORY_SYNC = False
|
||||||
|
|||||||
Reference in New Issue
Block a user