External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-01-11 18:09:57 -03:30
Code Issues Packages Projects Releases Wiki Activity

Merge remote branch 'upstream/master'

Browse Source
This commit is contained in:
chouseknecht 2013-10-24 18:39:49 +00:00
commit b2f2fb655d
10 changed files with 355 additions and 289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

602
awx/main/management/commands/inventory_import.py
View File

@ -10,6 +10,7 @@ import os
import shlex
import subprocess
import sys
import time
import traceback
# PyYAML
@ -26,191 +27,214 @@ from awx.main.licenses import LicenseReader
logger = logging.getLogger('awx.main.commands.inventory_import')
class ImportException(BaseException):
LICENSE_MESSAGE = '''\
Number of licensed instances exceeded, would bring available instances to %(new_count)d, system is licensed for %(available_instances)d.
See http://ansibleworks.com/ansibleworks-awx for license extension information.'''
def __init__(self, msg):
self.msg = msg
DEMO_LICENSE_MESSAGE = '''\
Demo mode free license count exceeded, would bring available instances to %(new_count)d, demo mode allows %(available_instances)d.
See http://ansibleworks.com/ansibleworks-awx for licensing information.'''
def __str__(self):
return "Import Error: %s" % msg
class MemGroup(object):
def __init__(self, name, inventory_base):
assert inventory_base is not None
self.inventory_base = inventory_base
class MemObject(object):
'''
Common code shared between in-memory groups and hosts.
'''
def __init__(self, name, source_dir):
assert name
assert source_dir
self.name = name
self.child_groups = []
self.source_dir = source_dir
def load_vars(self, path):
if os.path.exists(path) and os.path.isfile(path):
vars_name = os.path.basename(os.path.dirname(path))
logger.debug('Loading %s from %s', vars_name, path)
try:
v = yaml.safe_load(file(path, 'r').read())
return v if hasattr(v, 'items') else {}
except yaml.YAMLError, e:
if hasattr(e, 'problem_mark'):
logger.error('Invalid YAML in %s:%s col %s', path,
e.problem_mark.line + 1,
e.problem_mark.column + 1)
else:
logger.error('Error loading YAML from %s', path)
raise
return {}
class MemGroup(MemObject):
'''
In-memory representation of an inventory group.
'''
def __init__(self, name, source_dir):
super(MemGroup, self).__init__(name, source_dir)
self.children = []
self.hosts = []
self.variables = {}
self.parents = []
# Used on the "all" group in place of previous global variables.
# maps host and group names to hosts to prevent redudant additions
self.host_names = {}
self.group_names = {}
self.all_hosts = {}
self.all_groups = {}
group_vars = os.path.join(inventory_base, 'group_vars', name)
if os.path.exists(group_vars):
logger.debug("loading group_vars")
self.variables = yaml.load(open(group_vars).read())
def child_group_by_name(self, grp_name, loader):
logger.debug("looking for child group: %s" % grp_name)
if grp_name == 'all':
group_vars = os.path.join(self.source_dir, 'group_vars', self.name)
self.variables = self.load_vars(group_vars)
logger.debug('Loaded group: %s', self.name)
def child_group_by_name(self, name, loader):
if name == 'all':
return
logger.debug('Looking for %s as child group of %s', name, self.name)
# slight hack here, passing in 'self' for all_group but child=True won't use it
grp = loader.get_group(grp_name, self, child=True)
group = loader.get_group(name, self, child=True)
# don't add to child groups if already there
for x in self.child_groups:
if x.name == grp_name:
return x
logger.debug("adding child group %s to group %s" % (grp.name, self.name))
self.child_groups.append(grp)
return grp
for g in self.children:
if g.name == name:
return g
logger.debug('Adding child group %s to group %s', group.name, self.name)
self.children.append(group)
return group
def add_child_group(self, grp):
assert grp.name is not 'all'
logger.debug("adding child group %s to group %s" % (grp.name, self.name))
assert type(grp) == MemGroup
if grp not in self.child_groups:
self.child_groups.append(grp)
if not self in grp.parents:
grp.parents.append(self)
def add_child_group(self, group):
assert group.name is not 'all'
assert isinstance(group, MemGroup)
logger.debug('Adding child group %s to parent %s', group.name, self.name)
if group not in self.children:
self.children.append(group)
if not self in group.parents:
group.parents.append(self)
def add_host(self, host):
logger.debug("adding host %s to group %s" % (host.name, self.name))
assert type(host) == MemHost
assert isinstance(host, MemHost)
logger.debug('Adding host %s to group %s', host.name, self.name)
if host not in self.hosts:
self.hosts.append(host)
def debug_tree(self):
logger.debug("describing tree of group (%s)" % self.name)
logger.debug("group: %s, %s" % (self.name, self.variables))
for x in self.child_groups:
logger.debug(" child: %s" % (x.name))
for x in self.hosts:
logger.debug(" host: %s, %s" % (x.name, x.variables))
def debug_tree(self, group_names=None):
group_names = group_names or set()
if self.name in group_names:
return
logger.debug('Dumping tree for group "%s":', self.name)
logger.debug('- Vars: %r', self.variables)
for h in self.hosts:
logger.debug('- Host: %s, %r', h.name, h.variables)
for g in self.children:
logger.debug('- Child: %s', g.name)
logger.debug('----')
group_names.add(self.name)
for g in self.children:
g.debug_tree(group_names)
logger.debug("---")
for x in self.child_groups:
x.debug_tree()
class MemHost(object):
class MemHost(MemObject):
'''
In-memory representation of an inventory host.
'''
def __init__(self, name, inventory_base):
logger.debug("adding host name: %s" % name)
assert name is not None
assert inventory_base is not None
# set ansible_ssh_port if ":" in name
self.name = name
def __init__(self, name, source_dir):
super(MemHost, self).__init__(name, source_dir)
self.variables = {}
self.inventory_base = inventory_base
if ':' in name:
tokens = name.split(":")
self.name = tokens[0]
self.variables['ansible_ssh_port'] = int(tokens[1])
if "[" in name:
raise ImportException("block ranges like host[0:50].example.com are not yet supported by the importer")
if '[' in name:
raise ValueError('Block ranges like host[0:50].example.com are not yet supported by the importer')
host_vars = os.path.join(source_dir, 'host_vars', name)
self.variables.update(self.load_vars(host_vars))
logger.debug('Loaded host: %s', self.name)
host_vars = os.path.join(inventory_base, 'host_vars', name)
if os.path.exists(host_vars):
logger.debug("loading host_vars")
self.variables.update(yaml.load(open(host_vars).read()))
class BaseLoader(object):
'''
Common functions for an inventory loader from a given source.
'''
def __init__(self, inventory_base=None, all_group=None):
self.inventory_base = inventory_base
self.all_group = all_group
def __init__(self, source, all_group=None):
self.source = source
self.source_dir = os.path.dirname(self.source)
self.all_group = all_group or MemGroup('all', self.source_dir)
def get_host(self, name):
'''
Return a MemHost instance from host name, creating if needed.
'''
host_name = name.split(':')[0]
host = None
if not host_name in self.all_group.host_names:
host = MemHost(name, self.inventory_base)
self.all_group.host_names[host_name] = host
return self.all_group.host_names[host_name]
if not host_name in self.all_group.all_hosts:
host = MemHost(name, self.source_dir)
self.all_group.all_hosts[host_name] = host
return self.all_group.all_hosts[host_name]
def get_group(self, name, all_group=None, child=False):
'''
Return a MemGroup instance from group name, creating if needed.
'''
all_group = all_group or self.all_group
if name == 'all':
return all_group
if not name in self.all_group.group_names:
group = MemGroup(name, self.inventory_base)
if not name in self.all_group.all_groups:
group = MemGroup(name, self.source_dir)
if not child:
all_group.add_child_group(group)
self.all_group.group_names[name] = group
return self.all_group.group_names[name]
self.all_group.all_groups[name] = group
return self.all_group.all_groups[name]
def load(self, src):
def load(self):
raise NotImplementedError
class IniLoader(BaseLoader):
'''
Loader to read inventory from an INI-formatted text file.
'''
def __init__(self, inventory_base=None, all_group=None):
super(IniLoader, self).__init__(inventory_base, all_group)
logger.debug("processing ini")
def load(self, src):
logger.debug("loading: %s on %s" % (src, self.all_group))
if self.inventory_base is None:
self.inventory_base = os.path.dirname(src)
data = open(src).read()
lines = data.split("\n")
def load(self):
logger.info('Reading INI source: %s', self.source)
group = self.all_group
input_mode = 'host'
for line in lines:
for line in file(self.source, 'r'):
line = line.split('#')[0].strip()
if not line:
continue
elif line.startswith("["):
# mode change, possible new group name
line = line.replace("[","").replace("]","").lstrip().rstrip()
if line.find(":vars") != -1:
elif line.startswith('[') and line.endswith(']'):
# Mode change, possible new group name
line = line[1:-1].strip()
if line.endswith(':vars'):
input_mode = 'vars'
line = line.replace(":vars","")
group = self.get_group(line)
elif line.find(":children") != -1:
input_mode = 'children'
line = line.replace(":children","")
group = self.get_group(line)
line = line[:-5]
elif line.endswith(':children'):
input_mode = 'children'
line = line[:-9]
else:
input_mode = 'host'
group = self.get_group(line)
group = self.get_group(line)
else:
# add a host or variable to the existing group/host
# Add a host or variable to the existing group/host
tokens = shlex.split(line)
if input_mode == 'host':
new_host = self.get_host(tokens[0])
host = self.get_host(tokens[0])
if len(tokens) > 1:
variables = {}
for t in tokens[1:]:
(k,v) = t.split("=",1)
new_host.variables[k] = v
group.add_host(new_host)
k,v = t.split('=', 1)
host.variables[k] = v
group.add_host(host)
elif input_mode == 'children':
group.child_group_by_name(line, self)
elif input_mode == 'vars':
for t in tokens:
(k, v) = t.split("=", 1)
k, v = t.split('=', 1)
group.variables[k] = v
# TODO: expansion patterns are probably not going to be supported
# from API documentation:
#
# if called with --list, inventory outputs like so:
@ -236,12 +260,8 @@ class IniLoader(BaseLoader):
#
# if called with --host <host_record_name> outputs JSON for that host
class ExecutableJsonLoader(BaseLoader):
def __init__(self, inventory_base=None, all_group=None):
super(ExecutableJsonLoader, self).__init__(inventory_base, all_group)
logger.debug("processing executable JSON source")
self.child_group_names = {}
class ExecutableJsonLoader(BaseLoader):
def command_to_json(self, cmd):
data = {}
@ -250,101 +270,117 @@ class ExecutableJsonLoader(BaseLoader):
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = proc.communicate()
if proc.returncode != 0:
raise Exception("%s list failed %s with output: %s" % (cmd, stderr, proc.returncode))
raise RuntimeError('%r failed (rc=%d) with output: %s' % (cmd, proc.returncode, stderr))
data = json.loads(stdout)
if not isinstance(data, dict):
raise TypeError('Returned JSON must be a dictionary, got %s instead' % str(type(data)))
except:
traceback.print_exc()
raise Exception("failed to load JSON output: %s" % stdout)
assert type(data) == dict
logger.error('Failed to load JSON from: %s', stdout)
raise
return data
def load(self, src):
logger.debug("loading %s onto %s" % (src, self.all_group))
if self.inventory_base is None:
self.inventory_base = os.path.dirname(src)
data = self.command_to_json([src, "--list"])
group = None
def load(self):
logger.info('Reading executable JSON source: %s', self.source)
data = self.command_to_json([self.source, '--list'])
_meta = data.pop('_meta', {})
for (k,v) in data.iteritems():
for k,v in data.iteritems():
group = self.get_group(k)
if type(v) == dict:
# process hosts
host_details = v.get('hosts', None)
if host_details is not None:
if type(host_details) == dict:
for (hk, hv) in host_details.iteritems():
host = self.get_host(hk)
host.variables.update(hv)
group.add_host(host)
if type(host_details) == list:
for hk in host_details:
host = self.get_host(hk)
group.add_host(host)
# process variables
vars = v.get('vars', None)
if vars is not None:
# Load group hosts/vars/children from a dictionary.
if isinstance(v, dict):
# Process hosts within a group.
hosts = v.get('hosts', {})
if isinstance(hosts, dict):
for hk, hv in hosts.iteritems():
host = self.get_host(hk)
if isinstance(hv, dict):
host.variables.update(hv)
else:
self.logger.warning('Expected dict of vars for '
'host "%s", got %s instead',
hk, str(type(hv)))
group.add_host(host)
elif isinstance(hosts, (list, tuple)):
for hk in hosts:
host = self.get_host(hk)
group.add_host(host)
else:
logger.warning('Expected dict or list of "hosts" for '
'group "%s", got %s instead', k,
str(type(hosts)))
# Process group variables.
vars = v.get('vars', {})
if isinstance(vars, dict):
group.variables.update(vars)
# process child groups
children_details = v.get('children', None)
if children_details is not None:
for x in children_details:
child = self.get_group(x, self.inventory_base, child=True)
else:
self.logger.warning('Expected dict of vars for '
'group "%s", got %s instead',
k, str(type(vars)))
# Process child groups.
children = v.get('children', [])
if isinstance(children, (list, tuple)):
for c in children:
child = self.get_group(c, self.all_group, child=True)
group.add_child_group(child)
self.child_group_names[x] = child
else:
self.logger.warning('Expected list of children for '
'group "%s", got %s instead',
k, str(type(children)))
if type(v) in (tuple, list):
for x in v:
host = self.get_host(x)
group.add_host(host)
# Load host names from a list.
elif isinstance(v, (list, tuple)):
for h in v:
host = self.get_host(h)
group.add_host(host)
else:
logger.warning('')
self.logger.warning('Expected dict or list for group "%s", '
'got %s instead', k, str(type(v)))
if k != 'all':
self.all_group.add_child_group(group)
# then we invoke the executable once for each host name we've built up
# Invoke the executable once for each host name we've built up
# to set their variables
for (k,v) in self.all_group.host_names.iteritems():
for k,v in self.all_group.all_hosts.iteritems():
if 'hostvars' not in _meta:
data = self.command_to_json([src, "--host", k])
data = self.command_to_json([self.source, '--host', k])
else:
data = _meta['hostvars'].get(k, {})
v.variables.update(data)
if isinstance(data, dict):
v.variables.update(data)
else:
self.logger.warning('Expected dict of vars for '
'host "%s", got %s instead',
k, str(type(data)))
def load_generic(src):
logger.debug("analyzing type of source")
if not os.path.exists(src):
logger.debug("source missing")
raise CommandError("source does not exist")
if os.path.isdir(src):
all_group = MemGroup('all', src)
for f in glob.glob("%s/*" % src):
if f.endswith(".ini"):
# config files for inventory scripts should be ignored
continue
if not os.path.isdir(f):
if os.access(f, os.X_OK):
ExecutableJsonLoader(None, all_group).load(f)
else:
IniLoader(None, all_group).load(f)
elif os.access(src, os.X_OK):
all_group = MemGroup('all', os.path.dirname(src))
ExecutableJsonLoader(None, all_group).load(src)
def load_inventory_source(source, all_group=None):
'''
Load inventory from given source directory or file.
'''
logger.debug('Analyzing type of source: %s', source)
original_all_group = all_group
if not os.path.exists(source):
raise CommandError('Source does not exist: %s' % source)
if os.path.isdir(source):
all_group = all_group or MemGroup('all', source)
for filename in glob.glob(os.path.join(source, '*')):
if filename.endswith(".ini") or os.path.isdir(filename):
continue
load_inventory_source(filename, all_group)
else:
all_group = MemGroup('all', os.path.dirname(src))
IniLoader(None, all_group).load(src)
all_group = all_group or MemGroup('all', os.path.dirname(source))
if os.access(source, os.X_OK):
ExecutableJsonLoader(source, all_group).load()
else:
IniLoader(source, all_group).load()
logger.debug("loading process complete")
logger.debug('Finished loading from source: %s', source)
if original_all_group is None:
logger.info('Loaded %d groups, %d hosts', len(all_group.all_groups),
len(all_group.all_hosts))
return all_group
@ -382,7 +418,8 @@ class Command(NoArgsCommand):
self.logger = logging.getLogger('awx.main.commands.inventory_import')
self.logger.setLevel(log_levels.get(self.verbosity, 0))
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter('%(message)s'))
formatter = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s')
handler.setFormatter(formatter)
self.logger.addHandler(handler)
self.logger.propagate = False
@ -446,30 +483,32 @@ class Command(NoArgsCommand):
# source attached to a specific group, only delete hosts beneath that
# group. Delete each host individually so signal handlers will run.
if self.overwrite:
self.logger.debug('deleting any hosts not in the remote source')
if self.inventory_source.group:
del_hosts = self.inventory_source.group.all_hosts
# FIXME: Also include hosts from inventory_source.managed_hosts?
else:
del_hosts = self.inventory.hosts.all()
del_hosts = del_hosts.exclude(name__in=self.all_group.host_names.keys())
del_hosts = del_hosts.exclude(name__in=self.all_group.all_hosts.keys())
for host in del_hosts:
host_name = host.name
host.delete()
self.logger.info('Deleted host "%s"', host_name)
# If overwrite is set, for each group in the database that is NOT in
# the local list, delete it. When importing from a cloud inventory
# source attached to a specific group, only delete children of that
# group. Delete each group individually so signal handlers will run.
if self.overwrite:
self.logger.debug('deleting any groups not in the remote source')
if self.inventory_source.group:
del_groups = self.inventory_source.group.all_children
# FIXME: Also include groups from inventory_source.managed_groups?
else:
del_groups = self.inventory.groups.all()
del_groups = del_groups.exclude(name__in=self.all_group.group_names.keys())
del_groups = del_groups.exclude(name__in=self.all_group.all_groups.keys())
for group in del_groups:
group_name = group.name
group.delete()
self.logger.info('Deleted group "%s"', group_name)
# If overwrite is set, clear all invalid child relationships for groups
# and all invalid host memberships. When importing from a cloud
@ -477,47 +516,54 @@ class Command(NoArgsCommand):
# relationships for hosts and groups that are beneath the inventory
# source group.
if self.overwrite:
self.logger.info("clearing any child relationships to rebuild from remote source")
if self.inventory_source.group:
db_groups = self.inventory_source.group.all_children
else:
db_groups = self.inventory.groups.all()
for db_group in db_groups:
db_kids = db_group.children.all()
mem_kids = self.all_group.group_names[db_group.name].child_groups
mem_kid_names = [ k.name for k in mem_kids ]
for db_kid in db_kids:
if db_kid.name not in mem_kid_names:
self.logger.debug("removing non-DB kid: %s" % (db_kid.name))
db_group.children.remove(db_kid)
db_children = db_group.children.all()
mem_children = self.all_group.all_groups[db_group.name].children
mem_children_names = [g.name for g in mem_children]
for db_child in db_children.exclude(name__in=mem_children_names):
if db_child not in db_group.children.all():
continue
db_group.children.remove(db_child)
self.logger.info('Removed group "%s" from group "%s"',
db_child.name, db_group.name)
db_hosts = db_group.hosts.all()
mem_hosts = self.all_group.group_names[db_group.name].hosts
mem_host_names = [ h.name for h in mem_hosts ]
for db_host in db_hosts:
if db_host.name not in mem_host_names:
self.logger.debug("removing non-DB host: %s" % (db_host.name))
db_group.hosts.remove(db_host)
mem_hosts = self.all_group.all_groups[db_group.name].hosts
mem_host_names = [h.name for h in mem_hosts]
for db_host in db_hosts.exclude(name__in=mem_host_names):
if db_host not in db_group.hosts.all():
continue
db_group.hosts.remove(db_host)
self.logger.info('Removed host "%s" from group "%s"',
db_host.name, db_group.name)
# Update/overwrite variables from "all" group. If importing from a
# cloud source attached to a specific group, variables will be set on
# the base group, otherwise they will be set on the inventory.
# the base group, otherwise they will be set on the whole inventory.
if self.inventory_source.group:
all_obj = self.inventory_source.group
all_obj.inventory_sources.add(self.inventory_source)
all_name = 'group "%s"' % all_obj.name
else:
all_obj = self.inventory
all_name = 'inventory'
db_variables = all_obj.variables_dict
mem_variables = self.all_group.variables
if self.overwrite_vars or self.overwrite:
self.logger.info('replacing inventory variables from "all" group')
db_variables = mem_variables
db_variables = self.all_group.variables
else:
self.logger.info('updating inventory variables from "all" group')
db_variables.update(mem_variables)
all_obj.variables = json.dumps(db_variables)
all_obj.save(update_fields=['variables'])
db_variables.update(self.all_group.variables)
if db_variables != all_obj.variables_dict:
all_obj.variables = json.dumps(db_variables)
all_obj.save(update_fields=['variables'])
if self.overwrite_vars or self.overwrite:
self.logger.info('Replaced %s variables from "all" group', all_name)
else:
self.logger.info('Updated %s variables from "all" group', all_name)
else:
self.logger.info('%s variables unmodified', all_name.capitalize())
# FIXME: Attribute changes to superuser?
@ -525,23 +571,28 @@ class Command(NoArgsCommand):
# the database. Otherwise, update/replace database variables from the
# imported data. Associate with the inventory source group if
# importing from cloud inventory source.
for k,v in self.all_group.group_names.iteritems():
for k,v in self.all_group.all_groups.iteritems():
variables = json.dumps(v.variables)
defaults = dict(variables=variables, description='imported')
group, created = self.inventory.groups.get_or_create(name=k,
defaults=defaults)
if created:
self.logger.info('inserting new group %s' % k)
self.logger.info('Added new group "%s"', k)
else:
self.logger.info('updating existing group %s' % k)
db_variables = group.variables_dict
mem_variables = v.variables
if self.overwrite_vars or self.overwrite:
db_variables = mem_variables
db_variables = v.variables
else:
db_variables.update(mem_variables)
group.variables = json.dumps(db_variables)
group.save(update_fields=['variables'])
db_variables.update(v.variables)
if db_variables != group.variables_dict:
group.variables = json.dumps(db_variables)
group.save(update_fields=['variables'])
if self.overwrite_vars or self.overwrite:
self.logger.info('Replaced variables for group "%s"', k)
else:
self.logger.info('Updated variables for group "%s"', k)
else:
self.logger.info('Variables unmodified for group "%s"', k)
if self.inventory_source.group:
self.inventory_source.group.children.add(group)
group.inventory_sources.add(self.inventory_source)
@ -550,44 +601,59 @@ class Command(NoArgsCommand):
# the database. Otherwise, update/replace database variables from the
# imported data. Associate with the inventory source group if
# importing from cloud inventory source.
for k,v in self.all_group.host_names.iteritems():
for k,v in self.all_group.all_hosts.iteritems():
variables = json.dumps(v.variables)
defaults = dict(variables=variables, description='imported')
host, created = self.inventory.hosts.get_or_create(name=k,
defaults=defaults)
if created:
self.logger.info('inserting new host %s' % k)
self.logger.info('Added new host "%s"', k)
else:
self.logger.info('updating existing host %s' % k)
db_variables = host.variables_dict
mem_variables = v.variables
if self.overwrite_vars or self.overwrite:
db_variables = mem_variables
db_variables = v.variables
else:
db_variables.update(mem_variables)
host.variables = json.dumps(db_variables)
host.save(update_fields=['variables'])
db_variables.update(v.variables)
if db_variables != host.variables_dict:
host.variables = json.dumps(db_variables)
host.save(update_fields=['variables'])
if self.overwrite_vars or self.overwrite:
self.logger.info('Replaced variables for host "%s"', k)
else:
self.logger.info('Updated variables for host "%s"', k)
else:
self.logger.info('Variables unmodified for host "%s"', k)
if self.inventory_source.group:
self.inventory_source.group.hosts.add(host)
host.inventory_sources.add(self.inventory_source)
host.update_computed_fields(False, False)
# for each host in a mem group, add it to the parents to which it belongs
for (k,v) in self.all_group.group_names.iteritems():
self.logger.info("adding parent arrangements for %s" % k)
db_group = Group.objects.get(name=k, inventory__pk=self.inventory.pk)
mem_hosts = v.hosts
for h in mem_hosts:
db_host = Host.objects.get(name=h.name, inventory__pk=self.inventory.pk)
db_group.hosts.add(db_host)
self.logger.debug("*** ADDING %s to %s ***" % (db_host, db_group))
# For each host in a mem group, add it to the parent(s) to which it
# belongs.
for k,v in self.all_group.all_groups.iteritems():
if not v.hosts:
continue
db_group = self.inventory.groups.get(name=k)
for h in v.hosts:
db_host = self.inventory.hosts.get(name=h.name)
if db_host not in db_group.hosts.all():
db_group.hosts.add(db_host)
self.logger.info('Added host "%s" to group "%s"', h.name, k)
else:
self.logger.info('Host "%s" already in group "%s"', h.name, k)
# for each group, draw in child group arrangements
for (k,v) in self.all_group.group_names.iteritems():
db_group = Group.objects.get(inventory=self.inventory, name=k)
for mem_child_group in v.child_groups:
db_child = Group.objects.get(inventory=self.inventory, name=mem_child_group.name)
db_group.children.add(db_child)
for k,v in self.all_group.all_groups.iteritems():
if not v.children:
continue
db_group = self.inventory.groups.get(name=k)
for g in v.children:
db_child = self.inventory.groups.get(name=g.name)
if db_child not in db_group.hosts.all():
db_group.children.add(db_child)
self.logger.info('Added group "%s" as child of "%s"', g.name, k)
else:
self.logger.info('Group "%s" already child of group "%s"', g.name, k)
def check_license(self):
reader = LicenseReader()
@ -596,10 +662,15 @@ class Command(NoArgsCommand):
free_instances = license_info.get('free_instances', 0)
new_count = Host.objects.filter(active=True).count()
if free_instances < 0:
d = {
'new_count': new_count,
'available_instances': available_instances,
}
if license_info.get('demo', False):
raise CommandError("demo mode free license count exceeded, would bring available instances to %s, demo mode allows %s, see http://ansibleworks.com/ansibleworks-awx for licensing information" % (new_count, available_instances))
self.logger.error(DEMO_LICENSE_MESSAGE % d)
else:
raise CommandError("number of licensed instances exceeded, would bring available instances to %s, system is licensed for %s, see http://ansibleworks.com/ansibleworks-awx for license extension information" % (new_count, available_instances))
self.logger.error(LICENSE_MESSAGE % d)
raise CommandError('License count exceeded!')
@transaction.commit_on_success
def handle_noargs(self, **options):
@ -622,6 +693,7 @@ class Command(NoArgsCommand):
if not self.source:
raise CommandError('--source is required')
begin = time.time()
self.load_inventory_from_database()
status, tb, exc = 'error', '', None
@ -632,24 +704,28 @@ class Command(NoArgsCommand):
self.inventory_update.save()
transaction.commit()
self.logger.debug('preparing to load from %s' % self.source)
self.all_group = load_generic(self.source)
self.logger.debug('debugging loaded result:')
# Load inventory from source.
self.all_group = load_inventory_source(self.source)
self.all_group.debug_tree()
# now that memGroup is correct and supports JSON executables, INI, and trees
# now merge and/or overwrite with the database itself!
# Merge/overwrite inventory into database.
self.load_into_database()
self.check_license()
self.logger.info("inventory import complete, %s, id=%s" % \
(self.inventory.name, self.inventory.id))
if self.inventory_source.group:
inv_name = 'group "%s"' % (self.inventory_source.group.name)
else:
inv_name = '"%s" (id=%s)' % (self.inventory.name,
self.inventory.id)
self.logger.info('Inventory import completed for %s in %0.1fs',
inv_name, time.time() - begin)
status = 'successful'
except Exception, e:
if isinstance(e, KeyboardInterrupt):
status = 'canceled'
exc = e
elif isinstance(e, CommandError):
exc = e
else:
tb = traceback.format_exc()
exc = e

1
awx/main/models/__init__.py
View File

@ -686,6 +686,7 @@ class InventorySource(PrimordialModel):
blank=True,
default='',
)
# FIXME: Remove tags field when making other migrations for credential changes!
source_tags = models.CharField(
max_length=1024,
blank=True,

10
awx/main/serializers.py
View File

@ -672,9 +672,9 @@ class InventorySourceSerializer(BaseSerializer):
fields = ('id', 'url', 'related', 'summary_fields', 'created',
'modified', 'inventory', 'group', 'source', 'source_path',
'source_vars', 'source_username', 'source_password',
'source_regions', 'source_tags', 'overwrite',
'overwrite_vars', 'update_on_launch', 'update_interval',
'last_update_failed', 'status', 'last_updated')
'source_regions', 'overwrite', 'overwrite_vars',
'update_on_launch', 'update_interval', 'last_update_failed',
'status', 'last_updated')
read_only_fields = ('inventory', 'group')
def to_native(self, obj):
@ -764,10 +764,6 @@ class InventorySourceSerializer(BaseSerializer):
# FIXME
return attrs
def validate_source_tags(self, attrs, source):
# FIXME
return attrs
class InventoryUpdateSerializer(BaseSerializer):
class Meta:

2
awx/main/signals.py
View File

@ -41,8 +41,6 @@ def update_inventory_computed_fields(sender, **kwargs):
sender_name = unicode(sender._meta.verbose_name)
if kwargs['signal'] == post_save:
sender_action = 'saved'
if instance.active: # No need to update for active instances.
return
elif kwargs['signal'] == post_delete:
sender_action = 'deleted'
elif kwargs['signal'] == m2m_changed and kwargs['action'] in ('post_add', 'post_remove', 'post_clear'):

3
awx/main/tasks.py
View File

@ -805,7 +805,8 @@ class RunInventoryUpdate(BaseTask):
args.append(rax_path)
elif inventory_source.source == 'file':
args.append(inventory_source.source_path)
args.append('-v2')
verbosity = getattr(settings, 'INVENTORY_UPDATE_VERBOSITY', 1)
args.append('-v%d' % verbosity)
if settings.DEBUG:
args.append('--traceback')
return args

4
awx/main/tests/inventory.py
View File

@ -5,6 +5,7 @@
import datetime
import json
import os
import re
# Django
from django.conf import settings
@ -1021,6 +1022,9 @@ class InventoryUpdatesTest(BaseTransactionTest):
source_pks = group.inventory_sources.values_list('pk', flat=True)
self.assertTrue(inventory_source.pk in source_pks)
self.assertTrue(group.has_inventory_sources)
# Make sure EC2 instance ID groups are excluded.
self.assertFalse(re.match(r'^i-[0-9a-f]{8}$', group.name, re.I),
group.name)
def test_update_from_ec2(self):
source_username = getattr(settings, 'TEST_AWS_ACCESS_KEY_ID', '')

3
awx/plugins/inventory/ec2.py
View File

@ -336,8 +336,9 @@ class Ec2Inventory(object):
# Add to index
self.index[dest] = [region, instance.id]
# For AWX: do not output group based on instance ID!!!
# Inventory: Group by instance ID (always a group of 1)
self.inventory[instance.id] = [dest]
# self.inventory[instance.id] = [dest]
# Inventory: Group by region
self.push(self.inventory, region, dest)

5
awx/settings/local_settings.py.example
View File

@ -74,9 +74,12 @@ REMOTE_HOST_HEADERS = ['REMOTE_ADDR', 'REMOTE_HOST']
# the celery task.
#AWX_TASK_ENV['FOO'] = 'BAR'
# If set, uses -vvv for project updates instead of just -v for more output.
# If set, use -vvv for project updates instead of -v for more output.
# PROJECT_UPDATE_VVV=True
# Set verbosity for inventory import command when running inventory updates.
# INVENTORY_UPDATE_VERBOSITY=1
###############################################################################
# EMAIL SETTINGS
###############################################################################

13
awx/ui/static/js/forms/Groups.js
View File

@ -155,19 +155,6 @@ angular.module('GroupFormDefinition', [])
"Only hosts associated with the list of regions will be included in the update process.</p>",
dataContainer: 'body'
},
source_tags: {
label: 'Tags',
excludeModal: true,
type: 'text',
ngShow: "source.value == 'ec2'",
addRequired: false,
editRequired: false,
dataTitle: 'Source Regions',
dataPlacement: 'left',
awPopOver: "<p>Comma separated list of tags. Tag names must match those defined at the inventory source." +
" Only hosts associated with the list of tags will be included in the update process.</p>",
dataContainer: 'body'
},
source_vars: {
label: 'Source Variables',
ngShow: "source.value == 'file' || source.value == 'ec2'",

1
awx/ui/static/js/helpers/Groups.js
View File

@ -824,7 +824,6 @@ angular.module('GroupsHelper', [ 'RestServices', 'Utilities', 'ListGenerator', '
source_username: scope['source_username'],
source_password: scope['source_password'],
source_regions: scope['source_regions'],
source_tags: scope['source_tags'],
overwrite: scope['overwrite'],
overwrite_vars: scope['overwrite_vars'],
update_on_launch: scope['update_on_launch'],