External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-03-07 19:51:08 -03:30
Code Issues Packages Projects Releases Wiki Activity

Merge remote branch 'upstream/master'

Browse Source
This commit is contained in:
chouseknecht
2013-10-24 18:39:49 +00:00
parent a3b1fb7f41 710d55d82d
commit b2f2fb655d
10 changed files with 355 additions and 289 deletions
Download Patch File Download Diff File Expand all files Collapse all files

602
awx/main/management/commands/inventory_import.py
View File

@@ -10,6 +10,7 @@ import os
import shlex import shlex
import subprocess import subprocess
import sys import sys
import time
import traceback import traceback
# PyYAML # PyYAML
@@ -26,191 +27,214 @@ from awx.main.licenses import LicenseReader
logger = logging.getLogger('awx.main.commands.inventory_import') logger = logging.getLogger('awx.main.commands.inventory_import')
class ImportException(BaseException): LICENSE_MESSAGE = '''\
Number of licensed instances exceeded, would bring available instances to %(new_count)d, system is licensed for %(available_instances)d.
See http://ansibleworks.com/ansibleworks-awx for license extension information.'''
def __init__(self, msg): DEMO_LICENSE_MESSAGE = '''\
self.msg = msg Demo mode free license count exceeded, would bring available instances to %(new_count)d, demo mode allows %(available_instances)d.
See http://ansibleworks.com/ansibleworks-awx for licensing information.'''
def __str__(self):
return "Import Error: %s" % msg
class MemGroup(object):
def __init__(self, name, inventory_base):
assert inventory_base is not None
self.inventory_base = inventory_base
class MemObject(object):
'''
Common code shared between in-memory groups and hosts.
'''
def __init__(self, name, source_dir):
assert name
assert source_dir
self.name = name self.name = name
self.child_groups = [] self.source_dir = source_dir
def load_vars(self, path):
if os.path.exists(path) and os.path.isfile(path):
vars_name = os.path.basename(os.path.dirname(path))
logger.debug('Loading %s from %s', vars_name, path)
try:
v = yaml.safe_load(file(path, 'r').read())
return v if hasattr(v, 'items') else {}
except yaml.YAMLError, e:
if hasattr(e, 'problem_mark'):
logger.error('Invalid YAML in %s:%s col %s', path,
e.problem_mark.line + 1,
e.problem_mark.column + 1)
else:
logger.error('Error loading YAML from %s', path)
raise
return {}
class MemGroup(MemObject):
'''
In-memory representation of an inventory group.
'''
def __init__(self, name, source_dir):
super(MemGroup, self).__init__(name, source_dir)
self.children = []
self.hosts = [] self.hosts = []
self.variables = {} self.variables = {}
self.parents = [] self.parents = []
# Used on the "all" group in place of previous global variables. # Used on the "all" group in place of previous global variables.
# maps host and group names to hosts to prevent redudant additions # maps host and group names to hosts to prevent redudant additions
self.host_names = {} self.all_hosts = {}
self.group_names = {} self.all_groups = {}
group_vars = os.path.join(inventory_base, 'group_vars', name) group_vars = os.path.join(self.source_dir, 'group_vars', self.name)
if os.path.exists(group_vars): self.variables = self.load_vars(group_vars)
logger.debug("loading group_vars") logger.debug('Loaded group: %s', self.name)
self.variables = yaml.load(open(group_vars).read())
def child_group_by_name(self, name, loader):
def child_group_by_name(self, grp_name, loader): if name == 'all':
logger.debug("looking for child group: %s" % grp_name)
if grp_name == 'all':
return return
logger.debug('Looking for %s as child group of %s', name, self.name)
# slight hack here, passing in 'self' for all_group but child=True won't use it # slight hack here, passing in 'self' for all_group but child=True won't use it
grp = loader.get_group(grp_name, self, child=True) group = loader.get_group(name, self, child=True)
# don't add to child groups if already there # don't add to child groups if already there
for x in self.child_groups: for g in self.children:
if x.name == grp_name: if g.name == name:
return x return g
logger.debug("adding child group %s to group %s" % (grp.name, self.name)) logger.debug('Adding child group %s to group %s', group.name, self.name)
self.child_groups.append(grp) self.children.append(group)
return grp return group
def add_child_group(self, grp): def add_child_group(self, group):
assert grp.name is not 'all' assert group.name is not 'all'
assert isinstance(group, MemGroup)
logger.debug("adding child group %s to group %s" % (grp.name, self.name)) logger.debug('Adding child group %s to parent %s', group.name, self.name)
if group not in self.children:
assert type(grp) == MemGroup self.children.append(group)
if grp not in self.child_groups: if not self in group.parents:
self.child_groups.append(grp) group.parents.append(self)
if not self in grp.parents:
grp.parents.append(self)
def add_host(self, host): def add_host(self, host):
logger.debug("adding host %s to group %s" % (host.name, self.name)) assert isinstance(host, MemHost)
logger.debug('Adding host %s to group %s', host.name, self.name)
assert type(host) == MemHost
if host not in self.hosts: if host not in self.hosts:
self.hosts.append(host) self.hosts.append(host)
def debug_tree(self): def debug_tree(self, group_names=None):
logger.debug("describing tree of group (%s)" % self.name) group_names = group_names or set()
if self.name in group_names:
logger.debug("group: %s, %s" % (self.name, self.variables)) return
for x in self.child_groups: logger.debug('Dumping tree for group "%s":', self.name)
logger.debug(" child: %s" % (x.name)) logger.debug('- Vars: %r', self.variables)
for x in self.hosts: for h in self.hosts:
logger.debug(" host: %s, %s" % (x.name, x.variables)) logger.debug('- Host: %s, %r', h.name, h.variables)
for g in self.children:
logger.debug('- Child: %s', g.name)
logger.debug('----')
group_names.add(self.name)
for g in self.children:
g.debug_tree(group_names)
logger.debug("---")
for x in self.child_groups:
x.debug_tree()
class MemHost(object): class MemHost(MemObject):
'''
In-memory representation of an inventory host.
'''
def __init__(self, name, inventory_base): def __init__(self, name, source_dir):
logger.debug("adding host name: %s" % name) super(MemHost, self).__init__(name, source_dir)
assert name is not None
assert inventory_base is not None
# set ansible_ssh_port if ":" in name
self.name = name
self.variables = {} self.variables = {}
self.inventory_base = inventory_base
if ':' in name: if ':' in name:
tokens = name.split(":") tokens = name.split(":")
self.name = tokens[0] self.name = tokens[0]
self.variables['ansible_ssh_port'] = int(tokens[1]) self.variables['ansible_ssh_port'] = int(tokens[1])
if "[" in name: if '[' in name:
raise ImportException("block ranges like host[0:50].example.com are not yet supported by the importer") raise ValueError('Block ranges like host[0:50].example.com are not yet supported by the importer')
host_vars = os.path.join(source_dir, 'host_vars', name)
self.variables.update(self.load_vars(host_vars))
logger.debug('Loaded host: %s', self.name)
host_vars = os.path.join(inventory_base, 'host_vars', name)
if os.path.exists(host_vars):
logger.debug("loading host_vars")
self.variables.update(yaml.load(open(host_vars).read()))
class BaseLoader(object): class BaseLoader(object):
'''
Common functions for an inventory loader from a given source.
'''
def __init__(self, inventory_base=None, all_group=None): def __init__(self, source, all_group=None):
self.inventory_base = inventory_base self.source = source
self.all_group = all_group self.source_dir = os.path.dirname(self.source)
self.all_group = all_group or MemGroup('all', self.source_dir)
def get_host(self, name): def get_host(self, name):
'''
Return a MemHost instance from host name, creating if needed.
'''
host_name = name.split(':')[0] host_name = name.split(':')[0]
host = None host = None
if not host_name in self.all_group.host_names: if not host_name in self.all_group.all_hosts:
host = MemHost(name, self.inventory_base) host = MemHost(name, self.source_dir)
self.all_group.host_names[host_name] = host self.all_group.all_hosts[host_name] = host
return self.all_group.host_names[host_name] return self.all_group.all_hosts[host_name]
def get_group(self, name, all_group=None, child=False): def get_group(self, name, all_group=None, child=False):
'''
Return a MemGroup instance from group name, creating if needed.
'''
all_group = all_group or self.all_group all_group = all_group or self.all_group
if name == 'all': if name == 'all':
return all_group return all_group
if not name in self.all_group.group_names: if not name in self.all_group.all_groups:
group = MemGroup(name, self.inventory_base) group = MemGroup(name, self.source_dir)
if not child: if not child:
all_group.add_child_group(group) all_group.add_child_group(group)
self.all_group.group_names[name] = group self.all_group.all_groups[name] = group
return self.all_group.group_names[name] return self.all_group.all_groups[name]
def load(self, src): def load(self):
raise NotImplementedError raise NotImplementedError
class IniLoader(BaseLoader): class IniLoader(BaseLoader):
'''
Loader to read inventory from an INI-formatted text file.
'''
def __init__(self, inventory_base=None, all_group=None): def load(self):
super(IniLoader, self).__init__(inventory_base, all_group) logger.info('Reading INI source: %s', self.source)
logger.debug("processing ini")
def load(self, src):
logger.debug("loading: %s on %s" % (src, self.all_group))
if self.inventory_base is None:
self.inventory_base = os.path.dirname(src)
data = open(src).read()
lines = data.split("\n")
group = self.all_group group = self.all_group
input_mode = 'host' input_mode = 'host'
for line in file(self.source, 'r'):
for line in lines:
line = line.split('#')[0].strip() line = line.split('#')[0].strip()
if not line: if not line:
continue continue
elif line.startswith("["): elif line.startswith('[') and line.endswith(']'):
# mode change, possible new group name # Mode change, possible new group name
line = line.replace("[","").replace("]","").lstrip().rstrip() line = line[1:-1].strip()
if line.find(":vars") != -1: if line.endswith(':vars'):
input_mode = 'vars' input_mode = 'vars'
line = line.replace(":vars","") line = line[:-5]
group = self.get_group(line) elif line.endswith(':children'):
elif line.find(":children") != -1: input_mode = 'children'
input_mode = 'children' line = line[:-9]
line = line.replace(":children","")
group = self.get_group(line)
else: else:
input_mode = 'host' input_mode = 'host'
group = self.get_group(line) group = self.get_group(line)
else: else:
# add a host or variable to the existing group/host # Add a host or variable to the existing group/host
tokens = shlex.split(line) tokens = shlex.split(line)
if input_mode == 'host': if input_mode == 'host':
new_host = self.get_host(tokens[0]) host = self.get_host(tokens[0])
if len(tokens) > 1: if len(tokens) > 1:
variables = {}
for t in tokens[1:]: for t in tokens[1:]:
(k,v) = t.split("=",1) k,v = t.split('=', 1)
new_host.variables[k] = v host.variables[k] = v
group.add_host(new_host) group.add_host(host)
elif input_mode == 'children': elif input_mode == 'children':
group.child_group_by_name(line, self) group.child_group_by_name(line, self)
elif input_mode == 'vars': elif input_mode == 'vars':
for t in tokens: for t in tokens:
(k, v) = t.split("=", 1) k, v = t.split('=', 1)
group.variables[k] = v group.variables[k] = v
# TODO: expansion patterns are probably not going to be supported # TODO: expansion patterns are probably not going to be supported
# from API documentation: # from API documentation:
# #
# if called with --list, inventory outputs like so: # if called with --list, inventory outputs like so:
@@ -236,12 +260,8 @@ class IniLoader(BaseLoader):
# #
# if called with --host <host_record_name> outputs JSON for that host # if called with --host <host_record_name> outputs JSON for that host
class ExecutableJsonLoader(BaseLoader):
def __init__(self, inventory_base=None, all_group=None): class ExecutableJsonLoader(BaseLoader):
super(ExecutableJsonLoader, self).__init__(inventory_base, all_group)
logger.debug("processing executable JSON source")
self.child_group_names = {}
def command_to_json(self, cmd): def command_to_json(self, cmd):
data = {} data = {}
@@ -250,101 +270,117 @@ class ExecutableJsonLoader(BaseLoader):
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = proc.communicate() stdout, stderr = proc.communicate()
if proc.returncode != 0: if proc.returncode != 0:
raise Exception("%s list failed %s with output: %s" % (cmd, stderr, proc.returncode)) raise RuntimeError('%r failed (rc=%d) with output: %s' % (cmd, proc.returncode, stderr))
data = json.loads(stdout) data = json.loads(stdout)
if not isinstance(data, dict):
raise TypeError('Returned JSON must be a dictionary, got %s instead' % str(type(data)))
except: except:
traceback.print_exc() logger.error('Failed to load JSON from: %s', stdout)
raise Exception("failed to load JSON output: %s" % stdout) raise
assert type(data) == dict
return data return data
def load(self, src): def load(self):
logger.info('Reading executable JSON source: %s', self.source)
logger.debug("loading %s onto %s" % (src, self.all_group)) data = self.command_to_json([self.source, '--list'])
if self.inventory_base is None:
self.inventory_base = os.path.dirname(src)
data = self.command_to_json([src, "--list"])
group = None
_meta = data.pop('_meta', {}) _meta = data.pop('_meta', {})
for (k,v) in data.iteritems(): for k,v in data.iteritems():
group = self.get_group(k) group = self.get_group(k)
if type(v) == dict: # Load group hosts/vars/children from a dictionary.
if isinstance(v, dict):
# process hosts # Process hosts within a group.
host_details = v.get('hosts', None) hosts = v.get('hosts', {})
if host_details is not None: if isinstance(hosts, dict):
if type(host_details) == dict: for hk, hv in hosts.iteritems():
for (hk, hv) in host_details.iteritems(): host = self.get_host(hk)
host = self.get_host(hk) if isinstance(hv, dict):
host.variables.update(hv) host.variables.update(hv)
group.add_host(host) else:
if type(host_details) == list: self.logger.warning('Expected dict of vars for '
for hk in host_details: 'host "%s", got %s instead',
host = self.get_host(hk) hk, str(type(hv)))
group.add_host(host) group.add_host(host)
elif isinstance(hosts, (list, tuple)):
# process variables for hk in hosts:
vars = v.get('vars', None) host = self.get_host(hk)
if vars is not None: group.add_host(host)
else:
logger.warning('Expected dict or list of "hosts" for '
'group "%s", got %s instead', k,
str(type(hosts)))
# Process group variables.
vars = v.get('vars', {})
if isinstance(vars, dict):
group.variables.update(vars) group.variables.update(vars)
else:
# process child groups self.logger.warning('Expected dict of vars for '
children_details = v.get('children', None) 'group "%s", got %s instead',
if children_details is not None: k, str(type(vars)))
for x in children_details: # Process child groups.
child = self.get_group(x, self.inventory_base, child=True) children = v.get('children', [])
if isinstance(children, (list, tuple)):
for c in children:
child = self.get_group(c, self.all_group, child=True)
group.add_child_group(child) group.add_child_group(child)
self.child_group_names[x] = child else:
self.logger.warning('Expected list of children for '
'group "%s", got %s instead',
k, str(type(children)))
if type(v) in (tuple, list): # Load host names from a list.
for x in v: elif isinstance(v, (list, tuple)):
host = self.get_host(x) for h in v:
group.add_host(host) host = self.get_host(h)
group.add_host(host)
else:
logger.warning('')
self.logger.warning('Expected dict or list for group "%s", '
'got %s instead', k, str(type(v)))
if k != 'all': if k != 'all':
self.all_group.add_child_group(group) self.all_group.add_child_group(group)
# then we invoke the executable once for each host name we've built up # Invoke the executable once for each host name we've built up
# to set their variables # to set their variables
for (k,v) in self.all_group.host_names.iteritems(): for k,v in self.all_group.all_hosts.iteritems():
if 'hostvars' not in _meta: if 'hostvars' not in _meta:
data = self.command_to_json([src, "--host", k]) data = self.command_to_json([self.source, '--host', k])
else: else:
data = _meta['hostvars'].get(k, {}) data = _meta['hostvars'].get(k, {})
v.variables.update(data) if isinstance(data, dict):
v.variables.update(data)
else:
self.logger.warning('Expected dict of vars for '
'host "%s", got %s instead',
k, str(type(data)))
def load_generic(src): def load_inventory_source(source, all_group=None):
logger.debug("analyzing type of source") '''
if not os.path.exists(src): Load inventory from given source directory or file.
logger.debug("source missing") '''
raise CommandError("source does not exist") logger.debug('Analyzing type of source: %s', source)
if os.path.isdir(src): original_all_group = all_group
all_group = MemGroup('all', src) if not os.path.exists(source):
for f in glob.glob("%s/*" % src): raise CommandError('Source does not exist: %s' % source)
if f.endswith(".ini"): if os.path.isdir(source):
# config files for inventory scripts should be ignored all_group = all_group or MemGroup('all', source)
continue for filename in glob.glob(os.path.join(source, '*')):
if not os.path.isdir(f): if filename.endswith(".ini") or os.path.isdir(filename):
if os.access(f, os.X_OK): continue
ExecutableJsonLoader(None, all_group).load(f) load_inventory_source(filename, all_group)
else:
IniLoader(None, all_group).load(f)
elif os.access(src, os.X_OK):
all_group = MemGroup('all', os.path.dirname(src))
ExecutableJsonLoader(None, all_group).load(src)
else: else:
all_group = MemGroup('all', os.path.dirname(src)) all_group = all_group or MemGroup('all', os.path.dirname(source))
IniLoader(None, all_group).load(src) if os.access(source, os.X_OK):
ExecutableJsonLoader(source, all_group).load()
else:
IniLoader(source, all_group).load()
logger.debug("loading process complete") logger.debug('Finished loading from source: %s', source)
if original_all_group is None:
logger.info('Loaded %d groups, %d hosts', len(all_group.all_groups),
len(all_group.all_hosts))
return all_group return all_group
@@ -382,7 +418,8 @@ class Command(NoArgsCommand):
self.logger = logging.getLogger('awx.main.commands.inventory_import') self.logger = logging.getLogger('awx.main.commands.inventory_import')
self.logger.setLevel(log_levels.get(self.verbosity, 0)) self.logger.setLevel(log_levels.get(self.verbosity, 0))
handler = logging.StreamHandler() handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter('%(message)s')) formatter = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s')
handler.setFormatter(formatter)
self.logger.addHandler(handler) self.logger.addHandler(handler)
self.logger.propagate = False self.logger.propagate = False
@@ -446,30 +483,32 @@ class Command(NoArgsCommand):
# source attached to a specific group, only delete hosts beneath that # source attached to a specific group, only delete hosts beneath that
# group. Delete each host individually so signal handlers will run. # group. Delete each host individually so signal handlers will run.
if self.overwrite: if self.overwrite:
self.logger.debug('deleting any hosts not in the remote source')
if self.inventory_source.group: if self.inventory_source.group:
del_hosts = self.inventory_source.group.all_hosts del_hosts = self.inventory_source.group.all_hosts
# FIXME: Also include hosts from inventory_source.managed_hosts? # FIXME: Also include hosts from inventory_source.managed_hosts?
else: else:
del_hosts = self.inventory.hosts.all() del_hosts = self.inventory.hosts.all()
del_hosts = del_hosts.exclude(name__in=self.all_group.host_names.keys()) del_hosts = del_hosts.exclude(name__in=self.all_group.all_hosts.keys())
for host in del_hosts: for host in del_hosts:
host_name = host.name
host.delete() host.delete()
self.logger.info('Deleted host "%s"', host_name)
# If overwrite is set, for each group in the database that is NOT in # If overwrite is set, for each group in the database that is NOT in
# the local list, delete it. When importing from a cloud inventory # the local list, delete it. When importing from a cloud inventory
# source attached to a specific group, only delete children of that # source attached to a specific group, only delete children of that
# group. Delete each group individually so signal handlers will run. # group. Delete each group individually so signal handlers will run.
if self.overwrite: if self.overwrite:
self.logger.debug('deleting any groups not in the remote source')
if self.inventory_source.group: if self.inventory_source.group:
del_groups = self.inventory_source.group.all_children del_groups = self.inventory_source.group.all_children
# FIXME: Also include groups from inventory_source.managed_groups? # FIXME: Also include groups from inventory_source.managed_groups?
else: else:
del_groups = self.inventory.groups.all() del_groups = self.inventory.groups.all()
del_groups = del_groups.exclude(name__in=self.all_group.group_names.keys()) del_groups = del_groups.exclude(name__in=self.all_group.all_groups.keys())
for group in del_groups: for group in del_groups:
group_name = group.name
group.delete() group.delete()
self.logger.info('Deleted group "%s"', group_name)
# If overwrite is set, clear all invalid child relationships for groups # If overwrite is set, clear all invalid child relationships for groups
# and all invalid host memberships. When importing from a cloud # and all invalid host memberships. When importing from a cloud
@@ -477,47 +516,54 @@ class Command(NoArgsCommand):
# relationships for hosts and groups that are beneath the inventory # relationships for hosts and groups that are beneath the inventory
# source group. # source group.
if self.overwrite: if self.overwrite:
self.logger.info("clearing any child relationships to rebuild from remote source")
if self.inventory_source.group: if self.inventory_source.group:
db_groups = self.inventory_source.group.all_children db_groups = self.inventory_source.group.all_children
else: else:
db_groups = self.inventory.groups.all() db_groups = self.inventory.groups.all()
for db_group in db_groups: for db_group in db_groups:
db_kids = db_group.children.all() db_children = db_group.children.all()
mem_kids = self.all_group.group_names[db_group.name].child_groups mem_children = self.all_group.all_groups[db_group.name].children
mem_kid_names = [ k.name for k in mem_kids ] mem_children_names = [g.name for g in mem_children]
for db_kid in db_kids: for db_child in db_children.exclude(name__in=mem_children_names):
if db_kid.name not in mem_kid_names: if db_child not in db_group.children.all():
self.logger.debug("removing non-DB kid: %s" % (db_kid.name)) continue
db_group.children.remove(db_kid) db_group.children.remove(db_child)
self.logger.info('Removed group "%s" from group "%s"',
db_child.name, db_group.name)
db_hosts = db_group.hosts.all() db_hosts = db_group.hosts.all()
mem_hosts = self.all_group.group_names[db_group.name].hosts mem_hosts = self.all_group.all_groups[db_group.name].hosts
mem_host_names = [ h.name for h in mem_hosts ] mem_host_names = [h.name for h in mem_hosts]
for db_host in db_hosts: for db_host in db_hosts.exclude(name__in=mem_host_names):
if db_host.name not in mem_host_names: if db_host not in db_group.hosts.all():
self.logger.debug("removing non-DB host: %s" % (db_host.name)) continue
db_group.hosts.remove(db_host) db_group.hosts.remove(db_host)
self.logger.info('Removed host "%s" from group "%s"',
db_host.name, db_group.name)
# Update/overwrite variables from "all" group. If importing from a # Update/overwrite variables from "all" group. If importing from a
# cloud source attached to a specific group, variables will be set on # cloud source attached to a specific group, variables will be set on
# the base group, otherwise they will be set on the inventory. # the base group, otherwise they will be set on the whole inventory.
if self.inventory_source.group: if self.inventory_source.group:
all_obj = self.inventory_source.group all_obj = self.inventory_source.group
all_obj.inventory_sources.add(self.inventory_source) all_obj.inventory_sources.add(self.inventory_source)
all_name = 'group "%s"' % all_obj.name
else: else:
all_obj = self.inventory all_obj = self.inventory
all_name = 'inventory'
db_variables = all_obj.variables_dict db_variables = all_obj.variables_dict
mem_variables = self.all_group.variables
if self.overwrite_vars or self.overwrite: if self.overwrite_vars or self.overwrite:
self.logger.info('replacing inventory variables from "all" group') db_variables = self.all_group.variables
db_variables = mem_variables
else: else:
self.logger.info('updating inventory variables from "all" group') db_variables.update(self.all_group.variables)
db_variables.update(mem_variables) if db_variables != all_obj.variables_dict:
all_obj.variables = json.dumps(db_variables) all_obj.variables = json.dumps(db_variables)
all_obj.save(update_fields=['variables']) all_obj.save(update_fields=['variables'])
if self.overwrite_vars or self.overwrite:
self.logger.info('Replaced %s variables from "all" group', all_name)
else:
self.logger.info('Updated %s variables from "all" group', all_name)
else:
self.logger.info('%s variables unmodified', all_name.capitalize())
# FIXME: Attribute changes to superuser? # FIXME: Attribute changes to superuser?
@@ -525,23 +571,28 @@ class Command(NoArgsCommand):
# the database. Otherwise, update/replace database variables from the # the database. Otherwise, update/replace database variables from the
# imported data. Associate with the inventory source group if # imported data. Associate with the inventory source group if
# importing from cloud inventory source. # importing from cloud inventory source.
for k,v in self.all_group.group_names.iteritems(): for k,v in self.all_group.all_groups.iteritems():
variables = json.dumps(v.variables) variables = json.dumps(v.variables)
defaults = dict(variables=variables, description='imported') defaults = dict(variables=variables, description='imported')
group, created = self.inventory.groups.get_or_create(name=k, group, created = self.inventory.groups.get_or_create(name=k,
defaults=defaults) defaults=defaults)
if created: if created:
self.logger.info('inserting new group %s' % k) self.logger.info('Added new group "%s"', k)
else: else:
self.logger.info('updating existing group %s' % k)
db_variables = group.variables_dict db_variables = group.variables_dict
mem_variables = v.variables
if self.overwrite_vars or self.overwrite: if self.overwrite_vars or self.overwrite:
db_variables = mem_variables db_variables = v.variables
else: else:
db_variables.update(mem_variables) db_variables.update(v.variables)
group.variables = json.dumps(db_variables) if db_variables != group.variables_dict:
group.save(update_fields=['variables']) group.variables = json.dumps(db_variables)
group.save(update_fields=['variables'])
if self.overwrite_vars or self.overwrite:
self.logger.info('Replaced variables for group "%s"', k)
else:
self.logger.info('Updated variables for group "%s"', k)
else:
self.logger.info('Variables unmodified for group "%s"', k)
if self.inventory_source.group: if self.inventory_source.group:
self.inventory_source.group.children.add(group) self.inventory_source.group.children.add(group)
group.inventory_sources.add(self.inventory_source) group.inventory_sources.add(self.inventory_source)
@@ -550,44 +601,59 @@ class Command(NoArgsCommand):
# the database. Otherwise, update/replace database variables from the # the database. Otherwise, update/replace database variables from the
# imported data. Associate with the inventory source group if # imported data. Associate with the inventory source group if
# importing from cloud inventory source. # importing from cloud inventory source.
for k,v in self.all_group.host_names.iteritems(): for k,v in self.all_group.all_hosts.iteritems():
variables = json.dumps(v.variables) variables = json.dumps(v.variables)
defaults = dict(variables=variables, description='imported') defaults = dict(variables=variables, description='imported')
host, created = self.inventory.hosts.get_or_create(name=k, host, created = self.inventory.hosts.get_or_create(name=k,
defaults=defaults) defaults=defaults)
if created: if created:
self.logger.info('inserting new host %s' % k) self.logger.info('Added new host "%s"', k)
else: else:
self.logger.info('updating existing host %s' % k)
db_variables = host.variables_dict db_variables = host.variables_dict
mem_variables = v.variables
if self.overwrite_vars or self.overwrite: if self.overwrite_vars or self.overwrite:
db_variables = mem_variables db_variables = v.variables
else: else:
db_variables.update(mem_variables) db_variables.update(v.variables)
host.variables = json.dumps(db_variables) if db_variables != host.variables_dict:
host.save(update_fields=['variables']) host.variables = json.dumps(db_variables)
host.save(update_fields=['variables'])
if self.overwrite_vars or self.overwrite:
self.logger.info('Replaced variables for host "%s"', k)
else:
self.logger.info('Updated variables for host "%s"', k)
else:
self.logger.info('Variables unmodified for host "%s"', k)
if self.inventory_source.group: if self.inventory_source.group:
self.inventory_source.group.hosts.add(host) self.inventory_source.group.hosts.add(host)
host.inventory_sources.add(self.inventory_source) host.inventory_sources.add(self.inventory_source)
host.update_computed_fields(False, False) host.update_computed_fields(False, False)
# for each host in a mem group, add it to the parents to which it belongs # For each host in a mem group, add it to the parent(s) to which it
for (k,v) in self.all_group.group_names.iteritems(): # belongs.
self.logger.info("adding parent arrangements for %s" % k) for k,v in self.all_group.all_groups.iteritems():
db_group = Group.objects.get(name=k, inventory__pk=self.inventory.pk) if not v.hosts:
mem_hosts = v.hosts continue
for h in mem_hosts: db_group = self.inventory.groups.get(name=k)
db_host = Host.objects.get(name=h.name, inventory__pk=self.inventory.pk) for h in v.hosts:
db_group.hosts.add(db_host) db_host = self.inventory.hosts.get(name=h.name)
self.logger.debug("*** ADDING %s to %s ***" % (db_host, db_group)) if db_host not in db_group.hosts.all():
db_group.hosts.add(db_host)
self.logger.info('Added host "%s" to group "%s"', h.name, k)
else:
self.logger.info('Host "%s" already in group "%s"', h.name, k)
# for each group, draw in child group arrangements # for each group, draw in child group arrangements
for (k,v) in self.all_group.group_names.iteritems(): for k,v in self.all_group.all_groups.iteritems():
db_group = Group.objects.get(inventory=self.inventory, name=k) if not v.children:
for mem_child_group in v.child_groups: continue
db_child = Group.objects.get(inventory=self.inventory, name=mem_child_group.name) db_group = self.inventory.groups.get(name=k)
db_group.children.add(db_child) for g in v.children:
db_child = self.inventory.groups.get(name=g.name)
if db_child not in db_group.hosts.all():
db_group.children.add(db_child)
self.logger.info('Added group "%s" as child of "%s"', g.name, k)
else:
self.logger.info('Group "%s" already child of group "%s"', g.name, k)
def check_license(self): def check_license(self):
reader = LicenseReader() reader = LicenseReader()
@@ -596,10 +662,15 @@ class Command(NoArgsCommand):
free_instances = license_info.get('free_instances', 0) free_instances = license_info.get('free_instances', 0)
new_count = Host.objects.filter(active=True).count() new_count = Host.objects.filter(active=True).count()
if free_instances < 0: if free_instances < 0:
d = {
'new_count': new_count,
'available_instances': available_instances,
}
if license_info.get('demo', False): if license_info.get('demo', False):
raise CommandError("demo mode free license count exceeded, would bring available instances to %s, demo mode allows %s, see http://ansibleworks.com/ansibleworks-awx for licensing information" % (new_count, available_instances)) self.logger.error(DEMO_LICENSE_MESSAGE % d)
else: else:
raise CommandError("number of licensed instances exceeded, would bring available instances to %s, system is licensed for %s, see http://ansibleworks.com/ansibleworks-awx for license extension information" % (new_count, available_instances)) self.logger.error(LICENSE_MESSAGE % d)
raise CommandError('License count exceeded!')
@transaction.commit_on_success @transaction.commit_on_success
def handle_noargs(self, **options): def handle_noargs(self, **options):
@@ -622,6 +693,7 @@ class Command(NoArgsCommand):
if not self.source: if not self.source:
raise CommandError('--source is required') raise CommandError('--source is required')
begin = time.time()
self.load_inventory_from_database() self.load_inventory_from_database()
status, tb, exc = 'error', '', None status, tb, exc = 'error', '', None
@@ -632,24 +704,28 @@ class Command(NoArgsCommand):
self.inventory_update.save() self.inventory_update.save()
transaction.commit() transaction.commit()
self.logger.debug('preparing to load from %s' % self.source) # Load inventory from source.
self.all_group = load_generic(self.source) self.all_group = load_inventory_source(self.source)
self.logger.debug('debugging loaded result:')
self.all_group.debug_tree() self.all_group.debug_tree()
# now that memGroup is correct and supports JSON executables, INI, and trees # Merge/overwrite inventory into database.
# now merge and/or overwrite with the database itself!
self.load_into_database() self.load_into_database()
self.check_license() self.check_license()
self.logger.info("inventory import complete, %s, id=%s" % \ if self.inventory_source.group:
(self.inventory.name, self.inventory.id)) inv_name = 'group "%s"' % (self.inventory_source.group.name)
else:
inv_name = '"%s" (id=%s)' % (self.inventory.name,
self.inventory.id)
self.logger.info('Inventory import completed for %s in %0.1fs',
inv_name, time.time() - begin)
status = 'successful' status = 'successful'
except Exception, e: except Exception, e:
if isinstance(e, KeyboardInterrupt): if isinstance(e, KeyboardInterrupt):
status = 'canceled' status = 'canceled'
exc = e exc = e
elif isinstance(e, CommandError):
exc = e
else: else:
tb = traceback.format_exc() tb = traceback.format_exc()
exc = e exc = e

1
awx/main/models/__init__.py
View File

@@ -686,6 +686,7 @@ class InventorySource(PrimordialModel):
blank=True, blank=True,
default='', default='',
) )
# FIXME: Remove tags field when making other migrations for credential changes!
source_tags = models.CharField( source_tags = models.CharField(
max_length=1024, max_length=1024,
blank=True, blank=True,

10
awx/main/serializers.py
View File

@@ -672,9 +672,9 @@ class InventorySourceSerializer(BaseSerializer):
fields = ('id', 'url', 'related', 'summary_fields', 'created', fields = ('id', 'url', 'related', 'summary_fields', 'created',
'modified', 'inventory', 'group', 'source', 'source_path', 'modified', 'inventory', 'group', 'source', 'source_path',
'source_vars', 'source_username', 'source_password', 'source_vars', 'source_username', 'source_password',
'source_regions', 'source_tags', 'overwrite', 'source_regions', 'overwrite', 'overwrite_vars',
'overwrite_vars', 'update_on_launch', 'update_interval', 'update_on_launch', 'update_interval', 'last_update_failed',
'last_update_failed', 'status', 'last_updated') 'status', 'last_updated')
read_only_fields = ('inventory', 'group') read_only_fields = ('inventory', 'group')
def to_native(self, obj): def to_native(self, obj):
@@ -764,10 +764,6 @@ class InventorySourceSerializer(BaseSerializer):
# FIXME # FIXME
return attrs return attrs
def validate_source_tags(self, attrs, source):
# FIXME
return attrs
class InventoryUpdateSerializer(BaseSerializer): class InventoryUpdateSerializer(BaseSerializer):
class Meta: class Meta:

2
awx/main/signals.py
View File

@@ -41,8 +41,6 @@ def update_inventory_computed_fields(sender, **kwargs):
sender_name = unicode(sender._meta.verbose_name) sender_name = unicode(sender._meta.verbose_name)
if kwargs['signal'] == post_save: if kwargs['signal'] == post_save:
sender_action = 'saved' sender_action = 'saved'
if instance.active: # No need to update for active instances.
return
elif kwargs['signal'] == post_delete: elif kwargs['signal'] == post_delete:
sender_action = 'deleted' sender_action = 'deleted'
elif kwargs['signal'] == m2m_changed and kwargs['action'] in ('post_add', 'post_remove', 'post_clear'): elif kwargs['signal'] == m2m_changed and kwargs['action'] in ('post_add', 'post_remove', 'post_clear'):

3
awx/main/tasks.py
View File

@@ -805,7 +805,8 @@ class RunInventoryUpdate(BaseTask):
args.append(rax_path) args.append(rax_path)
elif inventory_source.source == 'file': elif inventory_source.source == 'file':
args.append(inventory_source.source_path) args.append(inventory_source.source_path)
args.append('-v2') verbosity = getattr(settings, 'INVENTORY_UPDATE_VERBOSITY', 1)
args.append('-v%d' % verbosity)
if settings.DEBUG: if settings.DEBUG:
args.append('--traceback') args.append('--traceback')
return args return args

4
awx/main/tests/inventory.py
View File

@@ -5,6 +5,7 @@
import datetime import datetime
import json import json
import os import os
import re
# Django # Django
from django.conf import settings from django.conf import settings
@@ -1021,6 +1022,9 @@ class InventoryUpdatesTest(BaseTransactionTest):
source_pks = group.inventory_sources.values_list('pk', flat=True) source_pks = group.inventory_sources.values_list('pk', flat=True)
self.assertTrue(inventory_source.pk in source_pks) self.assertTrue(inventory_source.pk in source_pks)
self.assertTrue(group.has_inventory_sources) self.assertTrue(group.has_inventory_sources)
# Make sure EC2 instance ID groups are excluded.
self.assertFalse(re.match(r'^i-[0-9a-f]{8}$', group.name, re.I),
group.name)
def test_update_from_ec2(self): def test_update_from_ec2(self):
source_username = getattr(settings, 'TEST_AWS_ACCESS_KEY_ID', '') source_username = getattr(settings, 'TEST_AWS_ACCESS_KEY_ID', '')

3
awx/plugins/inventory/ec2.py
View File

@@ -336,8 +336,9 @@ class Ec2Inventory(object):
# Add to index # Add to index
self.index[dest] = [region, instance.id] self.index[dest] = [region, instance.id]
# For AWX: do not output group based on instance ID!!!
# Inventory: Group by instance ID (always a group of 1) # Inventory: Group by instance ID (always a group of 1)
self.inventory[instance.id] = [dest] # self.inventory[instance.id] = [dest]
# Inventory: Group by region # Inventory: Group by region
self.push(self.inventory, region, dest) self.push(self.inventory, region, dest)

5
awx/settings/local_settings.py.example
View File

@@ -74,9 +74,12 @@ REMOTE_HOST_HEADERS = ['REMOTE_ADDR', 'REMOTE_HOST']
# the celery task. # the celery task.
#AWX_TASK_ENV['FOO'] = 'BAR' #AWX_TASK_ENV['FOO'] = 'BAR'
# If set, uses -vvv for project updates instead of just -v for more output. # If set, use -vvv for project updates instead of -v for more output.
# PROJECT_UPDATE_VVV=True # PROJECT_UPDATE_VVV=True
# Set verbosity for inventory import command when running inventory updates.
# INVENTORY_UPDATE_VERBOSITY=1
############################################################################### ###############################################################################
# EMAIL SETTINGS # EMAIL SETTINGS
############################################################################### ###############################################################################

13
awx/ui/static/js/forms/Groups.js
View File

@@ -155,19 +155,6 @@ angular.module('GroupFormDefinition', [])
"Only hosts associated with the list of regions will be included in the update process.</p>", "Only hosts associated with the list of regions will be included in the update process.</p>",
dataContainer: 'body' dataContainer: 'body'
}, },
source_tags: {
label: 'Tags',
excludeModal: true,
type: 'text',
ngShow: "source.value == 'ec2'",
addRequired: false,
editRequired: false,
dataTitle: 'Source Regions',
dataPlacement: 'left',
awPopOver: "<p>Comma separated list of tags. Tag names must match those defined at the inventory source." +
" Only hosts associated with the list of tags will be included in the update process.</p>",
dataContainer: 'body'
},
source_vars: { source_vars: {
label: 'Source Variables', label: 'Source Variables',
ngShow: "source.value == 'file' || source.value == 'ec2'", ngShow: "source.value == 'file' || source.value == 'ec2'",

1
awx/ui/static/js/helpers/Groups.js
View File

@@ -824,7 +824,6 @@ angular.module('GroupsHelper', [ 'RestServices', 'Utilities', 'ListGenerator', '
source_username: scope['source_username'], source_username: scope['source_username'],
source_password: scope['source_password'], source_password: scope['source_password'],
source_regions: scope['source_regions'], source_regions: scope['source_regions'],
source_tags: scope['source_tags'],
overwrite: scope['overwrite'], overwrite: scope['overwrite'],
overwrite_vars: scope['overwrite_vars'], overwrite_vars: scope['overwrite_vars'],
update_on_launch: scope['update_on_launch'], update_on_launch: scope['update_on_launch'],