diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py
index 9ce47a268b..dd25ffc18b 100644
--- a/awx/main/management/commands/inventory_import.py
+++ b/awx/main/management/commands/inventory_import.py
@@ -10,6 +10,7 @@ import os
import shlex
import subprocess
import sys
+import time
import traceback
# PyYAML
@@ -26,191 +27,214 @@ from awx.main.licenses import LicenseReader
logger = logging.getLogger('awx.main.commands.inventory_import')
-class ImportException(BaseException):
+LICENSE_MESSAGE = '''\
+Number of licensed instances exceeded, would bring available instances to %(new_count)d, system is licensed for %(available_instances)d.
+See http://ansibleworks.com/ansibleworks-awx for license extension information.'''
- def __init__(self, msg):
- self.msg = msg
+DEMO_LICENSE_MESSAGE = '''\
+Demo mode free license count exceeded, would bring available instances to %(new_count)d, demo mode allows %(available_instances)d.
+See http://ansibleworks.com/ansibleworks-awx for licensing information.'''
- def __str__(self):
- return "Import Error: %s" % msg
-
-class MemGroup(object):
-
- def __init__(self, name, inventory_base):
-
- assert inventory_base is not None
- self.inventory_base = inventory_base
+class MemObject(object):
+ '''
+ Common code shared between in-memory groups and hosts.
+ '''
+
+ def __init__(self, name, source_dir):
+ assert name
+ assert source_dir
self.name = name
- self.child_groups = []
+ self.source_dir = source_dir
+
+ def load_vars(self, path):
+ if os.path.exists(path) and os.path.isfile(path):
+ vars_name = os.path.basename(os.path.dirname(path))
+ logger.debug('Loading %s from %s', vars_name, path)
+ try:
+ v = yaml.safe_load(file(path, 'r').read())
+ return v if hasattr(v, 'items') else {}
+ except yaml.YAMLError, e:
+ if hasattr(e, 'problem_mark'):
+ logger.error('Invalid YAML in %s:%s col %s', path,
+ e.problem_mark.line + 1,
+ e.problem_mark.column + 1)
+ else:
+ logger.error('Error loading YAML from %s', path)
+ raise
+ return {}
+
+
+class MemGroup(MemObject):
+ '''
+ In-memory representation of an inventory group.
+ '''
+
+ def __init__(self, name, source_dir):
+ super(MemGroup, self).__init__(name, source_dir)
+ self.children = []
self.hosts = []
self.variables = {}
self.parents = []
# Used on the "all" group in place of previous global variables.
# maps host and group names to hosts to prevent redudant additions
- self.host_names = {}
- self.group_names = {}
+ self.all_hosts = {}
+ self.all_groups = {}
- group_vars = os.path.join(inventory_base, 'group_vars', name)
- if os.path.exists(group_vars):
- logger.debug("loading group_vars")
- self.variables = yaml.load(open(group_vars).read())
-
- def child_group_by_name(self, grp_name, loader):
- logger.debug("looking for child group: %s" % grp_name)
- if grp_name == 'all':
+ group_vars = os.path.join(self.source_dir, 'group_vars', self.name)
+ self.variables = self.load_vars(group_vars)
+ logger.debug('Loaded group: %s', self.name)
+
+ def child_group_by_name(self, name, loader):
+ if name == 'all':
return
+ logger.debug('Looking for %s as child group of %s', name, self.name)
# slight hack here, passing in 'self' for all_group but child=True won't use it
- grp = loader.get_group(grp_name, self, child=True)
+ group = loader.get_group(name, self, child=True)
# don't add to child groups if already there
- for x in self.child_groups:
- if x.name == grp_name:
- return x
- logger.debug("adding child group %s to group %s" % (grp.name, self.name))
- self.child_groups.append(grp)
- return grp
+ for g in self.children:
+ if g.name == name:
+ return g
+ logger.debug('Adding child group %s to group %s', group.name, self.name)
+ self.children.append(group)
+ return group
- def add_child_group(self, grp):
- assert grp.name is not 'all'
-
- logger.debug("adding child group %s to group %s" % (grp.name, self.name))
-
- assert type(grp) == MemGroup
- if grp not in self.child_groups:
- self.child_groups.append(grp)
- if not self in grp.parents:
- grp.parents.append(self)
+ def add_child_group(self, group):
+ assert group.name is not 'all'
+ assert isinstance(group, MemGroup)
+ logger.debug('Adding child group %s to parent %s', group.name, self.name)
+ if group not in self.children:
+ self.children.append(group)
+ if not self in group.parents:
+ group.parents.append(self)
def add_host(self, host):
- logger.debug("adding host %s to group %s" % (host.name, self.name))
-
- assert type(host) == MemHost
+ assert isinstance(host, MemHost)
+ logger.debug('Adding host %s to group %s', host.name, self.name)
if host not in self.hosts:
self.hosts.append(host)
- def debug_tree(self):
- logger.debug("describing tree of group (%s)" % self.name)
-
- logger.debug("group: %s, %s" % (self.name, self.variables))
- for x in self.child_groups:
- logger.debug(" child: %s" % (x.name))
- for x in self.hosts:
- logger.debug(" host: %s, %s" % (x.name, x.variables))
+ def debug_tree(self, group_names=None):
+ group_names = group_names or set()
+ if self.name in group_names:
+ return
+ logger.debug('Dumping tree for group "%s":', self.name)
+ logger.debug('- Vars: %r', self.variables)
+ for h in self.hosts:
+ logger.debug('- Host: %s, %r', h.name, h.variables)
+ for g in self.children:
+ logger.debug('- Child: %s', g.name)
+ logger.debug('----')
+ group_names.add(self.name)
+ for g in self.children:
+ g.debug_tree(group_names)
- logger.debug("---")
- for x in self.child_groups:
- x.debug_tree()
-class MemHost(object):
+class MemHost(MemObject):
+ '''
+ In-memory representation of an inventory host.
+ '''
- def __init__(self, name, inventory_base):
- logger.debug("adding host name: %s" % name)
- assert name is not None
- assert inventory_base is not None
-
- # set ansible_ssh_port if ":" in name
- self.name = name
+ def __init__(self, name, source_dir):
+ super(MemHost, self).__init__(name, source_dir)
self.variables = {}
- self.inventory_base = inventory_base
if ':' in name:
tokens = name.split(":")
self.name = tokens[0]
self.variables['ansible_ssh_port'] = int(tokens[1])
- if "[" in name:
- raise ImportException("block ranges like host[0:50].example.com are not yet supported by the importer")
+ if '[' in name:
+ raise ValueError('Block ranges like host[0:50].example.com are not yet supported by the importer')
+
+ host_vars = os.path.join(source_dir, 'host_vars', name)
+ self.variables.update(self.load_vars(host_vars))
+ logger.debug('Loaded host: %s', self.name)
+
- host_vars = os.path.join(inventory_base, 'host_vars', name)
- if os.path.exists(host_vars):
- logger.debug("loading host_vars")
- self.variables.update(yaml.load(open(host_vars).read()))
-
class BaseLoader(object):
+ '''
+ Common functions for an inventory loader from a given source.
+ '''
- def __init__(self, inventory_base=None, all_group=None):
- self.inventory_base = inventory_base
- self.all_group = all_group
+ def __init__(self, source, all_group=None):
+ self.source = source
+ self.source_dir = os.path.dirname(self.source)
+ self.all_group = all_group or MemGroup('all', self.source_dir)
def get_host(self, name):
+ '''
+ Return a MemHost instance from host name, creating if needed.
+ '''
host_name = name.split(':')[0]
host = None
- if not host_name in self.all_group.host_names:
- host = MemHost(name, self.inventory_base)
- self.all_group.host_names[host_name] = host
- return self.all_group.host_names[host_name]
+ if not host_name in self.all_group.all_hosts:
+ host = MemHost(name, self.source_dir)
+ self.all_group.all_hosts[host_name] = host
+ return self.all_group.all_hosts[host_name]
def get_group(self, name, all_group=None, child=False):
+ '''
+ Return a MemGroup instance from group name, creating if needed.
+ '''
all_group = all_group or self.all_group
if name == 'all':
return all_group
- if not name in self.all_group.group_names:
- group = MemGroup(name, self.inventory_base)
+ if not name in self.all_group.all_groups:
+ group = MemGroup(name, self.source_dir)
if not child:
all_group.add_child_group(group)
- self.all_group.group_names[name] = group
- return self.all_group.group_names[name]
+ self.all_group.all_groups[name] = group
+ return self.all_group.all_groups[name]
- def load(self, src):
+ def load(self):
raise NotImplementedError
+
class IniLoader(BaseLoader):
+ '''
+ Loader to read inventory from an INI-formatted text file.
+ '''
- def __init__(self, inventory_base=None, all_group=None):
- super(IniLoader, self).__init__(inventory_base, all_group)
- logger.debug("processing ini")
-
- def load(self, src):
- logger.debug("loading: %s on %s" % (src, self.all_group))
-
- if self.inventory_base is None:
- self.inventory_base = os.path.dirname(src)
-
- data = open(src).read()
- lines = data.split("\n")
+ def load(self):
+ logger.info('Reading INI source: %s', self.source)
group = self.all_group
input_mode = 'host'
-
- for line in lines:
+ for line in file(self.source, 'r'):
line = line.split('#')[0].strip()
if not line:
continue
- elif line.startswith("["):
- # mode change, possible new group name
- line = line.replace("[","").replace("]","").lstrip().rstrip()
- if line.find(":vars") != -1:
+ elif line.startswith('[') and line.endswith(']'):
+ # Mode change, possible new group name
+ line = line[1:-1].strip()
+ if line.endswith(':vars'):
input_mode = 'vars'
- line = line.replace(":vars","")
- group = self.get_group(line)
- elif line.find(":children") != -1:
- input_mode = 'children'
- line = line.replace(":children","")
- group = self.get_group(line)
+ line = line[:-5]
+ elif line.endswith(':children'):
+ input_mode = 'children'
+ line = line[:-9]
else:
input_mode = 'host'
- group = self.get_group(line)
+ group = self.get_group(line)
else:
- # add a host or variable to the existing group/host
+ # Add a host or variable to the existing group/host
tokens = shlex.split(line)
-
if input_mode == 'host':
- new_host = self.get_host(tokens[0])
+ host = self.get_host(tokens[0])
if len(tokens) > 1:
- variables = {}
for t in tokens[1:]:
- (k,v) = t.split("=",1)
- new_host.variables[k] = v
- group.add_host(new_host)
-
+ k,v = t.split('=', 1)
+ host.variables[k] = v
+ group.add_host(host)
elif input_mode == 'children':
group.child_group_by_name(line, self)
elif input_mode == 'vars':
for t in tokens:
- (k, v) = t.split("=", 1)
+ k, v = t.split('=', 1)
group.variables[k] = v
-
# TODO: expansion patterns are probably not going to be supported
+
# from API documentation:
#
# if called with --list, inventory outputs like so:
@@ -236,12 +260,8 @@ class IniLoader(BaseLoader):
#
# if called with --host outputs JSON for that host
-class ExecutableJsonLoader(BaseLoader):
- def __init__(self, inventory_base=None, all_group=None):
- super(ExecutableJsonLoader, self).__init__(inventory_base, all_group)
- logger.debug("processing executable JSON source")
- self.child_group_names = {}
+class ExecutableJsonLoader(BaseLoader):
def command_to_json(self, cmd):
data = {}
@@ -250,101 +270,117 @@ class ExecutableJsonLoader(BaseLoader):
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = proc.communicate()
if proc.returncode != 0:
- raise Exception("%s list failed %s with output: %s" % (cmd, stderr, proc.returncode))
+ raise RuntimeError('%r failed (rc=%d) with output: %s' % (cmd, proc.returncode, stderr))
data = json.loads(stdout)
+ if not isinstance(data, dict):
+ raise TypeError('Returned JSON must be a dictionary, got %s instead' % str(type(data)))
except:
- traceback.print_exc()
- raise Exception("failed to load JSON output: %s" % stdout)
- assert type(data) == dict
+ logger.error('Failed to load JSON from: %s', stdout)
+ raise
return data
- def load(self, src):
-
- logger.debug("loading %s onto %s" % (src, self.all_group))
-
- if self.inventory_base is None:
- self.inventory_base = os.path.dirname(src)
-
- data = self.command_to_json([src, "--list"])
-
- group = None
+ def load(self):
+ logger.info('Reading executable JSON source: %s', self.source)
+ data = self.command_to_json([self.source, '--list'])
_meta = data.pop('_meta', {})
- for (k,v) in data.iteritems():
-
+ for k,v in data.iteritems():
group = self.get_group(k)
- if type(v) == dict:
-
- # process hosts
- host_details = v.get('hosts', None)
- if host_details is not None:
- if type(host_details) == dict:
- for (hk, hv) in host_details.iteritems():
- host = self.get_host(hk)
- host.variables.update(hv)
- group.add_host(host)
- if type(host_details) == list:
- for hk in host_details:
- host = self.get_host(hk)
- group.add_host(host)
-
- # process variables
- vars = v.get('vars', None)
- if vars is not None:
+ # Load group hosts/vars/children from a dictionary.
+ if isinstance(v, dict):
+ # Process hosts within a group.
+ hosts = v.get('hosts', {})
+ if isinstance(hosts, dict):
+ for hk, hv in hosts.iteritems():
+ host = self.get_host(hk)
+ if isinstance(hv, dict):
+ host.variables.update(hv)
+ else:
+ self.logger.warning('Expected dict of vars for '
+ 'host "%s", got %s instead',
+ hk, str(type(hv)))
+ group.add_host(host)
+ elif isinstance(hosts, (list, tuple)):
+ for hk in hosts:
+ host = self.get_host(hk)
+ group.add_host(host)
+ else:
+ logger.warning('Expected dict or list of "hosts" for '
+ 'group "%s", got %s instead', k,
+ str(type(hosts)))
+ # Process group variables.
+ vars = v.get('vars', {})
+ if isinstance(vars, dict):
group.variables.update(vars)
-
- # process child groups
- children_details = v.get('children', None)
- if children_details is not None:
- for x in children_details:
- child = self.get_group(x, self.inventory_base, child=True)
+ else:
+ self.logger.warning('Expected dict of vars for '
+ 'group "%s", got %s instead',
+ k, str(type(vars)))
+ # Process child groups.
+ children = v.get('children', [])
+ if isinstance(children, (list, tuple)):
+ for c in children:
+ child = self.get_group(c, self.all_group, child=True)
group.add_child_group(child)
- self.child_group_names[x] = child
+ else:
+ self.logger.warning('Expected list of children for '
+ 'group "%s", got %s instead',
+ k, str(type(children)))
- if type(v) in (tuple, list):
- for x in v:
- host = self.get_host(x)
- group.add_host(host)
+ # Load host names from a list.
+ elif isinstance(v, (list, tuple)):
+ for h in v:
+ host = self.get_host(h)
+ group.add_host(host)
+ else:
+ logger.warning('')
+ self.logger.warning('Expected dict or list for group "%s", '
+ 'got %s instead', k, str(type(v)))
if k != 'all':
self.all_group.add_child_group(group)
-
- # then we invoke the executable once for each host name we've built up
+ # Invoke the executable once for each host name we've built up
# to set their variables
- for (k,v) in self.all_group.host_names.iteritems():
+ for k,v in self.all_group.all_hosts.iteritems():
if 'hostvars' not in _meta:
- data = self.command_to_json([src, "--host", k])
+ data = self.command_to_json([self.source, '--host', k])
else:
data = _meta['hostvars'].get(k, {})
- v.variables.update(data)
+ if isinstance(data, dict):
+ v.variables.update(data)
+ else:
+ self.logger.warning('Expected dict of vars for '
+ 'host "%s", got %s instead',
+ k, str(type(data)))
-def load_generic(src):
- logger.debug("analyzing type of source")
- if not os.path.exists(src):
- logger.debug("source missing")
- raise CommandError("source does not exist")
- if os.path.isdir(src):
- all_group = MemGroup('all', src)
- for f in glob.glob("%s/*" % src):
- if f.endswith(".ini"):
- # config files for inventory scripts should be ignored
- continue
- if not os.path.isdir(f):
- if os.access(f, os.X_OK):
- ExecutableJsonLoader(None, all_group).load(f)
- else:
- IniLoader(None, all_group).load(f)
- elif os.access(src, os.X_OK):
- all_group = MemGroup('all', os.path.dirname(src))
- ExecutableJsonLoader(None, all_group).load(src)
+def load_inventory_source(source, all_group=None):
+ '''
+ Load inventory from given source directory or file.
+ '''
+ logger.debug('Analyzing type of source: %s', source)
+ original_all_group = all_group
+ if not os.path.exists(source):
+ raise CommandError('Source does not exist: %s' % source)
+ if os.path.isdir(source):
+ all_group = all_group or MemGroup('all', source)
+ for filename in glob.glob(os.path.join(source, '*')):
+ if filename.endswith(".ini") or os.path.isdir(filename):
+ continue
+ load_inventory_source(filename, all_group)
else:
- all_group = MemGroup('all', os.path.dirname(src))
- IniLoader(None, all_group).load(src)
+ all_group = all_group or MemGroup('all', os.path.dirname(source))
+ if os.access(source, os.X_OK):
+ ExecutableJsonLoader(source, all_group).load()
+ else:
+ IniLoader(source, all_group).load()
- logger.debug("loading process complete")
+ logger.debug('Finished loading from source: %s', source)
+ if original_all_group is None:
+ logger.info('Loaded %d groups, %d hosts', len(all_group.all_groups),
+ len(all_group.all_hosts))
return all_group
@@ -382,7 +418,8 @@ class Command(NoArgsCommand):
self.logger = logging.getLogger('awx.main.commands.inventory_import')
self.logger.setLevel(log_levels.get(self.verbosity, 0))
handler = logging.StreamHandler()
- handler.setFormatter(logging.Formatter('%(message)s'))
+ formatter = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s')
+ handler.setFormatter(formatter)
self.logger.addHandler(handler)
self.logger.propagate = False
@@ -446,30 +483,32 @@ class Command(NoArgsCommand):
# source attached to a specific group, only delete hosts beneath that
# group. Delete each host individually so signal handlers will run.
if self.overwrite:
- self.logger.debug('deleting any hosts not in the remote source')
if self.inventory_source.group:
del_hosts = self.inventory_source.group.all_hosts
# FIXME: Also include hosts from inventory_source.managed_hosts?
else:
del_hosts = self.inventory.hosts.all()
- del_hosts = del_hosts.exclude(name__in=self.all_group.host_names.keys())
+ del_hosts = del_hosts.exclude(name__in=self.all_group.all_hosts.keys())
for host in del_hosts:
+ host_name = host.name
host.delete()
+ self.logger.info('Deleted host "%s"', host_name)
# If overwrite is set, for each group in the database that is NOT in
# the local list, delete it. When importing from a cloud inventory
# source attached to a specific group, only delete children of that
# group. Delete each group individually so signal handlers will run.
if self.overwrite:
- self.logger.debug('deleting any groups not in the remote source')
if self.inventory_source.group:
del_groups = self.inventory_source.group.all_children
# FIXME: Also include groups from inventory_source.managed_groups?
else:
del_groups = self.inventory.groups.all()
- del_groups = del_groups.exclude(name__in=self.all_group.group_names.keys())
+ del_groups = del_groups.exclude(name__in=self.all_group.all_groups.keys())
for group in del_groups:
+ group_name = group.name
group.delete()
+ self.logger.info('Deleted group "%s"', group_name)
# If overwrite is set, clear all invalid child relationships for groups
# and all invalid host memberships. When importing from a cloud
@@ -477,47 +516,54 @@ class Command(NoArgsCommand):
# relationships for hosts and groups that are beneath the inventory
# source group.
if self.overwrite:
- self.logger.info("clearing any child relationships to rebuild from remote source")
if self.inventory_source.group:
db_groups = self.inventory_source.group.all_children
else:
db_groups = self.inventory.groups.all()
-
for db_group in db_groups:
- db_kids = db_group.children.all()
- mem_kids = self.all_group.group_names[db_group.name].child_groups
- mem_kid_names = [ k.name for k in mem_kids ]
- for db_kid in db_kids:
- if db_kid.name not in mem_kid_names:
- self.logger.debug("removing non-DB kid: %s" % (db_kid.name))
- db_group.children.remove(db_kid)
-
+ db_children = db_group.children.all()
+ mem_children = self.all_group.all_groups[db_group.name].children
+ mem_children_names = [g.name for g in mem_children]
+ for db_child in db_children.exclude(name__in=mem_children_names):
+ if db_child not in db_group.children.all():
+ continue
+ db_group.children.remove(db_child)
+ self.logger.info('Removed group "%s" from group "%s"',
+ db_child.name, db_group.name)
db_hosts = db_group.hosts.all()
- mem_hosts = self.all_group.group_names[db_group.name].hosts
- mem_host_names = [ h.name for h in mem_hosts ]
- for db_host in db_hosts:
- if db_host.name not in mem_host_names:
- self.logger.debug("removing non-DB host: %s" % (db_host.name))
- db_group.hosts.remove(db_host)
+ mem_hosts = self.all_group.all_groups[db_group.name].hosts
+ mem_host_names = [h.name for h in mem_hosts]
+ for db_host in db_hosts.exclude(name__in=mem_host_names):
+ if db_host not in db_group.hosts.all():
+ continue
+ db_group.hosts.remove(db_host)
+ self.logger.info('Removed host "%s" from group "%s"',
+ db_host.name, db_group.name)
# Update/overwrite variables from "all" group. If importing from a
# cloud source attached to a specific group, variables will be set on
- # the base group, otherwise they will be set on the inventory.
+ # the base group, otherwise they will be set on the whole inventory.
if self.inventory_source.group:
all_obj = self.inventory_source.group
all_obj.inventory_sources.add(self.inventory_source)
+ all_name = 'group "%s"' % all_obj.name
else:
all_obj = self.inventory
+ all_name = 'inventory'
db_variables = all_obj.variables_dict
- mem_variables = self.all_group.variables
if self.overwrite_vars or self.overwrite:
- self.logger.info('replacing inventory variables from "all" group')
- db_variables = mem_variables
+ db_variables = self.all_group.variables
else:
- self.logger.info('updating inventory variables from "all" group')
- db_variables.update(mem_variables)
- all_obj.variables = json.dumps(db_variables)
- all_obj.save(update_fields=['variables'])
+ db_variables.update(self.all_group.variables)
+ if db_variables != all_obj.variables_dict:
+ all_obj.variables = json.dumps(db_variables)
+ all_obj.save(update_fields=['variables'])
+ if self.overwrite_vars or self.overwrite:
+ self.logger.info('Replaced %s variables from "all" group', all_name)
+ else:
+ self.logger.info('Updated %s variables from "all" group', all_name)
+ else:
+ self.logger.info('%s variables unmodified', all_name.capitalize())
# FIXME: Attribute changes to superuser?
@@ -525,23 +571,28 @@ class Command(NoArgsCommand):
# the database. Otherwise, update/replace database variables from the
# imported data. Associate with the inventory source group if
# importing from cloud inventory source.
- for k,v in self.all_group.group_names.iteritems():
+ for k,v in self.all_group.all_groups.iteritems():
variables = json.dumps(v.variables)
defaults = dict(variables=variables, description='imported')
group, created = self.inventory.groups.get_or_create(name=k,
defaults=defaults)
if created:
- self.logger.info('inserting new group %s' % k)
+ self.logger.info('Added new group "%s"', k)
else:
- self.logger.info('updating existing group %s' % k)
db_variables = group.variables_dict
- mem_variables = v.variables
if self.overwrite_vars or self.overwrite:
- db_variables = mem_variables
+ db_variables = v.variables
else:
- db_variables.update(mem_variables)
- group.variables = json.dumps(db_variables)
- group.save(update_fields=['variables'])
+ db_variables.update(v.variables)
+ if db_variables != group.variables_dict:
+ group.variables = json.dumps(db_variables)
+ group.save(update_fields=['variables'])
+ if self.overwrite_vars or self.overwrite:
+ self.logger.info('Replaced variables for group "%s"', k)
+ else:
+ self.logger.info('Updated variables for group "%s"', k)
+ else:
+ self.logger.info('Variables unmodified for group "%s"', k)
if self.inventory_source.group:
self.inventory_source.group.children.add(group)
group.inventory_sources.add(self.inventory_source)
@@ -550,44 +601,59 @@ class Command(NoArgsCommand):
# the database. Otherwise, update/replace database variables from the
# imported data. Associate with the inventory source group if
# importing from cloud inventory source.
- for k,v in self.all_group.host_names.iteritems():
+ for k,v in self.all_group.all_hosts.iteritems():
variables = json.dumps(v.variables)
defaults = dict(variables=variables, description='imported')
host, created = self.inventory.hosts.get_or_create(name=k,
defaults=defaults)
if created:
- self.logger.info('inserting new host %s' % k)
+ self.logger.info('Added new host "%s"', k)
else:
- self.logger.info('updating existing host %s' % k)
db_variables = host.variables_dict
- mem_variables = v.variables
if self.overwrite_vars or self.overwrite:
- db_variables = mem_variables
+ db_variables = v.variables
else:
- db_variables.update(mem_variables)
- host.variables = json.dumps(db_variables)
- host.save(update_fields=['variables'])
+ db_variables.update(v.variables)
+ if db_variables != host.variables_dict:
+ host.variables = json.dumps(db_variables)
+ host.save(update_fields=['variables'])
+ if self.overwrite_vars or self.overwrite:
+ self.logger.info('Replaced variables for host "%s"', k)
+ else:
+ self.logger.info('Updated variables for host "%s"', k)
+ else:
+ self.logger.info('Variables unmodified for host "%s"', k)
if self.inventory_source.group:
self.inventory_source.group.hosts.add(host)
host.inventory_sources.add(self.inventory_source)
host.update_computed_fields(False, False)
- # for each host in a mem group, add it to the parents to which it belongs
- for (k,v) in self.all_group.group_names.iteritems():
- self.logger.info("adding parent arrangements for %s" % k)
- db_group = Group.objects.get(name=k, inventory__pk=self.inventory.pk)
- mem_hosts = v.hosts
- for h in mem_hosts:
- db_host = Host.objects.get(name=h.name, inventory__pk=self.inventory.pk)
- db_group.hosts.add(db_host)
- self.logger.debug("*** ADDING %s to %s ***" % (db_host, db_group))
+ # For each host in a mem group, add it to the parent(s) to which it
+ # belongs.
+ for k,v in self.all_group.all_groups.iteritems():
+ if not v.hosts:
+ continue
+ db_group = self.inventory.groups.get(name=k)
+ for h in v.hosts:
+ db_host = self.inventory.hosts.get(name=h.name)
+ if db_host not in db_group.hosts.all():
+ db_group.hosts.add(db_host)
+ self.logger.info('Added host "%s" to group "%s"', h.name, k)
+ else:
+ self.logger.info('Host "%s" already in group "%s"', h.name, k)
# for each group, draw in child group arrangements
- for (k,v) in self.all_group.group_names.iteritems():
- db_group = Group.objects.get(inventory=self.inventory, name=k)
- for mem_child_group in v.child_groups:
- db_child = Group.objects.get(inventory=self.inventory, name=mem_child_group.name)
- db_group.children.add(db_child)
+ for k,v in self.all_group.all_groups.iteritems():
+ if not v.children:
+ continue
+ db_group = self.inventory.groups.get(name=k)
+ for g in v.children:
+ db_child = self.inventory.groups.get(name=g.name)
+ if db_child not in db_group.hosts.all():
+ db_group.children.add(db_child)
+ self.logger.info('Added group "%s" as child of "%s"', g.name, k)
+ else:
+ self.logger.info('Group "%s" already child of group "%s"', g.name, k)
def check_license(self):
reader = LicenseReader()
@@ -596,10 +662,15 @@ class Command(NoArgsCommand):
free_instances = license_info.get('free_instances', 0)
new_count = Host.objects.filter(active=True).count()
if free_instances < 0:
+ d = {
+ 'new_count': new_count,
+ 'available_instances': available_instances,
+ }
if license_info.get('demo', False):
- raise CommandError("demo mode free license count exceeded, would bring available instances to %s, demo mode allows %s, see http://ansibleworks.com/ansibleworks-awx for licensing information" % (new_count, available_instances))
+ self.logger.error(DEMO_LICENSE_MESSAGE % d)
else:
- raise CommandError("number of licensed instances exceeded, would bring available instances to %s, system is licensed for %s, see http://ansibleworks.com/ansibleworks-awx for license extension information" % (new_count, available_instances))
+ self.logger.error(LICENSE_MESSAGE % d)
+ raise CommandError('License count exceeded!')
@transaction.commit_on_success
def handle_noargs(self, **options):
@@ -622,6 +693,7 @@ class Command(NoArgsCommand):
if not self.source:
raise CommandError('--source is required')
+ begin = time.time()
self.load_inventory_from_database()
status, tb, exc = 'error', '', None
@@ -632,24 +704,28 @@ class Command(NoArgsCommand):
self.inventory_update.save()
transaction.commit()
- self.logger.debug('preparing to load from %s' % self.source)
- self.all_group = load_generic(self.source)
- self.logger.debug('debugging loaded result:')
+ # Load inventory from source.
+ self.all_group = load_inventory_source(self.source)
self.all_group.debug_tree()
- # now that memGroup is correct and supports JSON executables, INI, and trees
- # now merge and/or overwrite with the database itself!
-
+ # Merge/overwrite inventory into database.
self.load_into_database()
self.check_license()
- self.logger.info("inventory import complete, %s, id=%s" % \
- (self.inventory.name, self.inventory.id))
+ if self.inventory_source.group:
+ inv_name = 'group "%s"' % (self.inventory_source.group.name)
+ else:
+ inv_name = '"%s" (id=%s)' % (self.inventory.name,
+ self.inventory.id)
+ self.logger.info('Inventory import completed for %s in %0.1fs',
+ inv_name, time.time() - begin)
status = 'successful'
except Exception, e:
if isinstance(e, KeyboardInterrupt):
status = 'canceled'
exc = e
+ elif isinstance(e, CommandError):
+ exc = e
else:
tb = traceback.format_exc()
exc = e
diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py
index 04c9c022de..e39f7b667a 100644
--- a/awx/main/models/__init__.py
+++ b/awx/main/models/__init__.py
@@ -686,6 +686,7 @@ class InventorySource(PrimordialModel):
blank=True,
default='',
)
+ # FIXME: Remove tags field when making other migrations for credential changes!
source_tags = models.CharField(
max_length=1024,
blank=True,
diff --git a/awx/main/serializers.py b/awx/main/serializers.py
index d2bd05dea7..6cc7f8fb7f 100644
--- a/awx/main/serializers.py
+++ b/awx/main/serializers.py
@@ -672,9 +672,9 @@ class InventorySourceSerializer(BaseSerializer):
fields = ('id', 'url', 'related', 'summary_fields', 'created',
'modified', 'inventory', 'group', 'source', 'source_path',
'source_vars', 'source_username', 'source_password',
- 'source_regions', 'source_tags', 'overwrite',
- 'overwrite_vars', 'update_on_launch', 'update_interval',
- 'last_update_failed', 'status', 'last_updated')
+ 'source_regions', 'overwrite', 'overwrite_vars',
+ 'update_on_launch', 'update_interval', 'last_update_failed',
+ 'status', 'last_updated')
read_only_fields = ('inventory', 'group')
def to_native(self, obj):
@@ -764,10 +764,6 @@ class InventorySourceSerializer(BaseSerializer):
# FIXME
return attrs
- def validate_source_tags(self, attrs, source):
- # FIXME
- return attrs
-
class InventoryUpdateSerializer(BaseSerializer):
class Meta:
diff --git a/awx/main/signals.py b/awx/main/signals.py
index c2ac5ec58c..9bcf1bb9f3 100644
--- a/awx/main/signals.py
+++ b/awx/main/signals.py
@@ -41,8 +41,6 @@ def update_inventory_computed_fields(sender, **kwargs):
sender_name = unicode(sender._meta.verbose_name)
if kwargs['signal'] == post_save:
sender_action = 'saved'
- if instance.active: # No need to update for active instances.
- return
elif kwargs['signal'] == post_delete:
sender_action = 'deleted'
elif kwargs['signal'] == m2m_changed and kwargs['action'] in ('post_add', 'post_remove', 'post_clear'):
diff --git a/awx/main/tasks.py b/awx/main/tasks.py
index d8db8cc2d0..9fc963e7e3 100644
--- a/awx/main/tasks.py
+++ b/awx/main/tasks.py
@@ -805,7 +805,8 @@ class RunInventoryUpdate(BaseTask):
args.append(rax_path)
elif inventory_source.source == 'file':
args.append(inventory_source.source_path)
- args.append('-v2')
+ verbosity = getattr(settings, 'INVENTORY_UPDATE_VERBOSITY', 1)
+ args.append('-v%d' % verbosity)
if settings.DEBUG:
args.append('--traceback')
return args
diff --git a/awx/main/tests/inventory.py b/awx/main/tests/inventory.py
index 7386053956..ca29a11d88 100644
--- a/awx/main/tests/inventory.py
+++ b/awx/main/tests/inventory.py
@@ -5,6 +5,7 @@
import datetime
import json
import os
+import re
# Django
from django.conf import settings
@@ -1021,6 +1022,9 @@ class InventoryUpdatesTest(BaseTransactionTest):
source_pks = group.inventory_sources.values_list('pk', flat=True)
self.assertTrue(inventory_source.pk in source_pks)
self.assertTrue(group.has_inventory_sources)
+ # Make sure EC2 instance ID groups are excluded.
+ self.assertFalse(re.match(r'^i-[0-9a-f]{8}$', group.name, re.I),
+ group.name)
def test_update_from_ec2(self):
source_username = getattr(settings, 'TEST_AWS_ACCESS_KEY_ID', '')
diff --git a/awx/plugins/inventory/ec2.py b/awx/plugins/inventory/ec2.py
index 27d724ea53..ae8e4a1050 100755
--- a/awx/plugins/inventory/ec2.py
+++ b/awx/plugins/inventory/ec2.py
@@ -336,8 +336,9 @@ class Ec2Inventory(object):
# Add to index
self.index[dest] = [region, instance.id]
+ # For AWX: do not output group based on instance ID!!!
# Inventory: Group by instance ID (always a group of 1)
- self.inventory[instance.id] = [dest]
+ # self.inventory[instance.id] = [dest]
# Inventory: Group by region
self.push(self.inventory, region, dest)
diff --git a/awx/settings/local_settings.py.example b/awx/settings/local_settings.py.example
index 3c72570f3a..0d765844d9 100644
--- a/awx/settings/local_settings.py.example
+++ b/awx/settings/local_settings.py.example
@@ -74,9 +74,12 @@ REMOTE_HOST_HEADERS = ['REMOTE_ADDR', 'REMOTE_HOST']
# the celery task.
#AWX_TASK_ENV['FOO'] = 'BAR'
-# If set, uses -vvv for project updates instead of just -v for more output.
+# If set, use -vvv for project updates instead of -v for more output.
# PROJECT_UPDATE_VVV=True
+# Set verbosity for inventory import command when running inventory updates.
+# INVENTORY_UPDATE_VERBOSITY=1
+
###############################################################################
# EMAIL SETTINGS
###############################################################################
diff --git a/awx/ui/static/js/forms/Groups.js b/awx/ui/static/js/forms/Groups.js
index 74ff58b0e1..ebf7ee1148 100644
--- a/awx/ui/static/js/forms/Groups.js
+++ b/awx/ui/static/js/forms/Groups.js
@@ -155,19 +155,6 @@ angular.module('GroupFormDefinition', [])
"Only hosts associated with the list of regions will be included in the update process.
",
dataContainer: 'body'
},
- source_tags: {
- label: 'Tags',
- excludeModal: true,
- type: 'text',
- ngShow: "source.value == 'ec2'",
- addRequired: false,
- editRequired: false,
- dataTitle: 'Source Regions',
- dataPlacement: 'left',
- awPopOver: "Comma separated list of tags. Tag names must match those defined at the inventory source." +
- " Only hosts associated with the list of tags will be included in the update process.
",
- dataContainer: 'body'
- },
source_vars: {
label: 'Source Variables',
ngShow: "source.value == 'file' || source.value == 'ec2'",
diff --git a/awx/ui/static/js/helpers/Groups.js b/awx/ui/static/js/helpers/Groups.js
index fae4661233..cc8ba5786b 100644
--- a/awx/ui/static/js/helpers/Groups.js
+++ b/awx/ui/static/js/helpers/Groups.js
@@ -824,7 +824,6 @@ angular.module('GroupsHelper', [ 'RestServices', 'Utilities', 'ListGenerator', '
source_username: scope['source_username'],
source_password: scope['source_password'],
source_regions: scope['source_regions'],
- source_tags: scope['source_tags'],
overwrite: scope['overwrite'],
overwrite_vars: scope['overwrite_vars'],
update_on_launch: scope['update_on_launch'],