From 04824c8477d052ebb7b5d772c40c23e52dc0c64e Mon Sep 17 00:00:00 2001 From: Chris Church Date: Wed, 23 Oct 2013 22:22:59 -0400 Subject: [PATCH 1/3] AC-569 Fix signal handler for updating computed fields on inventory models. --- awx/main/signals.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/awx/main/signals.py b/awx/main/signals.py index c2ac5ec58c..9bcf1bb9f3 100644 --- a/awx/main/signals.py +++ b/awx/main/signals.py @@ -41,8 +41,6 @@ def update_inventory_computed_fields(sender, **kwargs): sender_name = unicode(sender._meta.verbose_name) if kwargs['signal'] == post_save: sender_action = 'saved' - if instance.active: # No need to update for active instances. - return elif kwargs['signal'] == post_delete: sender_action = 'deleted' elif kwargs['signal'] == m2m_changed and kwargs['action'] in ('post_add', 'post_remove', 'post_clear'): From 27ad680f4d19e2a1ef50c28b037a0a0c27976856 Mon Sep 17 00:00:00 2001 From: Chris Church Date: Wed, 23 Oct 2013 22:46:04 -0400 Subject: [PATCH 2/3] AC-567 Remove source_tags from API/UI. Modify EC2 inventory to not return instance ID groups. --- awx/main/models/__init__.py | 1 + awx/main/serializers.py | 10 +++------- awx/main/tests/inventory.py | 4 ++++ awx/plugins/inventory/ec2.py | 3 ++- awx/ui/static/js/forms/Groups.js | 13 ------------- awx/ui/static/js/helpers/Groups.js | 1 - 6 files changed, 10 insertions(+), 22 deletions(-) diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index 04c9c022de..e39f7b667a 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -686,6 +686,7 @@ class InventorySource(PrimordialModel): blank=True, default='', ) + # FIXME: Remove tags field when making other migrations for credential changes! source_tags = models.CharField( max_length=1024, blank=True, diff --git a/awx/main/serializers.py b/awx/main/serializers.py index d2bd05dea7..6cc7f8fb7f 100644 --- a/awx/main/serializers.py +++ b/awx/main/serializers.py @@ -672,9 +672,9 @@ class InventorySourceSerializer(BaseSerializer): fields = ('id', 'url', 'related', 'summary_fields', 'created', 'modified', 'inventory', 'group', 'source', 'source_path', 'source_vars', 'source_username', 'source_password', - 'source_regions', 'source_tags', 'overwrite', - 'overwrite_vars', 'update_on_launch', 'update_interval', - 'last_update_failed', 'status', 'last_updated') + 'source_regions', 'overwrite', 'overwrite_vars', + 'update_on_launch', 'update_interval', 'last_update_failed', + 'status', 'last_updated') read_only_fields = ('inventory', 'group') def to_native(self, obj): @@ -764,10 +764,6 @@ class InventorySourceSerializer(BaseSerializer): # FIXME return attrs - def validate_source_tags(self, attrs, source): - # FIXME - return attrs - class InventoryUpdateSerializer(BaseSerializer): class Meta: diff --git a/awx/main/tests/inventory.py b/awx/main/tests/inventory.py index 7386053956..ca29a11d88 100644 --- a/awx/main/tests/inventory.py +++ b/awx/main/tests/inventory.py @@ -5,6 +5,7 @@ import datetime import json import os +import re # Django from django.conf import settings @@ -1021,6 +1022,9 @@ class InventoryUpdatesTest(BaseTransactionTest): source_pks = group.inventory_sources.values_list('pk', flat=True) self.assertTrue(inventory_source.pk in source_pks) self.assertTrue(group.has_inventory_sources) + # Make sure EC2 instance ID groups are excluded. + self.assertFalse(re.match(r'^i-[0-9a-f]{8}$', group.name, re.I), + group.name) def test_update_from_ec2(self): source_username = getattr(settings, 'TEST_AWS_ACCESS_KEY_ID', '') diff --git a/awx/plugins/inventory/ec2.py b/awx/plugins/inventory/ec2.py index 27d724ea53..ae8e4a1050 100755 --- a/awx/plugins/inventory/ec2.py +++ b/awx/plugins/inventory/ec2.py @@ -336,8 +336,9 @@ class Ec2Inventory(object): # Add to index self.index[dest] = [region, instance.id] + # For AWX: do not output group based on instance ID!!! # Inventory: Group by instance ID (always a group of 1) - self.inventory[instance.id] = [dest] + # self.inventory[instance.id] = [dest] # Inventory: Group by region self.push(self.inventory, region, dest) diff --git a/awx/ui/static/js/forms/Groups.js b/awx/ui/static/js/forms/Groups.js index 74ff58b0e1..ebf7ee1148 100644 --- a/awx/ui/static/js/forms/Groups.js +++ b/awx/ui/static/js/forms/Groups.js @@ -155,19 +155,6 @@ angular.module('GroupFormDefinition', []) "Only hosts associated with the list of regions will be included in the update process.

", dataContainer: 'body' }, - source_tags: { - label: 'Tags', - excludeModal: true, - type: 'text', - ngShow: "source.value == 'ec2'", - addRequired: false, - editRequired: false, - dataTitle: 'Source Regions', - dataPlacement: 'left', - awPopOver: "

Comma separated list of tags. Tag names must match those defined at the inventory source." + - " Only hosts associated with the list of tags will be included in the update process.

", - dataContainer: 'body' - }, source_vars: { label: 'Source Variables', ngShow: "source.value == 'file' || source.value == 'ec2'", diff --git a/awx/ui/static/js/helpers/Groups.js b/awx/ui/static/js/helpers/Groups.js index 1215707640..3da93077f4 100644 --- a/awx/ui/static/js/helpers/Groups.js +++ b/awx/ui/static/js/helpers/Groups.js @@ -810,7 +810,6 @@ angular.module('GroupsHelper', [ 'RestServices', 'Utilities', 'ListGenerator', ' source_username: scope['source_username'], source_password: scope['source_password'], source_regions: scope['source_regions'], - source_tags: scope['source_tags'], overwrite: scope['overwrite'], overwrite_vars: scope['overwrite_vars'], update_on_launch: scope['update_on_launch'], From 710d55d82d44a2aa8282148219cad47ec2c999f5 Mon Sep 17 00:00:00 2001 From: Chris Church Date: Thu, 24 Oct 2013 02:09:49 -0400 Subject: [PATCH 3/3] AC-575 Updated logging/output for inventory_import command. --- .../management/commands/inventory_import.py | 602 ++++++++++-------- awx/main/tasks.py | 3 +- awx/settings/local_settings.py.example | 5 +- 3 files changed, 345 insertions(+), 265 deletions(-) diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index 9ce47a268b..dd25ffc18b 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -10,6 +10,7 @@ import os import shlex import subprocess import sys +import time import traceback # PyYAML @@ -26,191 +27,214 @@ from awx.main.licenses import LicenseReader logger = logging.getLogger('awx.main.commands.inventory_import') -class ImportException(BaseException): +LICENSE_MESSAGE = '''\ +Number of licensed instances exceeded, would bring available instances to %(new_count)d, system is licensed for %(available_instances)d. +See http://ansibleworks.com/ansibleworks-awx for license extension information.''' - def __init__(self, msg): - self.msg = msg +DEMO_LICENSE_MESSAGE = '''\ +Demo mode free license count exceeded, would bring available instances to %(new_count)d, demo mode allows %(available_instances)d. +See http://ansibleworks.com/ansibleworks-awx for licensing information.''' - def __str__(self): - return "Import Error: %s" % msg - -class MemGroup(object): - - def __init__(self, name, inventory_base): - - assert inventory_base is not None - self.inventory_base = inventory_base +class MemObject(object): + ''' + Common code shared between in-memory groups and hosts. + ''' + + def __init__(self, name, source_dir): + assert name + assert source_dir self.name = name - self.child_groups = [] + self.source_dir = source_dir + + def load_vars(self, path): + if os.path.exists(path) and os.path.isfile(path): + vars_name = os.path.basename(os.path.dirname(path)) + logger.debug('Loading %s from %s', vars_name, path) + try: + v = yaml.safe_load(file(path, 'r').read()) + return v if hasattr(v, 'items') else {} + except yaml.YAMLError, e: + if hasattr(e, 'problem_mark'): + logger.error('Invalid YAML in %s:%s col %s', path, + e.problem_mark.line + 1, + e.problem_mark.column + 1) + else: + logger.error('Error loading YAML from %s', path) + raise + return {} + + +class MemGroup(MemObject): + ''' + In-memory representation of an inventory group. + ''' + + def __init__(self, name, source_dir): + super(MemGroup, self).__init__(name, source_dir) + self.children = [] self.hosts = [] self.variables = {} self.parents = [] # Used on the "all" group in place of previous global variables. # maps host and group names to hosts to prevent redudant additions - self.host_names = {} - self.group_names = {} + self.all_hosts = {} + self.all_groups = {} - group_vars = os.path.join(inventory_base, 'group_vars', name) - if os.path.exists(group_vars): - logger.debug("loading group_vars") - self.variables = yaml.load(open(group_vars).read()) - - def child_group_by_name(self, grp_name, loader): - logger.debug("looking for child group: %s" % grp_name) - if grp_name == 'all': + group_vars = os.path.join(self.source_dir, 'group_vars', self.name) + self.variables = self.load_vars(group_vars) + logger.debug('Loaded group: %s', self.name) + + def child_group_by_name(self, name, loader): + if name == 'all': return + logger.debug('Looking for %s as child group of %s', name, self.name) # slight hack here, passing in 'self' for all_group but child=True won't use it - grp = loader.get_group(grp_name, self, child=True) + group = loader.get_group(name, self, child=True) # don't add to child groups if already there - for x in self.child_groups: - if x.name == grp_name: - return x - logger.debug("adding child group %s to group %s" % (grp.name, self.name)) - self.child_groups.append(grp) - return grp + for g in self.children: + if g.name == name: + return g + logger.debug('Adding child group %s to group %s', group.name, self.name) + self.children.append(group) + return group - def add_child_group(self, grp): - assert grp.name is not 'all' - - logger.debug("adding child group %s to group %s" % (grp.name, self.name)) - - assert type(grp) == MemGroup - if grp not in self.child_groups: - self.child_groups.append(grp) - if not self in grp.parents: - grp.parents.append(self) + def add_child_group(self, group): + assert group.name is not 'all' + assert isinstance(group, MemGroup) + logger.debug('Adding child group %s to parent %s', group.name, self.name) + if group not in self.children: + self.children.append(group) + if not self in group.parents: + group.parents.append(self) def add_host(self, host): - logger.debug("adding host %s to group %s" % (host.name, self.name)) - - assert type(host) == MemHost + assert isinstance(host, MemHost) + logger.debug('Adding host %s to group %s', host.name, self.name) if host not in self.hosts: self.hosts.append(host) - def debug_tree(self): - logger.debug("describing tree of group (%s)" % self.name) - - logger.debug("group: %s, %s" % (self.name, self.variables)) - for x in self.child_groups: - logger.debug(" child: %s" % (x.name)) - for x in self.hosts: - logger.debug(" host: %s, %s" % (x.name, x.variables)) + def debug_tree(self, group_names=None): + group_names = group_names or set() + if self.name in group_names: + return + logger.debug('Dumping tree for group "%s":', self.name) + logger.debug('- Vars: %r', self.variables) + for h in self.hosts: + logger.debug('- Host: %s, %r', h.name, h.variables) + for g in self.children: + logger.debug('- Child: %s', g.name) + logger.debug('----') + group_names.add(self.name) + for g in self.children: + g.debug_tree(group_names) - logger.debug("---") - for x in self.child_groups: - x.debug_tree() -class MemHost(object): +class MemHost(MemObject): + ''' + In-memory representation of an inventory host. + ''' - def __init__(self, name, inventory_base): - logger.debug("adding host name: %s" % name) - assert name is not None - assert inventory_base is not None - - # set ansible_ssh_port if ":" in name - self.name = name + def __init__(self, name, source_dir): + super(MemHost, self).__init__(name, source_dir) self.variables = {} - self.inventory_base = inventory_base if ':' in name: tokens = name.split(":") self.name = tokens[0] self.variables['ansible_ssh_port'] = int(tokens[1]) - if "[" in name: - raise ImportException("block ranges like host[0:50].example.com are not yet supported by the importer") + if '[' in name: + raise ValueError('Block ranges like host[0:50].example.com are not yet supported by the importer') + + host_vars = os.path.join(source_dir, 'host_vars', name) + self.variables.update(self.load_vars(host_vars)) + logger.debug('Loaded host: %s', self.name) + - host_vars = os.path.join(inventory_base, 'host_vars', name) - if os.path.exists(host_vars): - logger.debug("loading host_vars") - self.variables.update(yaml.load(open(host_vars).read())) - class BaseLoader(object): + ''' + Common functions for an inventory loader from a given source. + ''' - def __init__(self, inventory_base=None, all_group=None): - self.inventory_base = inventory_base - self.all_group = all_group + def __init__(self, source, all_group=None): + self.source = source + self.source_dir = os.path.dirname(self.source) + self.all_group = all_group or MemGroup('all', self.source_dir) def get_host(self, name): + ''' + Return a MemHost instance from host name, creating if needed. + ''' host_name = name.split(':')[0] host = None - if not host_name in self.all_group.host_names: - host = MemHost(name, self.inventory_base) - self.all_group.host_names[host_name] = host - return self.all_group.host_names[host_name] + if not host_name in self.all_group.all_hosts: + host = MemHost(name, self.source_dir) + self.all_group.all_hosts[host_name] = host + return self.all_group.all_hosts[host_name] def get_group(self, name, all_group=None, child=False): + ''' + Return a MemGroup instance from group name, creating if needed. + ''' all_group = all_group or self.all_group if name == 'all': return all_group - if not name in self.all_group.group_names: - group = MemGroup(name, self.inventory_base) + if not name in self.all_group.all_groups: + group = MemGroup(name, self.source_dir) if not child: all_group.add_child_group(group) - self.all_group.group_names[name] = group - return self.all_group.group_names[name] + self.all_group.all_groups[name] = group + return self.all_group.all_groups[name] - def load(self, src): + def load(self): raise NotImplementedError + class IniLoader(BaseLoader): + ''' + Loader to read inventory from an INI-formatted text file. + ''' - def __init__(self, inventory_base=None, all_group=None): - super(IniLoader, self).__init__(inventory_base, all_group) - logger.debug("processing ini") - - def load(self, src): - logger.debug("loading: %s on %s" % (src, self.all_group)) - - if self.inventory_base is None: - self.inventory_base = os.path.dirname(src) - - data = open(src).read() - lines = data.split("\n") + def load(self): + logger.info('Reading INI source: %s', self.source) group = self.all_group input_mode = 'host' - - for line in lines: + for line in file(self.source, 'r'): line = line.split('#')[0].strip() if not line: continue - elif line.startswith("["): - # mode change, possible new group name - line = line.replace("[","").replace("]","").lstrip().rstrip() - if line.find(":vars") != -1: + elif line.startswith('[') and line.endswith(']'): + # Mode change, possible new group name + line = line[1:-1].strip() + if line.endswith(':vars'): input_mode = 'vars' - line = line.replace(":vars","") - group = self.get_group(line) - elif line.find(":children") != -1: - input_mode = 'children' - line = line.replace(":children","") - group = self.get_group(line) + line = line[:-5] + elif line.endswith(':children'): + input_mode = 'children' + line = line[:-9] else: input_mode = 'host' - group = self.get_group(line) + group = self.get_group(line) else: - # add a host or variable to the existing group/host + # Add a host or variable to the existing group/host tokens = shlex.split(line) - if input_mode == 'host': - new_host = self.get_host(tokens[0]) + host = self.get_host(tokens[0]) if len(tokens) > 1: - variables = {} for t in tokens[1:]: - (k,v) = t.split("=",1) - new_host.variables[k] = v - group.add_host(new_host) - + k,v = t.split('=', 1) + host.variables[k] = v + group.add_host(host) elif input_mode == 'children': group.child_group_by_name(line, self) elif input_mode == 'vars': for t in tokens: - (k, v) = t.split("=", 1) + k, v = t.split('=', 1) group.variables[k] = v - # TODO: expansion patterns are probably not going to be supported + # from API documentation: # # if called with --list, inventory outputs like so: @@ -236,12 +260,8 @@ class IniLoader(BaseLoader): # # if called with --host outputs JSON for that host -class ExecutableJsonLoader(BaseLoader): - def __init__(self, inventory_base=None, all_group=None): - super(ExecutableJsonLoader, self).__init__(inventory_base, all_group) - logger.debug("processing executable JSON source") - self.child_group_names = {} +class ExecutableJsonLoader(BaseLoader): def command_to_json(self, cmd): data = {} @@ -250,101 +270,117 @@ class ExecutableJsonLoader(BaseLoader): proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = proc.communicate() if proc.returncode != 0: - raise Exception("%s list failed %s with output: %s" % (cmd, stderr, proc.returncode)) + raise RuntimeError('%r failed (rc=%d) with output: %s' % (cmd, proc.returncode, stderr)) data = json.loads(stdout) + if not isinstance(data, dict): + raise TypeError('Returned JSON must be a dictionary, got %s instead' % str(type(data))) except: - traceback.print_exc() - raise Exception("failed to load JSON output: %s" % stdout) - assert type(data) == dict + logger.error('Failed to load JSON from: %s', stdout) + raise return data - def load(self, src): - - logger.debug("loading %s onto %s" % (src, self.all_group)) - - if self.inventory_base is None: - self.inventory_base = os.path.dirname(src) - - data = self.command_to_json([src, "--list"]) - - group = None + def load(self): + logger.info('Reading executable JSON source: %s', self.source) + data = self.command_to_json([self.source, '--list']) _meta = data.pop('_meta', {}) - for (k,v) in data.iteritems(): - + for k,v in data.iteritems(): group = self.get_group(k) - if type(v) == dict: - - # process hosts - host_details = v.get('hosts', None) - if host_details is not None: - if type(host_details) == dict: - for (hk, hv) in host_details.iteritems(): - host = self.get_host(hk) - host.variables.update(hv) - group.add_host(host) - if type(host_details) == list: - for hk in host_details: - host = self.get_host(hk) - group.add_host(host) - - # process variables - vars = v.get('vars', None) - if vars is not None: + # Load group hosts/vars/children from a dictionary. + if isinstance(v, dict): + # Process hosts within a group. + hosts = v.get('hosts', {}) + if isinstance(hosts, dict): + for hk, hv in hosts.iteritems(): + host = self.get_host(hk) + if isinstance(hv, dict): + host.variables.update(hv) + else: + self.logger.warning('Expected dict of vars for ' + 'host "%s", got %s instead', + hk, str(type(hv))) + group.add_host(host) + elif isinstance(hosts, (list, tuple)): + for hk in hosts: + host = self.get_host(hk) + group.add_host(host) + else: + logger.warning('Expected dict or list of "hosts" for ' + 'group "%s", got %s instead', k, + str(type(hosts))) + # Process group variables. + vars = v.get('vars', {}) + if isinstance(vars, dict): group.variables.update(vars) - - # process child groups - children_details = v.get('children', None) - if children_details is not None: - for x in children_details: - child = self.get_group(x, self.inventory_base, child=True) + else: + self.logger.warning('Expected dict of vars for ' + 'group "%s", got %s instead', + k, str(type(vars))) + # Process child groups. + children = v.get('children', []) + if isinstance(children, (list, tuple)): + for c in children: + child = self.get_group(c, self.all_group, child=True) group.add_child_group(child) - self.child_group_names[x] = child + else: + self.logger.warning('Expected list of children for ' + 'group "%s", got %s instead', + k, str(type(children))) - if type(v) in (tuple, list): - for x in v: - host = self.get_host(x) - group.add_host(host) + # Load host names from a list. + elif isinstance(v, (list, tuple)): + for h in v: + host = self.get_host(h) + group.add_host(host) + else: + logger.warning('') + self.logger.warning('Expected dict or list for group "%s", ' + 'got %s instead', k, str(type(v))) if k != 'all': self.all_group.add_child_group(group) - - # then we invoke the executable once for each host name we've built up + # Invoke the executable once for each host name we've built up # to set their variables - for (k,v) in self.all_group.host_names.iteritems(): + for k,v in self.all_group.all_hosts.iteritems(): if 'hostvars' not in _meta: - data = self.command_to_json([src, "--host", k]) + data = self.command_to_json([self.source, '--host', k]) else: data = _meta['hostvars'].get(k, {}) - v.variables.update(data) + if isinstance(data, dict): + v.variables.update(data) + else: + self.logger.warning('Expected dict of vars for ' + 'host "%s", got %s instead', + k, str(type(data))) -def load_generic(src): - logger.debug("analyzing type of source") - if not os.path.exists(src): - logger.debug("source missing") - raise CommandError("source does not exist") - if os.path.isdir(src): - all_group = MemGroup('all', src) - for f in glob.glob("%s/*" % src): - if f.endswith(".ini"): - # config files for inventory scripts should be ignored - continue - if not os.path.isdir(f): - if os.access(f, os.X_OK): - ExecutableJsonLoader(None, all_group).load(f) - else: - IniLoader(None, all_group).load(f) - elif os.access(src, os.X_OK): - all_group = MemGroup('all', os.path.dirname(src)) - ExecutableJsonLoader(None, all_group).load(src) +def load_inventory_source(source, all_group=None): + ''' + Load inventory from given source directory or file. + ''' + logger.debug('Analyzing type of source: %s', source) + original_all_group = all_group + if not os.path.exists(source): + raise CommandError('Source does not exist: %s' % source) + if os.path.isdir(source): + all_group = all_group or MemGroup('all', source) + for filename in glob.glob(os.path.join(source, '*')): + if filename.endswith(".ini") or os.path.isdir(filename): + continue + load_inventory_source(filename, all_group) else: - all_group = MemGroup('all', os.path.dirname(src)) - IniLoader(None, all_group).load(src) + all_group = all_group or MemGroup('all', os.path.dirname(source)) + if os.access(source, os.X_OK): + ExecutableJsonLoader(source, all_group).load() + else: + IniLoader(source, all_group).load() - logger.debug("loading process complete") + logger.debug('Finished loading from source: %s', source) + if original_all_group is None: + logger.info('Loaded %d groups, %d hosts', len(all_group.all_groups), + len(all_group.all_hosts)) return all_group @@ -382,7 +418,8 @@ class Command(NoArgsCommand): self.logger = logging.getLogger('awx.main.commands.inventory_import') self.logger.setLevel(log_levels.get(self.verbosity, 0)) handler = logging.StreamHandler() - handler.setFormatter(logging.Formatter('%(message)s')) + formatter = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s') + handler.setFormatter(formatter) self.logger.addHandler(handler) self.logger.propagate = False @@ -446,30 +483,32 @@ class Command(NoArgsCommand): # source attached to a specific group, only delete hosts beneath that # group. Delete each host individually so signal handlers will run. if self.overwrite: - self.logger.debug('deleting any hosts not in the remote source') if self.inventory_source.group: del_hosts = self.inventory_source.group.all_hosts # FIXME: Also include hosts from inventory_source.managed_hosts? else: del_hosts = self.inventory.hosts.all() - del_hosts = del_hosts.exclude(name__in=self.all_group.host_names.keys()) + del_hosts = del_hosts.exclude(name__in=self.all_group.all_hosts.keys()) for host in del_hosts: + host_name = host.name host.delete() + self.logger.info('Deleted host "%s"', host_name) # If overwrite is set, for each group in the database that is NOT in # the local list, delete it. When importing from a cloud inventory # source attached to a specific group, only delete children of that # group. Delete each group individually so signal handlers will run. if self.overwrite: - self.logger.debug('deleting any groups not in the remote source') if self.inventory_source.group: del_groups = self.inventory_source.group.all_children # FIXME: Also include groups from inventory_source.managed_groups? else: del_groups = self.inventory.groups.all() - del_groups = del_groups.exclude(name__in=self.all_group.group_names.keys()) + del_groups = del_groups.exclude(name__in=self.all_group.all_groups.keys()) for group in del_groups: + group_name = group.name group.delete() + self.logger.info('Deleted group "%s"', group_name) # If overwrite is set, clear all invalid child relationships for groups # and all invalid host memberships. When importing from a cloud @@ -477,47 +516,54 @@ class Command(NoArgsCommand): # relationships for hosts and groups that are beneath the inventory # source group. if self.overwrite: - self.logger.info("clearing any child relationships to rebuild from remote source") if self.inventory_source.group: db_groups = self.inventory_source.group.all_children else: db_groups = self.inventory.groups.all() - for db_group in db_groups: - db_kids = db_group.children.all() - mem_kids = self.all_group.group_names[db_group.name].child_groups - mem_kid_names = [ k.name for k in mem_kids ] - for db_kid in db_kids: - if db_kid.name not in mem_kid_names: - self.logger.debug("removing non-DB kid: %s" % (db_kid.name)) - db_group.children.remove(db_kid) - + db_children = db_group.children.all() + mem_children = self.all_group.all_groups[db_group.name].children + mem_children_names = [g.name for g in mem_children] + for db_child in db_children.exclude(name__in=mem_children_names): + if db_child not in db_group.children.all(): + continue + db_group.children.remove(db_child) + self.logger.info('Removed group "%s" from group "%s"', + db_child.name, db_group.name) db_hosts = db_group.hosts.all() - mem_hosts = self.all_group.group_names[db_group.name].hosts - mem_host_names = [ h.name for h in mem_hosts ] - for db_host in db_hosts: - if db_host.name not in mem_host_names: - self.logger.debug("removing non-DB host: %s" % (db_host.name)) - db_group.hosts.remove(db_host) + mem_hosts = self.all_group.all_groups[db_group.name].hosts + mem_host_names = [h.name for h in mem_hosts] + for db_host in db_hosts.exclude(name__in=mem_host_names): + if db_host not in db_group.hosts.all(): + continue + db_group.hosts.remove(db_host) + self.logger.info('Removed host "%s" from group "%s"', + db_host.name, db_group.name) # Update/overwrite variables from "all" group. If importing from a # cloud source attached to a specific group, variables will be set on - # the base group, otherwise they will be set on the inventory. + # the base group, otherwise they will be set on the whole inventory. if self.inventory_source.group: all_obj = self.inventory_source.group all_obj.inventory_sources.add(self.inventory_source) + all_name = 'group "%s"' % all_obj.name else: all_obj = self.inventory + all_name = 'inventory' db_variables = all_obj.variables_dict - mem_variables = self.all_group.variables if self.overwrite_vars or self.overwrite: - self.logger.info('replacing inventory variables from "all" group') - db_variables = mem_variables + db_variables = self.all_group.variables else: - self.logger.info('updating inventory variables from "all" group') - db_variables.update(mem_variables) - all_obj.variables = json.dumps(db_variables) - all_obj.save(update_fields=['variables']) + db_variables.update(self.all_group.variables) + if db_variables != all_obj.variables_dict: + all_obj.variables = json.dumps(db_variables) + all_obj.save(update_fields=['variables']) + if self.overwrite_vars or self.overwrite: + self.logger.info('Replaced %s variables from "all" group', all_name) + else: + self.logger.info('Updated %s variables from "all" group', all_name) + else: + self.logger.info('%s variables unmodified', all_name.capitalize()) # FIXME: Attribute changes to superuser? @@ -525,23 +571,28 @@ class Command(NoArgsCommand): # the database. Otherwise, update/replace database variables from the # imported data. Associate with the inventory source group if # importing from cloud inventory source. - for k,v in self.all_group.group_names.iteritems(): + for k,v in self.all_group.all_groups.iteritems(): variables = json.dumps(v.variables) defaults = dict(variables=variables, description='imported') group, created = self.inventory.groups.get_or_create(name=k, defaults=defaults) if created: - self.logger.info('inserting new group %s' % k) + self.logger.info('Added new group "%s"', k) else: - self.logger.info('updating existing group %s' % k) db_variables = group.variables_dict - mem_variables = v.variables if self.overwrite_vars or self.overwrite: - db_variables = mem_variables + db_variables = v.variables else: - db_variables.update(mem_variables) - group.variables = json.dumps(db_variables) - group.save(update_fields=['variables']) + db_variables.update(v.variables) + if db_variables != group.variables_dict: + group.variables = json.dumps(db_variables) + group.save(update_fields=['variables']) + if self.overwrite_vars or self.overwrite: + self.logger.info('Replaced variables for group "%s"', k) + else: + self.logger.info('Updated variables for group "%s"', k) + else: + self.logger.info('Variables unmodified for group "%s"', k) if self.inventory_source.group: self.inventory_source.group.children.add(group) group.inventory_sources.add(self.inventory_source) @@ -550,44 +601,59 @@ class Command(NoArgsCommand): # the database. Otherwise, update/replace database variables from the # imported data. Associate with the inventory source group if # importing from cloud inventory source. - for k,v in self.all_group.host_names.iteritems(): + for k,v in self.all_group.all_hosts.iteritems(): variables = json.dumps(v.variables) defaults = dict(variables=variables, description='imported') host, created = self.inventory.hosts.get_or_create(name=k, defaults=defaults) if created: - self.logger.info('inserting new host %s' % k) + self.logger.info('Added new host "%s"', k) else: - self.logger.info('updating existing host %s' % k) db_variables = host.variables_dict - mem_variables = v.variables if self.overwrite_vars or self.overwrite: - db_variables = mem_variables + db_variables = v.variables else: - db_variables.update(mem_variables) - host.variables = json.dumps(db_variables) - host.save(update_fields=['variables']) + db_variables.update(v.variables) + if db_variables != host.variables_dict: + host.variables = json.dumps(db_variables) + host.save(update_fields=['variables']) + if self.overwrite_vars or self.overwrite: + self.logger.info('Replaced variables for host "%s"', k) + else: + self.logger.info('Updated variables for host "%s"', k) + else: + self.logger.info('Variables unmodified for host "%s"', k) if self.inventory_source.group: self.inventory_source.group.hosts.add(host) host.inventory_sources.add(self.inventory_source) host.update_computed_fields(False, False) - # for each host in a mem group, add it to the parents to which it belongs - for (k,v) in self.all_group.group_names.iteritems(): - self.logger.info("adding parent arrangements for %s" % k) - db_group = Group.objects.get(name=k, inventory__pk=self.inventory.pk) - mem_hosts = v.hosts - for h in mem_hosts: - db_host = Host.objects.get(name=h.name, inventory__pk=self.inventory.pk) - db_group.hosts.add(db_host) - self.logger.debug("*** ADDING %s to %s ***" % (db_host, db_group)) + # For each host in a mem group, add it to the parent(s) to which it + # belongs. + for k,v in self.all_group.all_groups.iteritems(): + if not v.hosts: + continue + db_group = self.inventory.groups.get(name=k) + for h in v.hosts: + db_host = self.inventory.hosts.get(name=h.name) + if db_host not in db_group.hosts.all(): + db_group.hosts.add(db_host) + self.logger.info('Added host "%s" to group "%s"', h.name, k) + else: + self.logger.info('Host "%s" already in group "%s"', h.name, k) # for each group, draw in child group arrangements - for (k,v) in self.all_group.group_names.iteritems(): - db_group = Group.objects.get(inventory=self.inventory, name=k) - for mem_child_group in v.child_groups: - db_child = Group.objects.get(inventory=self.inventory, name=mem_child_group.name) - db_group.children.add(db_child) + for k,v in self.all_group.all_groups.iteritems(): + if not v.children: + continue + db_group = self.inventory.groups.get(name=k) + for g in v.children: + db_child = self.inventory.groups.get(name=g.name) + if db_child not in db_group.hosts.all(): + db_group.children.add(db_child) + self.logger.info('Added group "%s" as child of "%s"', g.name, k) + else: + self.logger.info('Group "%s" already child of group "%s"', g.name, k) def check_license(self): reader = LicenseReader() @@ -596,10 +662,15 @@ class Command(NoArgsCommand): free_instances = license_info.get('free_instances', 0) new_count = Host.objects.filter(active=True).count() if free_instances < 0: + d = { + 'new_count': new_count, + 'available_instances': available_instances, + } if license_info.get('demo', False): - raise CommandError("demo mode free license count exceeded, would bring available instances to %s, demo mode allows %s, see http://ansibleworks.com/ansibleworks-awx for licensing information" % (new_count, available_instances)) + self.logger.error(DEMO_LICENSE_MESSAGE % d) else: - raise CommandError("number of licensed instances exceeded, would bring available instances to %s, system is licensed for %s, see http://ansibleworks.com/ansibleworks-awx for license extension information" % (new_count, available_instances)) + self.logger.error(LICENSE_MESSAGE % d) + raise CommandError('License count exceeded!') @transaction.commit_on_success def handle_noargs(self, **options): @@ -622,6 +693,7 @@ class Command(NoArgsCommand): if not self.source: raise CommandError('--source is required') + begin = time.time() self.load_inventory_from_database() status, tb, exc = 'error', '', None @@ -632,24 +704,28 @@ class Command(NoArgsCommand): self.inventory_update.save() transaction.commit() - self.logger.debug('preparing to load from %s' % self.source) - self.all_group = load_generic(self.source) - self.logger.debug('debugging loaded result:') + # Load inventory from source. + self.all_group = load_inventory_source(self.source) self.all_group.debug_tree() - # now that memGroup is correct and supports JSON executables, INI, and trees - # now merge and/or overwrite with the database itself! - + # Merge/overwrite inventory into database. self.load_into_database() self.check_license() - self.logger.info("inventory import complete, %s, id=%s" % \ - (self.inventory.name, self.inventory.id)) + if self.inventory_source.group: + inv_name = 'group "%s"' % (self.inventory_source.group.name) + else: + inv_name = '"%s" (id=%s)' % (self.inventory.name, + self.inventory.id) + self.logger.info('Inventory import completed for %s in %0.1fs', + inv_name, time.time() - begin) status = 'successful' except Exception, e: if isinstance(e, KeyboardInterrupt): status = 'canceled' exc = e + elif isinstance(e, CommandError): + exc = e else: tb = traceback.format_exc() exc = e diff --git a/awx/main/tasks.py b/awx/main/tasks.py index d8db8cc2d0..9fc963e7e3 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -805,7 +805,8 @@ class RunInventoryUpdate(BaseTask): args.append(rax_path) elif inventory_source.source == 'file': args.append(inventory_source.source_path) - args.append('-v2') + verbosity = getattr(settings, 'INVENTORY_UPDATE_VERBOSITY', 1) + args.append('-v%d' % verbosity) if settings.DEBUG: args.append('--traceback') return args diff --git a/awx/settings/local_settings.py.example b/awx/settings/local_settings.py.example index 3c72570f3a..0d765844d9 100644 --- a/awx/settings/local_settings.py.example +++ b/awx/settings/local_settings.py.example @@ -74,9 +74,12 @@ REMOTE_HOST_HEADERS = ['REMOTE_ADDR', 'REMOTE_HOST'] # the celery task. #AWX_TASK_ENV['FOO'] = 'BAR' -# If set, uses -vvv for project updates instead of just -v for more output. +# If set, use -vvv for project updates instead of -v for more output. # PROJECT_UPDATE_VVV=True +# Set verbosity for inventory import command when running inventory updates. +# INVENTORY_UPDATE_VERBOSITY=1 + ############################################################################### # EMAIL SETTINGS ###############################################################################