External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-05-19 14:57:39 -02:30
Code Issues Packages Projects Releases Wiki Activity

move code linting to a stricter pep8-esque auto-formatting tool, black

Browse Source
This commit is contained in:
Ryan Petrello
2021-03-19 12:44:51 -04:00
parent 9b702e46fe
commit c2ef0a6500
671 changed files with 20538 additions and 21924 deletions
Download Patch File Download Diff File Expand all files Collapse all files

331
awx/main/management/commands/inventory_import.py
View File

@@ -22,21 +22,13 @@ from django.utils.encoding import smart_text
from rest_framework.exceptions import PermissionDenied
# AWX inventory imports
from awx.main.models.inventory import (
Inventory,
InventorySource,
InventoryUpdate,
Host
)
from awx.main.models.inventory import Inventory, InventorySource, InventoryUpdate, Host
from awx.main.utils.mem_inventory import MemInventory, dict_to_mem_data
from awx.main.utils.safe_yaml import sanitize_jinja
# other AWX imports
from awx.main.models.rbac import batch_role_ancestor_rebuilding
from awx.main.utils import (
ignore_inventory_computed_fields,
get_licenser
)
from awx.main.utils import ignore_inventory_computed_fields, get_licenser
from awx.main.signals import disable_activity_stream
from awx.main.constants import STANDARD_INVENTORY_UPDATE_ENV
from awx.main.utils.pglock import advisory_lock
@@ -67,12 +59,12 @@ def functioning_dir(path):
class AnsibleInventoryLoader(object):
'''
"""
Given executable `source` (directory, executable, or file) this will
use the ansible-inventory CLI utility to convert it into in-memory
representational objects. Example:
/usr/bin/ansible/ansible-inventory -i hosts --list
'''
"""
def __init__(self, source, venv_path=None, verbosity=0):
self.source = source
@@ -88,17 +80,11 @@ class AnsibleInventoryLoader(object):
venv_exe = os.path.join(self.venv_path, 'bin', 'ansible-inventory')
if os.path.exists(venv_exe):
return venv_exe
elif os.path.exists(
os.path.join(self.venv_path, 'bin', 'ansible')
):
elif os.path.exists(os.path.join(self.venv_path, 'bin', 'ansible')):
# if bin/ansible exists but bin/ansible-inventory doesn't, it's
# probably a really old version of ansible that doesn't support
# ansible-inventory
raise RuntimeError(
"{} does not exist (please upgrade to ansible >= 2.4)".format(
venv_exe
)
)
raise RuntimeError("{} does not exist (please upgrade to ansible >= 2.4)".format(venv_exe))
return shutil.which('ansible-inventory')
def get_base_args(self):
@@ -126,8 +112,7 @@ class AnsibleInventoryLoader(object):
stderr = smart_text(stderr)
if proc.returncode != 0:
raise RuntimeError('%s failed (rc=%d) with stdout:\n%s\nstderr:\n%s' % (
'ansible-inventory', proc.returncode, stdout, stderr))
raise RuntimeError('%s failed (rc=%d) with stdout:\n%s\nstderr:\n%s' % ('ansible-inventory', proc.returncode, stdout, stderr))
for line in stderr.splitlines():
logger.error(line)
@@ -149,63 +134,78 @@ class AnsibleInventoryLoader(object):
class Command(BaseCommand):
'''
"""
Management command to import inventory from a directory, ini file, or
dynamic inventory script.
'''
"""
help = 'Import or sync external inventory sources'
def add_arguments(self, parser):
parser.add_argument('--inventory-name', dest='inventory_name',
type=str, default=None, metavar='n',
help='name of inventory to sync')
parser.add_argument('--inventory-id', dest='inventory_id', type=int,
default=None, metavar='i',
help='id of inventory to sync')
parser.add_argument('--venv', dest='venv', type=str, default=None,
help='absolute path to the AWX custom virtualenv to use')
parser.add_argument('--overwrite', dest='overwrite', action='store_true', default=False,
help='overwrite the destination hosts and groups')
parser.add_argument('--overwrite-vars', dest='overwrite_vars',
action='store_true', default=False,
help='overwrite (rather than merge) variables')
parser.add_argument('--keep-vars', dest='keep_vars', action='store_true', default=False,
help='DEPRECATED legacy option, has no effect')
parser.add_argument('--custom', dest='custom', action='store_true', default=False,
help='DEPRECATED indicates a custom inventory script, no longer used')
parser.add_argument('--source', dest='source', type=str, default=None,
metavar='s', help='inventory directory, file, or script to load')
parser.add_argument('--enabled-var', dest='enabled_var', type=str,
default=None, metavar='v', help='host variable used to '
'set/clear enabled flag when host is online/offline, may '
'be specified as "foo.bar" to traverse nested dicts.')
parser.add_argument('--enabled-value', dest='enabled_value', type=str,
default=None, metavar='v', help='value of host variable '
'specified by --enabled-var that indicates host is '
'enabled/online.')
parser.add_argument('--group-filter', dest='group_filter', type=str,
default=None, metavar='regex', help='regular expression '
'to filter group name(s); only matches are imported.')
parser.add_argument('--host-filter', dest='host_filter', type=str,
default=None, metavar='regex', help='regular expression '
'to filter host name(s); only matches are imported.')
parser.add_argument('--exclude-empty-groups', dest='exclude_empty_groups',
action='store_true', default=False, help='when set, '
'exclude all groups that have no child groups, hosts, or '
'variables.')
parser.add_argument('--instance-id-var', dest='instance_id_var', type=str,
default=None, metavar='v', help='host variable that '
'specifies the unique, immutable instance ID, may be '
'specified as "foo.bar" to traverse nested dicts.')
parser.add_argument('--inventory-name', dest='inventory_name', type=str, default=None, metavar='n', help='name of inventory to sync')
parser.add_argument('--inventory-id', dest='inventory_id', type=int, default=None, metavar='i', help='id of inventory to sync')
parser.add_argument('--venv', dest='venv', type=str, default=None, help='absolute path to the AWX custom virtualenv to use')
parser.add_argument('--overwrite', dest='overwrite', action='store_true', default=False, help='overwrite the destination hosts and groups')
parser.add_argument('--overwrite-vars', dest='overwrite_vars', action='store_true', default=False, help='overwrite (rather than merge) variables')
parser.add_argument('--keep-vars', dest='keep_vars', action='store_true', default=False, help='DEPRECATED legacy option, has no effect')
parser.add_argument(
'--custom', dest='custom', action='store_true', default=False, help='DEPRECATED indicates a custom inventory script, no longer used'
)
parser.add_argument('--source', dest='source', type=str, default=None, metavar='s', help='inventory directory, file, or script to load')
parser.add_argument(
'--enabled-var',
dest='enabled_var',
type=str,
default=None,
metavar='v',
help='host variable used to ' 'set/clear enabled flag when host is online/offline, may ' 'be specified as "foo.bar" to traverse nested dicts.',
)
parser.add_argument(
'--enabled-value',
dest='enabled_value',
type=str,
default=None,
metavar='v',
help='value of host variable ' 'specified by --enabled-var that indicates host is ' 'enabled/online.',
)
parser.add_argument(
'--group-filter',
dest='group_filter',
type=str,
default=None,
metavar='regex',
help='regular expression ' 'to filter group name(s); only matches are imported.',
)
parser.add_argument(
'--host-filter',
dest='host_filter',
type=str,
default=None,
metavar='regex',
help='regular expression ' 'to filter host name(s); only matches are imported.',
)
parser.add_argument(
'--exclude-empty-groups',
dest='exclude_empty_groups',
action='store_true',
default=False,
help='when set, ' 'exclude all groups that have no child groups, hosts, or ' 'variables.',
)
parser.add_argument(
'--instance-id-var',
dest='instance_id_var',
type=str,
default=None,
metavar='v',
help='host variable that ' 'specifies the unique, immutable instance ID, may be ' 'specified as "foo.bar" to traverse nested dicts.',
)
def set_logging_level(self, verbosity):
log_levels = dict(enumerate([logging.WARNING, logging.INFO,
logging.DEBUG, 0]))
log_levels = dict(enumerate([logging.WARNING, logging.INFO, logging.DEBUG, 0]))
logger.setLevel(log_levels.get(verbosity, 0))
def _get_instance_id(self, variables, default=''):
'''
"""
Retrieve the instance ID from the given dict of host variables.
The instance ID variable may be specified as 'foo.bar', in which case
@@ -216,7 +216,7 @@ class Command(BaseCommand):
Multiple ID variables may be specified as 'foo.bar,foobar', so that
it will first try to find 'bar' inside of 'foo', and if unable,
will try to find 'foobar' as a fallback
'''
"""
instance_id = default
if getattr(self, 'instance_id_var', None):
for single_instance_id in self.instance_id_var.split(','):
@@ -232,14 +232,14 @@ class Command(BaseCommand):
return smart_text(instance_id)
def _get_enabled(self, from_dict, default=None):
'''
"""
Retrieve the enabled state from the given dict of host variables.
The enabled variable may be specified as 'foo.bar', in which case
the lookup will traverse into nested dicts, equivalent to:
from_dict.get('foo', {}).get('bar', default)
'''
"""
enabled = default
if getattr(self, 'enabled_var', None):
default = object()
@@ -266,8 +266,7 @@ class Command(BaseCommand):
def get_source_absolute_path(source):
if not os.path.exists(source):
raise IOError('Source does not exist: %s' % source)
source = os.path.join(os.getcwd(), os.path.dirname(source),
os.path.basename(source))
source = os.path.join(os.getcwd(), os.path.dirname(source), os.path.basename(source))
source = os.path.normpath(os.path.abspath(source))
return source
@@ -284,15 +283,14 @@ class Command(BaseCommand):
self._batch_add_m2m_cache[key] = []
def _build_db_instance_id_map(self):
'''
"""
Find any hosts in the database without an instance_id set that may
still have one available via host variables.
'''
"""
self.db_instance_id_map = {}
if self.instance_id_var:
host_qs = self.inventory_source.hosts.all()
host_qs = host_qs.filter(instance_id='',
variables__contains=self.instance_id_var.split('.')[0])
host_qs = host_qs.filter(instance_id='', variables__contains=self.instance_id_var.split('.')[0])
for host in host_qs:
instance_id = self._get_instance_id(host.variables_dict)
if not instance_id:
@@ -300,38 +298,36 @@ class Command(BaseCommand):
self.db_instance_id_map[instance_id] = host.pk
def _build_mem_instance_id_map(self):
'''
"""
Update instance ID for each imported host and define a mapping of
instance IDs to MemHost instances.
'''
"""
self.mem_instance_id_map = {}
if self.instance_id_var:
for mem_host in self.all_group.all_hosts.values():
instance_id = self._get_instance_id(mem_host.variables)
if not instance_id:
logger.warning('Host "%s" has no "%s" variable(s)',
mem_host.name, self.instance_id_var)
logger.warning('Host "%s" has no "%s" variable(s)', mem_host.name, self.instance_id_var)
continue
mem_host.instance_id = instance_id
self.mem_instance_id_map[instance_id] = mem_host.name
def _existing_host_pks(self):
'''Returns cached set of existing / previous host primary key values
"""Returns cached set of existing / previous host primary key values
this is the starting set, meaning that it is pre-modification
by deletions and other things done in the course of this import
'''
"""
if not hasattr(self, '_cached_host_pk_set'):
self._cached_host_pk_set = frozenset(
self.inventory_source.hosts.values_list('pk', flat=True))
self._cached_host_pk_set = frozenset(self.inventory_source.hosts.values_list('pk', flat=True))
return self._cached_host_pk_set
def _delete_hosts(self):
'''
"""
For each host in the database that is NOT in the local list, delete
it. When importing from a cloud inventory source attached to a
specific group, only delete hosts beneath that group. Delete each
host individually so signal handlers will run.
'''
"""
if settings.SQL_DEBUG:
queries_before = len(connection.queries)
hosts_qs = self.inventory_source.hosts
@@ -341,38 +337,36 @@ class Command(BaseCommand):
all_instance_ids = list(self.mem_instance_id_map.keys())
instance_ids = []
for offset in range(0, len(all_instance_ids), self._batch_size):
instance_ids = all_instance_ids[offset:(offset + self._batch_size)]
instance_ids = all_instance_ids[offset : (offset + self._batch_size)]
for host_pk in hosts_qs.filter(instance_id__in=instance_ids).values_list('pk', flat=True):
del_host_pks.discard(host_pk)
for host_pk in set([v for k,v in self.db_instance_id_map.items() if k in instance_ids]):
for host_pk in set([v for k, v in self.db_instance_id_map.items() if k in instance_ids]):
del_host_pks.discard(host_pk)
all_host_names = list(set(self.mem_instance_id_map.values()) - set(self.all_group.all_hosts.keys()))
else:
all_host_names = list(self.all_group.all_hosts.keys())
for offset in range(0, len(all_host_names), self._batch_size):
host_names = all_host_names[offset:(offset + self._batch_size)]
host_names = all_host_names[offset : (offset + self._batch_size)]
for host_pk in hosts_qs.filter(name__in=host_names).values_list('pk', flat=True):
del_host_pks.discard(host_pk)
# Now delete all remaining hosts in batches.
all_del_pks = sorted(list(del_host_pks))
for offset in range(0, len(all_del_pks), self._batch_size):
del_pks = all_del_pks[offset:(offset + self._batch_size)]
del_pks = all_del_pks[offset : (offset + self._batch_size)]
for host in hosts_qs.filter(pk__in=del_pks):
host_name = host.name
host.delete()
logger.debug('Deleted host "%s"', host_name)
if settings.SQL_DEBUG:
logger.warning('host deletions took %d queries for %d hosts',
len(connection.queries) - queries_before,
len(all_del_pks))
logger.warning('host deletions took %d queries for %d hosts', len(connection.queries) - queries_before, len(all_del_pks))
def _delete_groups(self):
'''
"""
# If overwrite is set, for each group in the database that is NOT in
# the local list, delete it. When importing from a cloud inventory
# source attached to a specific group, only delete children of that
# group. Delete each group individually so signal handlers will run.
'''
"""
if settings.SQL_DEBUG:
queries_before = len(connection.queries)
groups_qs = self.inventory_source.groups.all()
@@ -380,30 +374,28 @@ class Command(BaseCommand):
del_group_pks = set(groups_qs.values_list('pk', flat=True))
all_group_names = list(self.all_group.all_groups.keys())
for offset in range(0, len(all_group_names), self._batch_size):
group_names = all_group_names[offset:(offset + self._batch_size)]
group_names = all_group_names[offset : (offset + self._batch_size)]
for group_pk in groups_qs.filter(name__in=group_names).values_list('pk', flat=True):
del_group_pks.discard(group_pk)
# Now delete all remaining groups in batches.
all_del_pks = sorted(list(del_group_pks))
for offset in range(0, len(all_del_pks), self._batch_size):
del_pks = all_del_pks[offset:(offset + self._batch_size)]
del_pks = all_del_pks[offset : (offset + self._batch_size)]
for group in groups_qs.filter(pk__in=del_pks):
group_name = group.name
with ignore_inventory_computed_fields():
group.delete()
logger.debug('Group "%s" deleted', group_name)
if settings.SQL_DEBUG:
logger.warning('group deletions took %d queries for %d groups',
len(connection.queries) - queries_before,
len(all_del_pks))
logger.warning('group deletions took %d queries for %d groups', len(connection.queries) - queries_before, len(all_del_pks))
def _delete_group_children_and_hosts(self):
'''
"""
Clear all invalid child relationships for groups and all invalid host
memberships. When importing from a cloud inventory source attached to
a specific group, only clear relationships for hosts and groups that
are beneath the inventory source group.
'''
"""
# FIXME: Optimize performance!
if settings.SQL_DEBUG:
queries_before = len(connection.queries)
@@ -432,12 +424,11 @@ class Command(BaseCommand):
# Removal list is complete - now perform the removals
del_child_group_pks = list(set(db_children_name_pk_map.values()))
for offset in range(0, len(del_child_group_pks), self._batch_size):
child_group_pks = del_child_group_pks[offset:(offset + self._batch_size)]
child_group_pks = del_child_group_pks[offset : (offset + self._batch_size)]
for db_child in db_children.filter(pk__in=child_group_pks):
group_group_count += 1
db_group.children.remove(db_child)
logger.debug('Group "%s" removed from group "%s"',
db_child.name, db_group.name)
logger.debug('Group "%s" removed from group "%s"', db_child.name, db_group.name)
# FIXME: Inventory source group relationships
# Delete group/host relationships not present in imported data.
db_hosts = db_group.hosts
@@ -451,37 +442,38 @@ class Command(BaseCommand):
mem_hosts = self.all_group.all_groups[db_group.name].hosts
all_mem_host_names = [h.name for h in mem_hosts if not h.instance_id]
for offset in range(0, len(all_mem_host_names), self._batch_size):
mem_host_names = all_mem_host_names[offset:(offset + self._batch_size)]
mem_host_names = all_mem_host_names[offset : (offset + self._batch_size)]
for db_host_pk in db_hosts.filter(name__in=mem_host_names).values_list('pk', flat=True):
del_host_pks.discard(db_host_pk)
all_mem_instance_ids = [h.instance_id for h in mem_hosts if h.instance_id]
for offset in range(0, len(all_mem_instance_ids), self._batch_size):
mem_instance_ids = all_mem_instance_ids[offset:(offset + self._batch_size)]
mem_instance_ids = all_mem_instance_ids[offset : (offset + self._batch_size)]
for db_host_pk in db_hosts.filter(instance_id__in=mem_instance_ids).values_list('pk', flat=True):
del_host_pks.discard(db_host_pk)
all_db_host_pks = [v for k,v in self.db_instance_id_map.items() if k in all_mem_instance_ids]
all_db_host_pks = [v for k, v in self.db_instance_id_map.items() if k in all_mem_instance_ids]
for db_host_pk in all_db_host_pks:
del_host_pks.discard(db_host_pk)
# Removal list is complete - now perform the removals
del_host_pks = list(del_host_pks)
for offset in range(0, len(del_host_pks), self._batch_size):
del_pks = del_host_pks[offset:(offset + self._batch_size)]
del_pks = del_host_pks[offset : (offset + self._batch_size)]
for db_host in db_hosts.filter(pk__in=del_pks):
group_host_count += 1
if db_host not in db_group.hosts.all():
continue
db_group.hosts.remove(db_host)
logger.debug('Host "%s" removed from group "%s"',
db_host.name, db_group.name)
logger.debug('Host "%s" removed from group "%s"', db_host.name, db_group.name)
if settings.SQL_DEBUG:
logger.warning('group-group and group-host deletions took %d queries for %d relationships',
len(connection.queries) - queries_before,
group_group_count + group_host_count)
logger.warning(
'group-group and group-host deletions took %d queries for %d relationships',
len(connection.queries) - queries_before,
group_group_count + group_host_count,
)
def _update_inventory(self):
'''
"""
Update inventory variables from "all" group.
'''
"""
# TODO: We disable variable overwrite here in case user-defined inventory variables get
# mangled. But we still need to figure out a better way of processing multiple inventory
# update variables mixing with each other.
@@ -496,24 +488,24 @@ class Command(BaseCommand):
logger.debug('Inventory variables unmodified')
def _create_update_groups(self):
'''
"""
For each group in the local list, create it if it doesn't exist in the
database. Otherwise, update/replace database variables from the
imported data. Associate with the inventory source group if importing
from cloud inventory source.
'''
"""
if settings.SQL_DEBUG:
queries_before = len(connection.queries)
all_group_names = sorted(self.all_group.all_groups.keys())
root_group_names = set()
for k,v in self.all_group.all_groups.items():
for k, v in self.all_group.all_groups.items():
if not v.parents:
root_group_names.add(k)
if len(v.parents) == 1 and v.parents[0].name == 'all':
root_group_names.add(k)
existing_group_names = set()
for offset in range(0, len(all_group_names), self._batch_size):
group_names = all_group_names[offset:(offset + self._batch_size)]
group_names = all_group_names[offset : (offset + self._batch_size)]
for group in self.inventory.groups.filter(name__in=group_names):
mem_group = self.all_group.all_groups[group.name]
db_variables = group.variables_dict
@@ -537,20 +529,14 @@ class Command(BaseCommand):
continue
mem_group = self.all_group.all_groups[group_name]
group_desc = mem_group.variables.pop('_awx_description', 'imported')
group = self.inventory.groups.update_or_create(
name=group_name,
defaults={
'variables':json.dumps(mem_group.variables),
'description':group_desc
}
)[0]
group = self.inventory.groups.update_or_create(name=group_name, defaults={'variables': json.dumps(mem_group.variables), 'description': group_desc})[
0
]
logger.debug('Group "%s" added', group.name)
self._batch_add_m2m(self.inventory_source.groups, group)
self._batch_add_m2m(self.inventory_source.groups, flush=True)
if settings.SQL_DEBUG:
logger.warning('group updates took %d queries for %d groups',
len(connection.queries) - queries_before,
len(self.all_group.all_groups))
logger.warning('group updates took %d queries for %d groups', len(connection.queries) - queries_before, len(self.all_group.all_groups))
def _update_db_host_from_mem_host(self, db_host, mem_host):
# Update host variables.
@@ -604,12 +590,12 @@ class Command(BaseCommand):
self._batch_add_m2m(self.inventory_source.hosts, db_host)
def _create_update_hosts(self):
'''
"""
For each host in the local list, create it if it doesn't exist in the
database. Otherwise, update/replace database variables from the
imported data. Associate with the inventory source group if importing
from cloud inventory source.
'''
"""
if settings.SQL_DEBUG:
queries_before = len(connection.queries)
host_pks_updated = set()
@@ -617,7 +603,7 @@ class Command(BaseCommand):
mem_host_instance_id_map = {}
mem_host_name_map = {}
mem_host_names_to_update = set(self.all_group.all_hosts.keys())
for k,v in self.all_group.all_hosts.items():
for k, v in self.all_group.all_hosts.items():
mem_host_name_map[k] = v
instance_id = self._get_instance_id(v.variables)
if instance_id in self.db_instance_id_map:
@@ -628,8 +614,8 @@ class Command(BaseCommand):
# Update all existing hosts where we know the PK based on instance_id.
all_host_pks = sorted(mem_host_pk_map.keys())
for offset in range(0, len(all_host_pks), self._batch_size):
host_pks = all_host_pks[offset:(offset + self._batch_size)]
for db_host in self.inventory.hosts.filter( pk__in=host_pks):
host_pks = all_host_pks[offset : (offset + self._batch_size)]
for db_host in self.inventory.hosts.filter(pk__in=host_pks):
if db_host.pk in host_pks_updated:
continue
mem_host = mem_host_pk_map[db_host.pk]
@@ -640,8 +626,8 @@ class Command(BaseCommand):
# Update all existing hosts where we know the instance_id.
all_instance_ids = sorted(mem_host_instance_id_map.keys())
for offset in range(0, len(all_instance_ids), self._batch_size):
instance_ids = all_instance_ids[offset:(offset + self._batch_size)]
for db_host in self.inventory.hosts.filter( instance_id__in=instance_ids):
instance_ids = all_instance_ids[offset : (offset + self._batch_size)]
for db_host in self.inventory.hosts.filter(instance_id__in=instance_ids):
if db_host.pk in host_pks_updated:
continue
mem_host = mem_host_instance_id_map[db_host.instance_id]
@@ -652,8 +638,8 @@ class Command(BaseCommand):
# Update all existing hosts by name.
all_host_names = sorted(mem_host_name_map.keys())
for offset in range(0, len(all_host_names), self._batch_size):
host_names = all_host_names[offset:(offset + self._batch_size)]
for db_host in self.inventory.hosts.filter( name__in=host_names):
host_names = all_host_names[offset : (offset + self._batch_size)]
for db_host in self.inventory.hosts.filter(name__in=host_names):
if db_host.pk in host_pks_updated:
continue
mem_host = mem_host_name_map[db_host.name]
@@ -687,27 +673,25 @@ class Command(BaseCommand):
self._batch_add_m2m(self.inventory_source.hosts, flush=True)
if settings.SQL_DEBUG:
logger.warning('host updates took %d queries for %d hosts',
len(connection.queries) - queries_before,
len(self.all_group.all_hosts))
logger.warning('host updates took %d queries for %d hosts', len(connection.queries) - queries_before, len(self.all_group.all_hosts))
@transaction.atomic
def _create_update_group_children(self):
'''
"""
For each imported group, create all parent-child group relationships.
'''
"""
if settings.SQL_DEBUG:
queries_before = len(connection.queries)
all_group_names = sorted([k for k,v in self.all_group.all_groups.items() if v.children])
all_group_names = sorted([k for k, v in self.all_group.all_groups.items() if v.children])
group_group_count = 0
for offset in range(0, len(all_group_names), self._batch_size):
group_names = all_group_names[offset:(offset + self._batch_size)]
group_names = all_group_names[offset : (offset + self._batch_size)]
for db_group in self.inventory.groups.filter(name__in=group_names):
mem_group = self.all_group.all_groups[db_group.name]
group_group_count += len(mem_group.children)
all_child_names = sorted([g.name for g in mem_group.children])
for offset2 in range(0, len(all_child_names), self._batch_size):
child_names = all_child_names[offset2:(offset2 + self._batch_size)]
child_names = all_child_names[offset2 : (offset2 + self._batch_size)]
db_children_qs = self.inventory.groups.filter(name__in=child_names)
for db_child in db_children_qs.filter(children__id=db_group.id):
logger.debug('Group "%s" already child of group "%s"', db_child.name, db_group.name)
@@ -716,8 +700,7 @@ class Command(BaseCommand):
logger.debug('Group "%s" added as child of "%s"', db_child.name, db_group.name)
self._batch_add_m2m(db_group.children, flush=True)
if settings.SQL_DEBUG:
logger.warning('Group-group updates took %d queries for %d group-group relationships',
len(connection.queries) - queries_before, group_group_count)
logger.warning('Group-group updates took %d queries for %d group-group relationships', len(connection.queries) - queries_before, group_group_count)
@transaction.atomic
def _create_update_group_hosts(self):
@@ -725,16 +708,16 @@ class Command(BaseCommand):
# belongs.
if settings.SQL_DEBUG:
queries_before = len(connection.queries)
all_group_names = sorted([k for k,v in self.all_group.all_groups.items() if v.hosts])
all_group_names = sorted([k for k, v in self.all_group.all_groups.items() if v.hosts])
group_host_count = 0
for offset in range(0, len(all_group_names), self._batch_size):
group_names = all_group_names[offset:(offset + self._batch_size)]
group_names = all_group_names[offset : (offset + self._batch_size)]
for db_group in self.inventory.groups.filter(name__in=group_names):
mem_group = self.all_group.all_groups[db_group.name]
group_host_count += len(mem_group.hosts)
all_host_names = sorted([h.name for h in mem_group.hosts if not h.instance_id])
for offset2 in range(0, len(all_host_names), self._batch_size):
host_names = all_host_names[offset2:(offset2 + self._batch_size)]
host_names = all_host_names[offset2 : (offset2 + self._batch_size)]
db_hosts_qs = self.inventory.hosts.filter(name__in=host_names)
for db_host in db_hosts_qs.filter(groups__id=db_group.id):
logger.debug('Host "%s" already in group "%s"', db_host.name, db_group.name)
@@ -743,7 +726,7 @@ class Command(BaseCommand):
logger.debug('Host "%s" added to group "%s"', db_host.name, db_group.name)
all_instance_ids = sorted([h.instance_id for h in mem_group.hosts if h.instance_id])
for offset2 in range(0, len(all_instance_ids), self._batch_size):
instance_ids = all_instance_ids[offset2:(offset2 + self._batch_size)]
instance_ids = all_instance_ids[offset2 : (offset2 + self._batch_size)]
db_hosts_qs = self.inventory.hosts.filter(instance_id__in=instance_ids)
for db_host in db_hosts_qs.filter(groups__id=db_group.id):
logger.debug('Host "%s" already in group "%s"', db_host.name, db_group.name)
@@ -752,14 +735,13 @@ class Command(BaseCommand):
logger.debug('Host "%s" added to group "%s"', db_host.name, db_group.name)
self._batch_add_m2m(db_group.hosts, flush=True)
if settings.SQL_DEBUG:
logger.warning('Group-host updates took %d queries for %d group-host relationships',
len(connection.queries) - queries_before, group_host_count)
logger.warning('Group-host updates took %d queries for %d group-host relationships', len(connection.queries) - queries_before, group_host_count)
def load_into_database(self):
'''
"""
Load inventory from in-memory groups to the database, overwriting or
merging as appropriate.
'''
"""
# FIXME: Attribute changes to superuser?
# Perform __in queries in batches (mainly for unit tests using SQLite).
self._batch_size = 500
@@ -782,9 +764,7 @@ class Command(BaseCommand):
if remote_license_type is None:
raise PermissionDenied('Unexpected Error: Tower inventory plugin missing needed metadata!')
if local_license_type != remote_license_type:
raise PermissionDenied('Tower server licenses must match: source: {} local: {}'.format(
remote_license_type, local_license_type
))
raise PermissionDenied('Tower server licenses must match: source: {} local: {}'.format(remote_license_type, local_license_type))
def check_license(self):
license_info = get_licenser().validate()
@@ -875,7 +855,6 @@ class Command(BaseCommand):
raise CommandError('Inventory with %s = %s returned multiple results' % list(q.items())[0])
logger.info('Updating inventory %d: %s' % (inventory.pk, inventory.name))
# Create ad-hoc inventory source and inventory update objects
with ignore_inventory_computed_fields():
source = Command.get_source_absolute_path(raw_source)
@@ -888,15 +867,10 @@ class Command(BaseCommand):
overwrite_vars=bool(options.get('overwrite_vars', False)),
)
inventory_update = inventory_source.create_inventory_update(
_eager_fields=dict(
job_args=json.dumps(sys.argv),
job_env=dict(os.environ.items()),
job_cwd=os.getcwd())
_eager_fields=dict(job_args=json.dumps(sys.argv), job_env=dict(os.environ.items()), job_cwd=os.getcwd())
)
data = AnsibleInventoryLoader(
source=source, venv_path=venv_path, verbosity=verbosity
).load()
data = AnsibleInventoryLoader(source=source, venv_path=venv_path, verbosity=verbosity).load()
logger.debug('Finished loading from source: %s', source)
@@ -992,12 +966,10 @@ class Command(BaseCommand):
self.inventory_update.save()
logger.info('Processing JSON output...')
inventory = MemInventory(
group_filter_re=self.group_filter_re, host_filter_re=self.host_filter_re)
inventory = MemInventory(group_filter_re=self.group_filter_re, host_filter_re=self.host_filter_re)
inventory = dict_to_mem_data(data, inventory=inventory)
logger.info('Loaded %d groups, %d hosts', len(inventory.all_group.all_groups),
len(inventory.all_group.all_hosts))
logger.info('Loaded %d groups, %d hosts', len(inventory.all_group.all_groups), len(inventory.all_group.all_hosts))
if self.exclude_empty_groups:
inventory.delete_empty_groups()
@@ -1036,8 +1008,7 @@ class Command(BaseCommand):
queries_before2 = len(connection.queries)
self.inventory.update_computed_fields()
if settings.SQL_DEBUG:
logger.warning('update computed fields took %d queries',
len(connection.queries) - queries_before2)
logger.warning('update computed fields took %d queries', len(connection.queries) - queries_before2)
# Check if the license is valid.
# If the license is not valid, a CommandError will be thrown,
@@ -1057,17 +1028,13 @@ class Command(BaseCommand):
raise e
if settings.SQL_DEBUG:
logger.warning('Inventory import completed for %s in %0.1fs',
self.inventory_source.name, time.time() - begin)
logger.warning('Inventory import completed for %s in %0.1fs', self.inventory_source.name, time.time() - begin)
else:
logger.info('Inventory import completed for %s in %0.1fs',
self.inventory_source.name, time.time() - begin)
logger.info('Inventory import completed for %s in %0.1fs', self.inventory_source.name, time.time() - begin)
# If we're in debug mode, then log the queries and time
# used to do the operation.
if settings.SQL_DEBUG:
queries_this_import = connection.queries[queries_before:]
sqltime = sum(float(x['time']) for x in queries_this_import)
logger.warning('Inventory import required %d queries '
'taking %0.3fs', len(queries_this_import),
sqltime)
logger.warning('Inventory import required %d queries ' 'taking %0.3fs', len(queries_this_import), sqltime)