From 8c9a98befc6be8e95801971aa8036d0de0d41bf8 Mon Sep 17 00:00:00 2001 From: Chris Church Date: Sat, 22 Feb 2014 16:23:11 -0500 Subject: [PATCH] AC-406 Added support for importing inventory host patterns/ranges from INI files. --- .../management/commands/inventory_import.py | 66 ++++++++-- awx/main/tests/commands.py | 122 +++++++++++++++++- 2 files changed, 172 insertions(+), 16 deletions(-) diff --git a/awx/main/management/commands/inventory_import.py b/awx/main/management/commands/inventory_import.py index 6669d9ac1f..31df67c6df 100644 --- a/awx/main/management/commands/inventory_import.py +++ b/awx/main/management/commands/inventory_import.py @@ -7,7 +7,9 @@ import json import logging from optparse import make_option import os +import re import shlex +import string import subprocess import sys import time @@ -140,15 +142,10 @@ class MemHost(MemObject): def __init__(self, name, source_dir): super(MemHost, self).__init__(name, source_dir) self.variables = {} - if ':' in name: - tokens = name.split(":") + tokens = name.split(':') self.name = tokens[0] self.variables['ansible_ssh_port'] = int(tokens[1]) - - if '[' in name: - raise ValueError('Block ranges like host[0:50].example.com are not yet supported by the importer') - host_vars = os.path.join(source_dir, 'host_vars', name) self.variables.update(self.load_vars(host_vars)) logger.debug('Loaded host: %s', self.name) @@ -168,6 +165,8 @@ class BaseLoader(object): ''' Return a MemHost instance from host name, creating if needed. ''' + if '[' in name or ']' in name: + raise ValueError('host ranges like %s are not supported by this importer' % name) host_name = name.split(':')[0] host = None if not host_name in self.all_group.all_hosts: @@ -175,6 +174,44 @@ class BaseLoader(object): self.all_group.all_hosts[host_name] = host return self.all_group.all_hosts[host_name] + def get_hosts(self, name): + ''' + Return iterator over one or more MemHost instances from host name or + host pattern. + ''' + def iternest(*args): + if args: + for i in args[0]: + for j in iternest(*args[1:]): + yield ''.join([str(i), j]) + else: + yield '' + pattern_re = re.compile(r'(\[(?:(?:\d+\:\d+)|(?:[A-Za-z]\:[A-Za-z]))(?:\:\d+)??\])') + iters = [] + for s in re.split(pattern_re, name): + if re.match(pattern_re, s): + start, end, step = (s[1:-1] + ':1').split(':')[:3] + mapfunc = str + if start in string.ascii_letters: + istart = string.ascii_letters.index(start) + iend = string.ascii_letters.index(end) + 1 + if istart >= iend: + raise ValueError('invalid host range specified') + seq = string.ascii_letters[istart:iend:int(step)] + else: + if start[0] == '0' and len(start) > 1: + if len(start) != len(end): + raise ValueError('invalid host range specified') + mapfunc = lambda x: str(x).zfill(len(start)) + seq = xrange(int(start), int(end) + 1, int(step)) + iters.append(map(mapfunc, seq)) + elif re.search(r'[\[\]]', s): + raise ValueError('invalid host range specified') + elif s: + iters.append([s]) + for iname in iternest(*iters): + yield self.get_host(iname) + def get_group(self, name, all_group=None, child=False): ''' Return a MemGroup instance from group name, creating if needed. @@ -219,22 +256,23 @@ class IniLoader(BaseLoader): input_mode = 'host' group = self.get_group(line) else: - # Add a host or variable to the existing group/host + # Add hosts with inline variables, or variables/children to + # an existing group. tokens = shlex.split(line) if input_mode == 'host': - host = self.get_host(tokens[0]) - if len(tokens) > 1: - for t in tokens[1:]: - k,v = t.split('=', 1) - host.variables[k] = v - group.add_host(host) + for host in self.get_hosts(tokens[0]): + if len(tokens) > 1: + for t in tokens[1:]: + k,v = t.split('=', 1) + host.variables[k] = v + group.add_host(host) elif input_mode == 'children': group.child_group_by_name(line, self) elif input_mode == 'vars': for t in tokens: k, v = t.split('=', 1) group.variables[k] = v - # TODO: expansion patterns are probably not going to be supported + # TODO: expansion patterns are probably not going to be supported. YES THEY ARE! # from API documentation: diff --git a/awx/main/tests/commands.py b/awx/main/tests/commands.py index 6def182bcd..b4ef59a60f 100644 --- a/awx/main/tests/commands.py +++ b/awx/main/tests/commands.py @@ -5,6 +5,7 @@ import json import os import shutil +import string import StringIO import sys import tempfile @@ -65,6 +66,27 @@ varb=B vara=A ''' +TEST_INVENTORY_INI_WITH_HOST_PATTERNS = '''\ +[dotcom] +web[00:63].example.com ansible_ssh_user=example +dns.example.com + +[dotnet] +db-[a:z].example.net +ns.example.net + +[dotorg] +[A:F][0:9].example.org:1022 ansible_ssh_user=example +mx.example.org + +[dotus] +lb[00:08:2].example.us even_odd=even +lb[01:09:2].example.us even_odd=odd + +[dotcc] +media[0:9][0:9].example.cc +''' + TEST_GROUP_VARS = '''\ test_username: test test_email: test@example.com @@ -431,10 +453,11 @@ class InventoryImportTest(BaseCommandMixin, BaseLiveServerTest): self._temp_files.append(license_path) os.environ['AWX_LICENSE_FILE'] = license_path - def create_test_ini(self, inv_dir=None): + def create_test_ini(self, inv_dir=None, ini_content=None): + ini_content = ini_content or TEST_INVENTORY_INI handle, self.ini_path = tempfile.mkstemp(suffix='.txt', dir=inv_dir) ini_file = os.fdopen(handle, 'w') - ini_file.write(TEST_INVENTORY_INI) + ini_file.write(ini_content) ini_file.close() self._temp_files.append(self.ini_path) @@ -690,6 +713,101 @@ class InventoryImportTest(BaseCommandMixin, BaseLiveServerTest): def test_overwrite_from_ini_file(self): self.test_merge_from_ini_file(overwrite=True) + def test_ini_file_with_host_patterns(self): + self.create_test_ini(ini_content=TEST_INVENTORY_INI_WITH_HOST_PATTERNS) + # New empty inventory. + new_inv = self.organizations[0].inventories.create(name='newb') + self.assertEqual(new_inv.hosts.count(), 0) + self.assertEqual(new_inv.groups.count(), 0) + result, stdout, stderr = self.run_command('inventory_import', + inventory_id=new_inv.pk, + source=self.ini_path) + self.assertEqual(result, None, stdout + stderr) + # Check that inventory is populated as expected. + new_inv = Inventory.objects.get(pk=new_inv.pk) + expected_group_names = set(['dotcom', 'dotnet', 'dotorg', 'dotus', 'dotcc']) + group_names = set(new_inv.groups.values_list('name', flat=True)) + self.assertEqual(expected_group_names, group_names) + # Check that all host ranges are expanded into host names. + expected_host_names = set() + expected_host_names.update(['web%02d.example.com' % x for x in xrange(64)]) + expected_host_names.add('dns.example.com') + expected_host_names.update(['db-%s.example.net' % x for x in string.ascii_lowercase]) + expected_host_names.add('ns.example.net') + for x in 'ABCDEF': + for y in xrange(10): + expected_host_names.add('%s%d.example.org' % (x, y)) + expected_host_names.add('mx.example.org') + expected_host_names.update(['lb%02d.example.us' % x for x in xrange(10)]) + expected_host_names.update(['media%02d.example.cc' % x for x in xrange(100)]) + host_names = set(new_inv.hosts.values_list('name', flat=True)) + self.assertEqual(expected_host_names, host_names) + # Check hosts in dotcom group. + group = new_inv.groups.get(name='dotcom') + self.assertEqual(group.hosts.count(), 65) + for host in group.hosts.filter(active=True, name__startswith='web'): + self.assertEqual(host.variables_dict.get('ansible_ssh_user', ''), 'example') + # Check hosts in dotnet group. + group = new_inv.groups.get(name='dotnet') + self.assertEqual(group.hosts.count(), 27) + # Check hosts in dotorg group. + group = new_inv.groups.get(name='dotorg') + self.assertEqual(group.hosts.count(), 61) + for host in group.hosts.filter(active=True): + if host.name.startswith('mx.'): + continue + self.assertEqual(host.variables_dict.get('ansible_ssh_user', ''), 'example') + self.assertEqual(host.variables_dict.get('ansible_ssh_port', 22), 1022) + # Check hosts in dotus group. + group = new_inv.groups.get(name='dotus') + self.assertEqual(group.hosts.count(), 10) + for host in group.hosts.filter(active=True): + if int(host.name[2:4]) % 2 == 0: + self.assertEqual(host.variables_dict.get('even_odd', ''), 'even') + else: + self.assertEqual(host.variables_dict.get('even_odd', ''), 'odd') + # Check hosts in dotcc group. + group = new_inv.groups.get(name='dotcc') + self.assertEqual(group.hosts.count(), 100) + # Check inventory source/update after running command. + self.check_adhoc_inventory_source(new_inv) + # Test with invalid host pattern -- alpha begin > end. + self.create_test_ini(ini_content='[invalid]\nhost[X:P]') + result, stdout, stderr = self.run_command('inventory_import', + inventory_id=new_inv.pk, + source=self.ini_path) + self.assertTrue(isinstance(result, ValueError), result) + # Test with invalid host pattern -- different numeric pattern lengths. + self.create_test_ini(ini_content='[invalid]\nhost[001:08]') + result, stdout, stderr = self.run_command('inventory_import', + inventory_id=new_inv.pk, + source=self.ini_path) + self.assertTrue(isinstance(result, ValueError), result) + # Test with invalid host pattern -- invalid range/slice spec. + self.create_test_ini(ini_content='[invalid]\nhost[1:2:3:4]') + result, stdout, stderr = self.run_command('inventory_import', + inventory_id=new_inv.pk, + source=self.ini_path) + self.assertTrue(isinstance(result, ValueError), result) + # Test with invalid host pattern -- no begin. + self.create_test_ini(ini_content='[invalid]\nhost[:9]') + result, stdout, stderr = self.run_command('inventory_import', + inventory_id=new_inv.pk, + source=self.ini_path) + self.assertTrue(isinstance(result, ValueError), result) + # Test with invalid host pattern -- no end. + self.create_test_ini(ini_content='[invalid]\nhost[0:]') + result, stdout, stderr = self.run_command('inventory_import', + inventory_id=new_inv.pk, + source=self.ini_path) + self.assertTrue(isinstance(result, ValueError), result) + # Test with invalid host pattern -- invalid slice. + self.create_test_ini(ini_content='[invalid]\nhost[0:9:Q]') + result, stdout, stderr = self.run_command('inventory_import', + inventory_id=new_inv.pk, + source=self.ini_path) + self.assertTrue(isinstance(result, ValueError), result) + def test_executable_file(self): # New empty inventory. old_inv = self.inventories[1]