modularization of inventory_import command

This separates file parsing logic that was mixed
in with other important code inside of the
inventory import command.

The logic around MemObject data structures was
moved to utils, and the file parsing was moved
to a legacy module. As of this commit, that
module can operate within the Tower environment
but it will be removed.

Also refactor the loggers to fix old bug and
work inside of the different contexts - the
Loader classes, mem objects, and hopefully
the inventory modules eventually.

awx/api/serializers.py

@@ -1458,7 +1458,7 @@ class InventorySourceOptionsSerializer(BaseSerializer):
     class Meta:
         fields = ('*', 'source', 'source_path', 'source_script', 'source_vars', 'credential',
                   'source_regions', 'instance_filters', 'group_by', 'overwrite', 'overwrite_vars',
-                  'timeout')
+                  'timeout', 'verbosity')
 
     def get_related(self, obj):
         res = super(InventorySourceOptionsSerializer, self).get_related(obj)

awx/main/migrations/0038_v320_release.py

@@ -114,4 +114,16 @@ class Migration(migrations.Migration):
             name='notificationtemplate',
             unique_together=set([('organization', 'name')]),
         ),
+
+        # Add verbosity option to inventory updates
+        migrations.AddField(
+            model_name='inventorysource',
+            name='verbosity',
+            field=models.PositiveIntegerField(default=1, blank=True, choices=[(0, b'0 (WARNING)'), (1, b'1 (INFO)'), (2, b'2 (DEBUG)')]),
+        ),
+        migrations.AddField(
+            model_name='inventoryupdate',
+            name='verbosity',
+            field=models.PositiveIntegerField(default=1, blank=True, choices=[(0, b'0 (WARNING)'), (1, b'1 (INFO)'), (2, b'2 (DEBUG)')]),
+        ),
     ]

awx/main/models/inventory.py

@@ -733,6 +733,13 @@ class InventorySourceOptions(BaseModel):
         ('custom', _('Custom Script')),
     ]
 
+    # From the options of the Django management base command
+    INVENTORY_UPDATE_VERBOSITY_CHOICES = [
+        (0, '0 (WARNING)'),
+        (1, '1 (INFO)'),
+        (2, '2 (DEBUG)'),
+    ]
+
     # Use tools/scripts/get_ec2_filter_names.py to build this list.
     INSTANCE_FILTER_NAMES = [
         "architecture",
@@ -879,6 +886,11 @@ class InventorySourceOptions(BaseModel):
         blank=True,
         default=0,
     )
+    verbosity = models.PositiveIntegerField(
+        choices=INVENTORY_UPDATE_VERBOSITY_CHOICES,
+        blank=True,
+        default=1,
+    )
 
     @classmethod
     def get_ec2_region_choices(cls):
@@ -1116,7 +1128,7 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions):
     def _get_unified_job_field_names(cls):
         return ['name', 'description', 'source', 'source_path', 'source_script', 'source_vars', 'schedule',
                 'credential', 'source_regions', 'instance_filters', 'group_by', 'overwrite', 'overwrite_vars',
-                'timeout', 'launch_type', 'scm_project_update',]
+                'timeout', 'verbosity', 'launch_type', 'scm_project_update',]
 
     def save(self, *args, **kwargs):
         # If update_fields has been specified, add our field names to it,

awx/main/tasks.py

@@ -1761,6 +1761,16 @@ class RunInventoryUpdate(BaseTask):
 
         elif inventory_update.source == 'file':
             args.append(inventory_update.get_actual_source_path())
+            if hasattr(settings, 'ANSIBLE_INVENTORY_MODULE'):
+                module_name = settings.ANSIBLE_INVENTORY_MODULE
+            else:
+                module_name = 'backport'
+                v = get_ansible_version()
+                if Version(v) > Version('2.4'):
+                    module_name = 'modern'
+                elif Version(v) < Version('2.2'):
+                    module_name = 'legacy'
+            args.extend(['--method', module_name])
         elif inventory_update.source == 'custom':
             runpath = tempfile.mkdtemp(prefix='ansible_tower_launch_')
             handle, path = tempfile.mkstemp(dir=runpath)
@@ -1770,11 +1780,10 @@ class RunInventoryUpdate(BaseTask):
             f.write(inventory_update.source_script.script.encode('utf-8'))
             f.close()
             os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
-            args.append(runpath)
+            args.append(path)
             args.append("--custom")
             self.custom_dir_path.append(runpath)
-        verbosity = getattr(settings, 'INVENTORY_UPDATE_VERBOSITY', 1)
-        args.append('-v%d' % verbosity)
+        args.append('-v%d' % inventory_update.verbosity)
         if settings.DEBUG:
             args.append('--traceback')
         return args

awx/main/tests/unit/models/test_survey_models.py

@@ -79,6 +79,7 @@ def test_job_args_unredacted_passwords(job):
     assert extra_vars['secret_key'] == 'my_password'
 
 
+@pytest.mark.survey
 def test_update_kwargs_survey_invalid_default(survey_spec_factory):
     spec = survey_spec_factory('var2')
     spec['spec'][0]['required'] = False
@@ -91,6 +92,7 @@ def test_update_kwargs_survey_invalid_default(survey_spec_factory):
     assert json.loads(defaulted_extra_vars['extra_vars'])['var2'] == 2
 
 
+@pytest.mark.survey
 @pytest.mark.parametrize("question_type,default,expect_use,expect_value", [
     ("multiplechoice", "",       False, 'N/A'),  # historical bug
     ("multiplechoice", "zeb",    False, 'N/A'),  # zeb not in choices
@@ -125,6 +127,7 @@ def test_optional_survey_question_defaults(
         assert 'c' not in defaulted_extra_vars['extra_vars']
 
 
+@pytest.mark.survey
 class TestWorkflowSurveys:
     def test_update_kwargs_survey_defaults(self, survey_spec_factory):
         "Assure that the survey default over-rides a JT variable"

awx/main/tests/unit/plugins/test_tower_inventory_legacy.py

@@ -0,0 +1,88 @@
+import pytest
+import mock
+
+# python
+import os
+import sys
+
+# AWX main
+from awx.main.utils.mem_inventory import MemGroup
+
+# Add awx/plugins to sys.path so we can use the plugin
+TEST_DIR = os.path.dirname(__file__)
+path = os.path.abspath(os.path.join(
+    TEST_DIR, '..', '..', '..', '..', 'plugins', 'ansible_inventory'))
+if path not in sys.path:
+    sys.path.insert(0, path)
+
+# AWX plugin
+from legacy import IniLoader  # noqa
+
+
+@pytest.fixture
+def loader():
+    return IniLoader(TEST_DIR, MemGroup('all'))
+
+
+@pytest.mark.inventory_import
+class TestHostPatterns:
+
+    def test_simple_host_pattern(self, loader):
+        assert [h.name for h in loader.get_host_names_from_entry('server[1:3].io')] == [
+            'server1.io', 'server2.io', 'server3.io']
+
+    def test_host_with_port(self, loader):
+        assert [h.name for h in loader.get_host_names_from_entry('server.com:8080')] == ['server.com']
+        assert [h.variables['ansible_port'] for h in loader.get_host_names_from_entry('server.com:8080')] == [8080]
+
+    def test_host_pattern_with_step(self, loader):
+        assert [h.name for h in loader.get_host_names_from_entry('server[0:10:5].io')] == [
+            'server0.io', 'server5.io', 'server10.io']
+
+    def test_invalid_host_pattern_with_step(self, loader):
+        with pytest.raises(ValueError):
+            print [h.name for h in loader.get_host_names_from_entry('server[00:010:5].io')]
+
+    def test_alphanumeric_pattern(self, loader):
+        assert [h.name for h in loader.get_host_names_from_entry('server[a:c].io')] == [
+            'servera.io', 'serverb.io', 'serverc.io']
+
+    def test_invalid_alphanumeric_pattern(self, loader):
+        with pytest.raises(ValueError):
+            print [h.name for h in loader.get_host_names_from_entry('server[c:a].io')]
+
+
+@pytest.mark.inventory_import
+class TestLoader:
+
+    def test_group_and_host(self, loader):
+        group_and_host = mock.MagicMock(return_value=[
+            '[my_group]',
+            'my_host'
+        ])
+        with mock.patch.object(loader, 'file_line_iterable', group_and_host):
+            inventory = loader.load()
+            g = inventory.all_group.children[0]
+            assert g.name == 'my_group'
+            assert g.hosts[0].name
+
+    def test_host_comment(self, loader):
+        group_and_host = mock.MagicMock(return_value=['my_host     # and a comment'])
+        with mock.patch.object(loader, 'file_line_iterable', group_and_host):
+            inventory = loader.load()
+            assert inventory.all_group.hosts[0].name == 'my_host'
+
+    def test_group_parentage(self, loader):
+        group_and_host = mock.MagicMock(return_value=[
+            '[my_group]     # and a comment',
+            '[my_group:children]  # and a comment',
+            'child_group       # and a comment'
+        ])
+        with mock.patch.object(loader, 'file_line_iterable', group_and_host):
+            inventory = loader.load()
+            g = inventory.get_group('my_group')
+            assert g.name == 'my_group'
+            child = g.children[0]
+            assert child.name == 'child_group'
+            # We can not list non-root-level groups in the all_group
+            assert child not in inventory.all_group.children

awx/main/tests/unit/utils/test_mem_inventory.py

@@ -0,0 +1,128 @@
+# AWX utils
+from awx.main.utils.mem_inventory import (
+    MemInventory,
+    mem_data_to_dict, dict_to_mem_data
+)
+
+import pytest
+import json
+
+
+@pytest.fixture
+def memory_inventory():
+    inventory = MemInventory()
+    h = inventory.get_host('my_host')
+    h.variables = {'foo': 'bar'}
+    g = inventory.get_group('my_group')
+    g.variables = {'foobar': 'barfoo'}
+    h2 = inventory.get_host('group_host')
+    g.add_host(h2)
+    return inventory
+
+
+@pytest.fixture
+def JSON_of_inv():
+    # Implemented as fixture becuase it may be change inside of tests
+    return {
+        "_meta": {
+            "hostvars": {
+                "group_host": {},
+                "my_host": {"foo": "bar"}
+            }
+        }, 
+        "all": {"children": ["my_group",  "ungrouped"]},
+        "my_group": {
+            "hosts": ["group_host"],
+            "vars": {"foobar": "barfoo"}
+        },
+        "ungrouped": {"hosts": ["my_host"]}
+    }
+
+
+# Structure mentioned in official docs
+# https://docs.ansible.com/ansible/dev_guide/developing_inventory.html
+@pytest.fixture
+def JSON_with_lists():
+    docs_example = '''{
+       "databases"   : {
+           "hosts"   : [ "host1.example.com", "host2.example.com" ],
+           "vars"    : {
+               "a"   : true
+           }
+       },
+       "webservers"  : [ "host2.example.com", "host3.example.com" ],
+       "atlanta"     : {
+           "hosts"   : [ "host1.example.com", "host4.example.com", "host5.example.com" ],
+           "vars"    : {
+               "b"   : false
+           },
+           "children": [ "marietta", "5points" ]
+       },
+       "marietta"    : [ "host6.example.com" ],
+       "5points"     : [ "host7.example.com" ]
+    }'''
+    return json.loads(docs_example)
+
+
+# MemObject basic operations tests
+
+@pytest.mark.inventory_import
+def test_inventory_create_all_group():
+    inventory = MemInventory()
+    assert inventory.all_group.name == 'all'
+
+
+@pytest.mark.inventory_import
+def test_create_child_group():
+    inventory = MemInventory()
+    g1 = inventory.get_group('g1')
+    # Create new group by name as child of g1
+    g2 = inventory.get_group('g2', g1)
+    # Check that child is in the children of the parent group
+    assert g1.children == [g2]
+    # Check that _only_ the parent group is listed as a root group
+    assert inventory.all_group.children == [g1]
+    # Check that _both_ are tracked by the global `all_groups` dict
+    assert set(inventory.all_group.all_groups.values()) == set([g1, g2])
+
+
+@pytest.mark.inventory_import
+def test_ungrouped_mechanics():
+    # ansible-inventory returns a group called `ungrouped`
+    # we can safely treat this the same as the `all_group`
+    inventory = MemInventory()
+    ug = inventory.get_group('ungrouped')
+    assert ug is inventory.all_group
+
+
+# MemObject --> JSON tests
+
+@pytest.mark.inventory_import
+def test_convert_memory_to_JSON_with_vars(memory_inventory):
+    data = mem_data_to_dict(memory_inventory)
+    # Assertions about the variables on the objects
+    assert data['_meta']['hostvars']['my_host'] == {'foo': 'bar'}
+    assert data['my_group']['vars'] == {'foobar': 'barfoo'}
+    # Orphan host should be found in ungrouped false group
+    assert data['ungrouped']['hosts'] == ['my_host']
+
+
+# JSON --> MemObject tests
+
+@pytest.mark.inventory_import
+def test_convert_JSON_to_memory_with_vars(JSON_of_inv):
+    inventory = dict_to_mem_data(JSON_of_inv)
+    # Assertions about the variables on the objects
+    assert inventory.get_host('my_host').variables == {'foo': 'bar'}
+    assert inventory.get_group('my_group').variables == {'foobar': 'barfoo'}
+    # Host should be child of group
+    assert inventory.get_host('group_host') in inventory.get_group('my_group').hosts
+
+
+@pytest.mark.inventory_import
+def test_host_lists_accepted(JSON_with_lists):
+    inventory = dict_to_mem_data(JSON_with_lists)
+    assert inventory.get_group('marietta').name == 'marietta'
+    # Check that marietta's hosts was saved
+    h = inventory.get_host('host6.example.com')
+    assert h.name == 'host6.example.com'

awx/main/utils/formatters.py

@@ -5,6 +5,16 @@ from logstash.formatter import LogstashFormatterVersion1
 from copy import copy
 import json
 import time
+import logging
+
+
+class TimeFormatter(logging.Formatter):
+    '''
+    Custom log formatter used for inventory imports
+    '''
+    def format(self, record):
+        record.relativeSeconds = record.relativeCreated / 1000.0
+        return logging.Formatter.format(self, record)
 
 
 class LogstashFormatter(LogstashFormatterVersion1):

awx/main/utils/mem_inventory.py

@@ -0,0 +1,315 @@
+# Copyright (c) 2017 Ansible by Red Hat
+# All Rights Reserved.
+
+# Python
+import re
+import logging
+from collections import OrderedDict
+
+
+# Logger is used for any data-related messages so that the log level
+# can be adjusted on command invocation
+logger = logging.getLogger('awx.main.commands.inventory_import')
+
+
+__all__ = ['MemHost', 'MemGroup', 'MemInventory',
+           'mem_data_to_dict', 'dict_to_mem_data']
+
+
+ipv6_port_re = re.compile(r'^\[([A-Fa-f0-9:]{3,})\]:(\d+?)$')
+
+
+# Models for in-memory objects that represent an inventory
+
+
+class MemObject(object):
+    '''
+    Common code shared between in-memory groups and hosts.
+    '''
+
+    def __init__(self, name):
+        assert name, 'no name'
+        self.name = name
+
+
+class MemGroup(MemObject):
+    '''
+    In-memory representation of an inventory group.
+    '''
+
+    def __init__(self, name):
+        super(MemGroup, self).__init__(name)
+        self.children = []
+        self.hosts = []
+        self.variables = {}
+        self.parents = []
+        # Used on the "all" group in place of previous global variables.
+        # maps host and group names to hosts to prevent redudant additions
+        self.all_hosts = {}
+        self.all_groups = {}
+        self.variables = {}
+        logger.debug('Loaded group: %s', self.name)
+
+    def __repr__(self):
+        return '<_in-memory-group_ `{}`>'.format(self.name)
+
+    def add_child_group(self, group):
+        assert group.name is not 'all', 'group name is all'
+        assert isinstance(group, MemGroup), 'not MemGroup instance'
+        logger.debug('Adding child group %s to parent %s', group.name, self.name)
+        if group not in self.children:
+            self.children.append(group)
+        if self not in group.parents:
+            group.parents.append(self)
+
+    def add_host(self, host):
+        assert isinstance(host, MemHost), 'not MemHost instance'
+        logger.debug('Adding host %s to group %s', host.name, self.name)
+        if host not in self.hosts:
+            self.hosts.append(host)
+
+    def debug_tree(self, group_names=None):
+        group_names = group_names or set()
+        if self.name in group_names:
+            return
+        logger.debug('Dumping tree for group "%s":', self.name)
+        logger.debug('- Vars: %r', self.variables)
+        for h in self.hosts:
+            logger.debug('- Host: %s, %r',  h.name, h.variables)
+        for g in self.children:
+            logger.debug('- Child: %s', g.name)
+        logger.debug('----')
+        group_names.add(self.name)
+        for g in self.children:
+            g.debug_tree(group_names)
+
+
+class MemHost(MemObject):
+    '''
+    In-memory representation of an inventory host.
+    '''
+
+    def __init__(self, name, port=None):
+        super(MemHost, self).__init__(name)
+        self.variables = {}
+        self.instance_id = None
+        self.name = name
+        if port:
+            # was `ansible_ssh_port` in older Ansible/Tower versions
+            self.variables['ansible_port'] = port
+        logger.debug('Loaded host: %s', self.name)
+
+    def __repr__(self):
+        return '<_in-memory-host_ `{}`>'.format(self.name)
+
+
+class MemInventory(object):
+    '''
+    Common functions for an inventory loader from a given source.
+    '''
+    def __init__(self, all_group=None, group_filter_re=None, host_filter_re=None):
+        if all_group:
+            assert isinstance(all_group, MemGroup), '{} is not MemGroup instance'.format(all_group)
+            self.all_group = all_group
+        else:
+            self.all_group = self.create_group('all')
+        self.group_filter_re = group_filter_re
+        self.host_filter_re = host_filter_re
+
+    def create_host(self, host_name, port):
+        host = MemHost(host_name, port)
+        self.all_group.all_hosts[host_name] = host
+        return host
+
+    def get_host(self, name):
+        '''
+        Return a MemHost instance from host name, creating if needed.  If name
+        contains brackets, they will NOT be interpreted as a host pattern.
+        '''
+        m = ipv6_port_re.match(name)
+        if m:
+            host_name = m.groups()[0]
+            port = int(m.groups()[1])
+        elif name.count(':') == 1:
+            host_name = name.split(':')[0]
+            try:
+                port = int(name.split(':')[1])
+            except (ValueError, UnicodeDecodeError):
+                logger.warning(u'Invalid port "%s" for host "%s"',
+                               name.split(':')[1], host_name)
+                port = None
+        else:
+            host_name = name
+            port = None
+        if self.host_filter_re and not self.host_filter_re.match(host_name):
+            logger.debug('Filtering host %s', host_name)
+            return None
+        if host_name not in self.all_group.all_hosts:
+            self.create_host(host_name, port)
+        return self.all_group.all_hosts[host_name]
+
+    def create_group(self, group_name):
+        group = MemGroup(group_name)
+        if group_name not in ['all', 'ungrouped']:
+            self.all_group.all_groups[group_name] = group
+        return group
+
+    def get_group(self, name, all_group=None, child=False):
+        '''
+        Return a MemGroup instance from group name, creating if needed.
+        '''
+        all_group = all_group or self.all_group
+        if name in ['all', 'ungrouped']:
+            return all_group
+        if self.group_filter_re and not self.group_filter_re.match(name):
+            logger.debug('Filtering group %s', name)
+            return None
+        if name not in self.all_group.all_groups:
+            group = self.create_group(name)
+            if not child:
+                all_group.add_child_group(group)
+        return self.all_group.all_groups[name]
+
+    def delete_empty_groups(self):
+        for name, group in self.all_group.all_groups.items():
+            if not group.children and not group.hosts and not group.variables:
+                logger.debug('Removing empty group %s', name)
+                for parent in group.parents:
+                    if group in parent.children:
+                        parent.children.remove(group)
+                del self.all_group.all_groups[name]
+
+
+# Conversion utilities
+
+def mem_data_to_dict(inventory):
+    '''
+    Given an in-memory construct of an inventory, returns a dictionary that
+    follows Ansible guidelines on the structure of dynamic inventory sources
+
+    May be replaced by removing in-memory constructs within this file later
+    '''
+    all_group = inventory.all_group
+    inventory_data = OrderedDict([])
+    # Save hostvars to _meta
+    inventory_data['_meta'] = OrderedDict([])
+    hostvars = OrderedDict([])
+    for name, host_obj in all_group.all_hosts.items():
+        hostvars[name] = host_obj.variables
+    inventory_data['_meta']['hostvars'] = hostvars
+    # Save children of `all` group
+    inventory_data['all'] = OrderedDict([])
+    if all_group.variables:
+        inventory_data['all']['vars'] = all_group.variables
+    inventory_data['all']['children'] = [c.name for c in all_group.children]
+    inventory_data['all']['children'].append('ungrouped')
+    # Save details of declared groups individually
+    ungrouped_hosts = set(all_group.all_hosts.keys())
+    for name, group_obj in all_group.all_groups.items():
+        group_host_names = [h.name for h in group_obj.hosts]
+        group_children_names = [c.name for c in group_obj.children]
+        group_data = OrderedDict([])
+        if group_host_names:
+            group_data['hosts'] = group_host_names
+            ungrouped_hosts.difference_update(group_host_names)
+        if group_children_names:
+            group_data['children'] = group_children_names
+        if group_obj.variables:
+            group_data['vars'] = group_obj.variables
+        inventory_data[name] = group_data
+    # Save ungrouped hosts
+    inventory_data['ungrouped'] = OrderedDict([])
+    if ungrouped_hosts:
+        inventory_data['ungrouped']['hosts'] = list(ungrouped_hosts)
+    return inventory_data
+
+
+def dict_to_mem_data(data, inventory=None):
+    '''
+    In-place operation on `inventory`, adds contents from `data` to the
+    in-memory representation of memory.
+    May be destructive on `data`
+    '''
+    assert isinstance(data, dict), 'Expected dict, received {}'.format(type(data))
+    if inventory is None:
+        inventory = MemInventory()
+
+    _meta = data.pop('_meta', {})
+
+    for k,v in data.iteritems():
+        group = inventory.get_group(k)
+        if not group:
+            continue
+
+        # Load group hosts/vars/children from a dictionary.
+        if isinstance(v, dict):
+            # Process hosts within a group.
+            hosts = v.get('hosts', {})
+            if isinstance(hosts, dict):
+                for hk, hv in hosts.iteritems():
+                    host = inventory.get_host(hk)
+                    if not host:
+                        continue
+                    if isinstance(hv, dict):
+                        host.variables.update(hv)
+                    else:
+                        logger.warning('Expected dict of vars for '
+                                       'host "%s", got %s instead',
+                                       hk, str(type(hv)))
+                    group.add_host(host)
+            elif isinstance(hosts, (list, tuple)):
+                for hk in hosts:
+                    host = inventory.get_host(hk)
+                    if not host:
+                        continue
+                    group.add_host(host)
+            else:
+                logger.warning('Expected dict or list of "hosts" for '
+                               'group "%s", got %s instead', k,
+                               str(type(hosts)))
+            # Process group variables.
+            vars = v.get('vars', {})
+            if isinstance(vars, dict):
+                group.variables.update(vars)
+            else:
+                logger.warning('Expected dict of vars for '
+                               'group "%s", got %s instead',
+                               k, str(type(vars)))
+            # Process child groups.
+            children = v.get('children', [])
+            if isinstance(children, (list, tuple)):
+                for c in children:
+                    child = inventory.get_group(c, inventory.all_group, child=True)
+                    if child and c != 'ungrouped':
+                        group.add_child_group(child)
+            else:
+                logger.warning('Expected list of children for '
+                               'group "%s", got %s instead',
+                               k, str(type(children)))
+
+        # Load host names from a list.
+        elif isinstance(v, (list, tuple)):
+            for h in v:
+                host = inventory.get_host(h)
+                if not host:
+                    continue
+                group.add_host(host)
+        else:
+            logger.warning('')
+            logger.warning('Expected dict or list for group "%s", '
+                           'got %s instead', k, str(type(v)))
+
+        if k not in ['all', 'ungrouped']:
+            inventory.all_group.add_child_group(group)
+
+    if _meta:
+        for k,v in inventory.all_group.all_hosts.iteritems():
+            meta_hostvars = _meta['hostvars'].get(k, {})
+            if isinstance(meta_hostvars, dict):
+                v.variables.update(meta_hostvars)
+            else:
+                logger.warning('Expected dict of vars for '
+                               'host "%s", got %s instead',
+                               k, str(type(meta_hostvars)))
+
+    return inventory

awx/plugins/ansible_inventory/legacy.py

@@ -0,0 +1,253 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2017 Ansible by Red Hat
+# All Rights Reserved.
+
+# Python
+import glob
+import json
+import logging
+import os
+import shlex
+import argparse
+import re
+import string
+import yaml
+
+# import sys
+# # Add awx/plugins to sys.path so we can use the plugin
+# TEST_DIR = os.path.dirname(__file__)
+# path = os.path.abspath(os.path.join(TEST_DIR, '..', '..', 'main', 'utils'))
+# if path not in sys.path:
+#     sys.path.insert(0, path)
+
+# AWX
+from awx.main.utils.mem_inventory import (
+    MemGroup, MemInventory, mem_data_to_dict, ipv6_port_re
+)  # NOQA
+
+
+# Logger is used for any data-related messages so that the log level
+# can be adjusted on command invocation
+# logger = logging.getLogger('awx.plugins.ansible_inventory.tower_inventory_legacy')
+logger = logging.getLogger('awx.main.management.commands.inventory_import')
+
+
+class FileMemInventory(MemInventory):
+    '''
+    Adds on file-specific actions
+    '''
+    def __init__(self, source_dir, all_group, group_filter_re, host_filter_re, **kwargs):
+        super(FileMemInventory, self).__init__(all_group, group_filter_re, host_filter_re, **kwargs)
+        self.source_dir = source_dir
+
+    def load_vars(self, mem_object, dir_path):
+        all_vars = {}
+        files_found = 0
+        for suffix in ('', '.yml', '.yaml', '.json'):
+            path = ''.join([dir_path, suffix]).encode("utf-8")
+            if not os.path.exists(path):
+                continue
+            if not os.path.isfile(path):
+                continue
+            files_found += 1
+            if files_found > 1:
+                raise RuntimeError(
+                    'Multiple variable files found. There should only '
+                    'be one. %s ' % self.name)
+            vars_name = os.path.basename(os.path.dirname(path))
+            logger.debug('Loading %s from %s', vars_name, path)
+            try:
+                v = yaml.safe_load(file(path, 'r').read())
+                if hasattr(v, 'items'): # is a dict
+                    all_vars.update(v)
+            except yaml.YAMLError as e:
+                if hasattr(e, 'problem_mark'):
+                    logger.error('Invalid YAML in %s:%s col %s', path,
+                                 e.problem_mark.line + 1,
+                                 e.problem_mark.column + 1)
+                else:
+                    logger.error('Error loading YAML from %s', path)
+                raise
+        return all_vars
+
+    def create_host(self, host_name, port):
+        host = super(FileMemInventory, self).create_host(host_name, port)
+        host_vars_dir = os.path.join(self.source_dir, 'host_vars', host.name)
+        host.variables.update(self.load_vars(host, host_vars_dir))
+        return host
+
+    def create_group(self, group_name):
+        group = super(FileMemInventory, self).create_group(group_name)
+        group_vars_dir = os.path.join(self.source_dir, 'group_vars', group.name)
+        group.variables.update(self.load_vars(group, group_vars_dir))
+        return group
+
+
+class IniLoader(object):
+    '''
+    Loader to read inventory from an INI-formatted text file.
+    '''
+    def __init__(self, source, all_group=None, group_filter_re=None, host_filter_re=None):
+        self.source = source
+        self.source_dir = os.path.dirname(self.source)
+        self.inventory = FileMemInventory(
+            self.source_dir, all_group,
+            group_filter_re=group_filter_re, host_filter_re=host_filter_re)
+
+    def get_host_names_from_entry(self, name):
+        '''
+        Given an entry in an Ansible inventory file, return an iterable of
+        the resultant host names, accounting for expansion patterns.
+        Examples:
+        web1.server.com -> web1.server.com
+        web[1:2].server.com -> web1.server.com, web2.server.com
+        '''
+        def iternest(*args):
+            if args:
+                for i in args[0]:
+                    for j in iternest(*args[1:]):
+                        yield ''.join([str(i), j])
+            else:
+                yield ''
+        if ipv6_port_re.match(name):
+            yield self.inventory.get_host(name)
+            return
+        pattern_re = re.compile(r'(\[(?:(?:\d+\:\d+)|(?:[A-Za-z]\:[A-Za-z]))(?:\:\d+)??\])')
+        iters = []
+        for s in re.split(pattern_re, name):
+            if re.match(pattern_re, s):
+                start, end, step = (s[1:-1] + ':1').split(':')[:3]
+                mapfunc = str
+                if start in string.ascii_letters:
+                    istart = string.ascii_letters.index(start)
+                    iend = string.ascii_letters.index(end) + 1
+                    if istart >= iend:
+                        raise ValueError('invalid host range specified')
+                    seq = string.ascii_letters[istart:iend:int(step)]
+                else:
+                    if start[0] == '0' and len(start) > 1:
+                        if len(start) != len(end):
+                            raise ValueError('invalid host range specified')
+                        mapfunc = lambda x: str(x).zfill(len(start))
+                    seq = xrange(int(start), int(end) + 1, int(step))
+                iters.append(map(mapfunc, seq))
+            elif re.search(r'[\[\]]', s):
+                raise ValueError('invalid host range specified')
+            elif s:
+                iters.append([s])
+        for iname in iternest(*iters):
+            yield self.inventory.get_host(iname)
+
+    @staticmethod
+    def file_line_iterable(filename):
+        return file(filename, 'r')
+
+    def load(self):
+        logger.info('Reading INI source: %s', self.source)
+        group = self.inventory.all_group
+        input_mode = 'host'
+        for line in self.file_line_iterable(self.source):
+            line = line.split('#')[0].strip()
+            if not line:
+                continue
+            elif line.startswith('[') and line.endswith(']'):
+                # Mode change, possible new group name
+                line = line[1:-1].strip()
+                if line.endswith(':vars'):
+                    input_mode = 'vars'
+                    line = line[:-5]
+                elif line.endswith(':children'):
+                    input_mode = 'children'
+                    line = line[:-9]
+                else:
+                    input_mode = 'host'
+                group = self.inventory.get_group(line)
+            elif group:
+                # If group is None, we are skipping this group and shouldn't
+                # capture any children/variables/hosts under it.
+                # Add hosts with inline variables, or variables/children to
+                # an existing group.
+                tokens = shlex.split(line)
+                if input_mode == 'host':
+                    for host in self.get_host_names_from_entry(tokens[0]):
+                        if not host:
+                            continue
+                        if len(tokens) > 1:
+                            for t in tokens[1:]:
+                                k,v = t.split('=', 1)
+                                host.variables[k] = v
+                        group.add_host(host)
+                elif input_mode == 'children':
+                    self.inventory.get_group(line, group)
+                elif input_mode == 'vars':
+                    for t in tokens:
+                        k, v = t.split('=', 1)
+                        group.variables[k] = v
+        return self.inventory
+
+
+def load_inventory_source(source, all_group=None, group_filter_re=None,
+                          host_filter_re=None, exclude_empty_groups=False):
+    '''
+    Load inventory from given source directory or file.
+    '''
+    original_all_group = all_group
+    if not os.path.exists(source):
+        raise IOError('Source does not exist: %s' % source)
+    source = os.path.join(os.getcwd(), os.path.dirname(source),
+                          os.path.basename(source))
+    source = os.path.normpath(os.path.abspath(source))
+    if os.path.isdir(source):
+        all_group = all_group or MemGroup('all')
+        for filename in glob.glob(os.path.join(source, '*')):
+            if filename.endswith(".ini") or os.path.isdir(filename):
+                continue
+            load_inventory_source(filename, all_group, group_filter_re,
+                                  host_filter_re)
+    elif os.access(source, os.X_OK):
+        raise NotImplementedError(
+            'Source has been marked as executable, but script-based sources '
+            'are not supported by the legacy file import plugin. '
+            'This problem may be solved by upgrading to use `ansible-inventory`.')
+    else:
+        all_group = all_group or MemGroup('all', os.path.dirname(source))
+        IniLoader(source, all_group, group_filter_re, host_filter_re).load()
+
+    logger.debug('Finished loading from source: %s', source)
+    # Exclude groups that are completely empty.
+    if original_all_group is None and exclude_empty_groups:
+        for name, group in all_group.all_groups.items():
+            if not group.children and not group.hosts and not group.variables:
+                logger.debug('Removing empty group %s', name)
+                for parent in group.parents:
+                    if group in parent.children:
+                        parent.children.remove(group)
+                del all_group.all_groups[name]
+    if original_all_group is None:
+        logger.info('Loaded %d groups, %d hosts', len(all_group.all_groups),
+                    len(all_group.all_hosts))
+    return all_group
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Ansible Inventory Import Plugin - Fallback Option')
+    parser.add_argument(
+        '-i', '--inventory-file', dest='inventory', required=True,
+        help="Specify inventory host path (does not support CSV host paths)")
+    parser.add_argument(
+        '--list', action='store_true', dest='list', default=None, required=True,
+        help='Output all hosts info, works as inventory script')
+    # --host and --graph and not supported
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+    args = parse_args()
+    source = args.inventory
+    memory_data = load_inventory_source(
+        source, group_filter_re=None,
+        host_filter_re=None, exclude_empty_groups=False)
+    mem_inventory = MemInventory(all_group=memory_data)
+    inventory_dict = mem_data_to_dict(mem_inventory)
+    print json.dumps(inventory_dict, indent=4)

awx/settings/defaults.py

@@ -907,6 +907,10 @@ LOGGING = {
         },
         'json': {
             '()': 'awx.main.utils.formatters.LogstashFormatter'
+        },
+        'timed_import': {
+            '()': 'awx.main.utils.formatters.TimeFormatter',
+            'format': '%(relativeSeconds)9.3f %(levelname)-8s %(message)s'
         }
     },
     'handlers': {
@@ -958,6 +962,11 @@ LOGGING = {
             'backupCount': 5,
             'formatter':'simple',
         },
+        'inventory_import': {
+            'level': 'DEBUG',
+            'class':'logging.StreamHandler',
+            'formatter': 'timed_import',
+        },
         'task_system': {
             'level': 'INFO',
             'class':'logging.handlers.RotatingFileHandler',
@@ -1029,6 +1038,10 @@ LOGGING = {
         'awx.main.commands.run_callback_receiver': {
             'handlers': ['callback_receiver'],
         },
+        'awx.main.commands.inventory_import': {
+            'handlers': ['inventory_import'],
+            'propagate': False
+        },
         'awx.main.tasks': {
             'handlers': ['task_system'],
         },

pytest.ini

@@ -8,3 +8,5 @@ markers =
     ac: access control test
     license_feature: ensure license features are accessible or not depending on license
     mongo_db: drop mongodb test database before test runs
+    survey: tests related to survey feature
+    inventory_import: tests of code used by inventory import command