Clean-up intiail commit for Host filter / DynamicInventory

awx/api/views.py

@@ -79,7 +79,7 @@ from awx.api.metadata import RoleMetadata
 from awx.main.consumers import emit_channel_notification
 from awx.main.models.unified_jobs import ACTIVE_STATES
 from awx.main.scheduler.tasks import run_job_complete
-from awx.main.fields import DynamicFilterField
+from awx.main.querysets import DynamicFilterQuerySet
 
 logger = logging.getLogger('awx.api.views')
 
@@ -1764,10 +1764,10 @@ class HostList(ListCreateAPIView):
     capabilities_prefetch = ['inventory.admin']
 
     def get_queryset(self):
-        qs = super(HostList, self).get_queryset()
+        qs = DynamicFilterQuerySet(HostList, using=self._db)
         filter_string = self.request.query_params.get('host_filter', None)
         if filter_string:
-            filter_q = DynamicFilterField.filter_string_to_q(filter_string)
+            filter_q = qs.query_from_string(filter_string)
             qs = qs.filter(filter_q)
         return qs
 

awx/main/fields.py

@@ -4,10 +4,7 @@
 # Python
 import copy
 import json
-import re
-import sys
 import six
-from pyparsing import infixNotation, opAssoc, Optional, Literal, CharsNotIn
 
 from jinja2 import Environment, StrictUndefined
 from jinja2.exceptions import UndefinedError
@@ -28,7 +25,6 @@ from django.db.models.fields.related import (
     ReverseManyRelatedObjectsDescriptor,
 )
 from django.utils.encoding import smart_text
-from django.db.models import Q
 from django.utils.translation import ugettext_lazy as _
 
 # jsonschema
@@ -39,6 +35,7 @@ from jsonfield import JSONField as upstream_JSONField
 from jsonbfield.fields import JSONField as upstream_JSONBField
 
 # AWX
+from awx.main.querysets import DynamicFilterQuerySet
 from awx.main.models.rbac import batch_role_ancestor_rebuilding, Role
 from awx.main import utils
 
@@ -332,202 +329,15 @@ class ImplicitRoleField(models.ForeignKey):
         Role.rebuild_role_ancestor_list([], child_ids)
 
 
-unicode_spaces = [unichr(c) for c in xrange(sys.maxunicode) if unichr(c).isspace()]
-unicode_spaces_other = unicode_spaces + [u'(', u')', u'=', u'"']
-
-
-def string_to_type(t):
-    if t == u'true':
-        return True
-    elif t == u'false':
-        return False
-
-    if re.search('^[-+]?[0-9]+$',t):
-        return int(t)
-
-    if re.search('^[-+]?[0-9]+\.[0-9]+$',t):
-        return float(t)
-
-    return t
-
-
 class DynamicFilterField(models.TextField):
-    SEARCHABLE_RELATIONSHIP = 'ansible_facts'
+    def get_prep_value(self, value):
-
+        if value is None:
-    class BoolOperand(object):
+            return value
-        def __init__(self, t):
-            kwargs = dict()
-            k, v = self._extract_key_value(t)
-            k, v = self._json_path_to_contains(k, v)
-            kwargs[k] = v
-            self.result = Q(**kwargs)
-
-        def strip_quotes_traditional_logic(self, v):
-            if type(v) is unicode and v.startswith('"') and v.endswith('"'):
-                return v[1:-1]
-            return v
-
-        def strip_quotes_json_logic(self, v):
-            if type(v) is unicode and v.startswith('"') and v.endswith('"') and v != u'"null"':
-                return v[1:-1]
-            return v
-
-        '''
-        TODO: We should be able to express this in the grammar and let 
-              pyparsing do the heavy lifting.
-        TODO: separate django filter requests from our custom json filter
-              request so we don't process the key any. This could be
-              accomplished using a whitelist or introspecting the
-              relationship refered to to see if it's a jsonb type.
-        '''
-        def _json_path_to_contains(self, k, v):
-            if not k.startswith(DynamicFilterField.SEARCHABLE_RELATIONSHIP):
-                v = self.strip_quotes_traditional_logic(v)
-                return (k, v)
-
-            # Strip off leading relationship key
-            if k.startswith(DynamicFilterField.SEARCHABLE_RELATIONSHIP + '__'):
-                strip_len = len(DynamicFilterField.SEARCHABLE_RELATIONSHIP) + 2
-            else:
-                strip_len = len(DynamicFilterField.SEARCHABLE_RELATIONSHIP)
-            k = k[strip_len:]
-
-            pieces = k.split(u'__')
-
-            assembled_k = u'%s__contains' % (DynamicFilterField.SEARCHABLE_RELATIONSHIP)
-            assembled_v = None
-
-            last_v = None
-            last_kv = None
-
-            for i, piece in enumerate(pieces):
-                new_kv = dict()
-                if piece.endswith(u'[]'):
-                    new_v = []
-                    new_kv[piece[0:-2]] = new_v
-                else:
-                    new_v = dict()
-                    new_kv[piece] = new_v
-
-                if last_kv is None:
-                    assembled_v = new_kv
-                elif type(last_v) is list:
-                    last_v.append(new_kv)
-                elif type(last_v) is dict:
-                    last_kv[last_kv.keys()[0]] = new_kv
-
-                last_v = new_v
-                last_kv = new_kv
-
-            v = self.strip_quotes_json_logic(v)
-
-            if type(last_v) is list:
-                last_v.append(v)
-            elif type(last_v) is dict:
-                last_kv[last_kv.keys()[0]] = v
-
-            return (assembled_k, assembled_v)
-
-        def _extract_key_value(self, t):
-            t_len = len(t)
-
-            k = None
-            v = None
-
-            # key
-            # "something"=
-            v_offset = 2
-            if t_len >= 2 and t[0] == "\"" and t[2] == "\"":
-                k = t[1]
-                v_offset = 4
-            # something=
-            else:
-                k = t[0]
-
-            # value
-            # ="something"
-            if t_len > (v_offset + 2) and t[v_offset] == "\"" and t[v_offset + 2] == "\"":
-                v = u'"' + unicode(t[v_offset + 1]) + u'"'
-                #v = t[v_offset + 1]
-            # empty ""
-            elif t_len > (v_offset + 1):
-                v = u""
-            # no ""
-            else:
-                v = string_to_type(t[v_offset])
-
-            return (k, v)
-
-
-    class BoolBinOp(object):
-        def __init__(self, t):
-            self.result = None
-            i = 2
-            while i < len(t[0]):
-                if not self.result:
-                    self.result = t[0][0].result
-                right = t[0][i].result
-                self.result = self.execute_logic(self.result, right)
-                i += 2
-
-
-    class BoolAnd(BoolBinOp):
-        def execute_logic(self, left, right):
-            return left & right
-
-
-    class BoolOr(BoolBinOp):
-        def execute_logic(self, left, right):
-            return left | right
-
-
-    class BoolNot(object):
-        def __init__(self, t):
-            self.right = t[0][1].result
-
-            self.result = self.execute_logic(self.right)
-
-        def execute_logic(self, right):
-            return ~right
-
-
-    @classmethod
-    def filter_string_to_q(cls, filter_string):
-
-        '''
-        TODO:
-        * handle values with " via: a.b.c.d="hello\"world"
-        * handle keys with " via: a.\"b.c="yeah"
-        * handle key with __ in it
-
-        '''
-        filter_string_raw = filter_string
-        filter_string = unicode(filter_string)
-
-        atom = CharsNotIn(unicode_spaces_other)
-        atom_inside_quotes = CharsNotIn(u'"')
-        atom_quoted = Literal('"') + Optional(atom_inside_quotes) + Literal('"')
-        EQUAL = Literal('=')
-
-        grammar = ((atom_quoted | atom) + EQUAL + Optional((atom_quoted | atom)))
-        grammar.setParseAction(cls.BoolOperand)
-
-        boolExpr = infixNotation(grammar, [
-            ("not", 1, opAssoc.RIGHT, cls.BoolNot),
-            ("and", 2, opAssoc.LEFT, cls.BoolAnd),
-            ("or",  2, opAssoc.LEFT, cls.BoolOr),
-        ])
-
         try:
-            res = boolExpr.parseString('(' + filter_string + ')')
+            DynamicFilterQuerySet().query_from_string(value)
-        #except ParseException as e:
+        except RuntimeError, e:
-        except Exception:
+            raise models.base.ValidationError(e)
-            raise RuntimeError(u"Invalid query %s" % filter_string_raw)
+        return super(DynamicFilterField, self).get_prep_value(value)
-
-        if len(res) > 0:
-            return res[0].result
-
-        raise RuntimeError("Parsing the filter_string %s went terribly wrong" % filter_string)
 
 
 class JSONSchemaField(JSONBField):

awx/main/managers.py

@@ -9,6 +9,8 @@ from django.utils.timezone import now
 from django.db.models import Sum
 from django.conf import settings
 
+from awx.main.querysets import DynamicFilterQuerySet
+
 
 class HostManager(models.Manager):
     """Custom manager class for Hosts model."""
@@ -20,6 +22,24 @@ class HostManager(models.Manager):
         except NotImplementedError: # For unit tests only, SQLite doesn't support distinct('name')
             return len(set(self.values_list('name', flat=True)))
 
+    def get_queryset(self):
+        """When the Inventory this host belongs to has a `host_filter` set
+        generate the QuerySet using that filter. Otherwise just return the default filter.
+        """
+        qs = DynamicFilterQuerySet(self.model, using=self._db)
+        if self.instance is not None:
+            if hasattr(self.instance, 'host_filter') and self.instance.host_filter is not None:
+                q = qs.query_from_string(self.instance.host_filter)
+                # If we are using host_filters, disable the core_filters, this allows
+                # us to access all of the available Host entries, not just the ones associated
+                # with a specific FK/relation.
+                #
+                # If we don't disable this, a filter of {'inventory': self.instance} gets automatically
+                # injected by the related object mapper.
+                self.core_filters = {}
+                return qs.filter(q)
+        return qs
+
 
 class InstanceManager(models.Manager):
     """A custom manager class for the Instance model.

awx/main/migrations/0038_v320_release.py

@@ -36,6 +36,11 @@ class Migration(migrations.Migration):
             name='inventory',
             field=models.ForeignKey(related_name='inventory_sources', default=None, to='main.Inventory', null=True),
         ),
+        migrations.AddField(
+            model_name='inventory',
+            name='host_filter',
+            field=awx.main.fields.DynamicFilterField(default=None, help_text='Filter that will be applied to the hosts of this inventory.', null=True, blank=True),
+        ),
 
         # Facts
         migrations.AlterField(

awx/main/models/inventory.py

@@ -20,7 +20,11 @@ from django.utils.timezone import now
 # AWX
 from awx.api.versioning import reverse
 from awx.main.constants import CLOUD_PROVIDERS
-from awx.main.fields import ImplicitRoleField, JSONBField
+from awx.main.fields import (
+    ImplicitRoleField,
+    JSONBField,
+    DynamicFilterField,
+)
 from awx.main.managers import HostManager
 from awx.main.models.base import * # noqa
 from awx.main.models.unified_jobs import * # noqa
@@ -99,6 +103,13 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin):
         editable=False,
         help_text=_('Number of external inventory sources in this inventory with failures.'),
     )
+    host_filter = DynamicFilterField(
+        blank=True,
+        null=True,
+        default=None,
+        help_text=_('Filter that will be applied to the hosts of this inventory.'),
+    )
+
     admin_role = ImplicitRoleField(
         parent_role='organization.admin_role',
     )

awx/main/querysets.py

@@ -0,0 +1,218 @@
+import re
+import sys
+from pyparsing import (
+    infixNotation,
+    opAssoc,
+    Optional,
+    Literal,
+    CharsNotIn,
+)
+
+from django.db import models
+
+
+__all__ = ['DynamicFilterQuerySet']
+
+
+unicode_spaces = [unichr(c) for c in xrange(sys.maxunicode) if unichr(c).isspace()]
+unicode_spaces_other = unicode_spaces + [u'(', u')', u'=', u'"']
+
+
+def string_to_type(t):
+    if t == u'true':
+        return True
+    elif t == u'false':
+        return False
+
+    if re.search('^[-+]?[0-9]+$',t):
+        return int(t)
+
+    if re.search('^[-+]?[0-9]+\.[0-9]+$',t):
+        return float(t)
+
+    return t
+
+
+class DynamicFilterQuerySet(models.QuerySet):
+
+
+    class BoolOperand(object):
+        def __init__(self, t):
+            kwargs = dict()
+            k, v = self._extract_key_value(t)
+            k, v = self._json_path_to_contains(k, v)
+            kwargs[k] = v
+            self.result = models.Q(**kwargs)
+
+        '''
+        TODO: We should be able to express this in the grammar and let
+              pyparsing do the heavy lifting.
+        TODO: separate django filter requests from our custom json filter
+              request so we don't process the key any. This could be
+              accomplished using a whitelist or introspecting the
+              relationship refered to to see if it's a jsonb type.
+        '''
+        def _json_path_to_contains(self, k, v):
+            pieces = k.split('__')
+
+            flag_first_arr_found = False
+
+            assembled_k = ''
+            assembled_v = v
+
+            last_kv = None
+            last_v = None
+
+            contains_count = 0
+            for i, piece in enumerate(pieces):
+                if flag_first_arr_found is False and piece.endswith('[]'):
+                    assembled_k += u'%s__contains' % (piece[0:-2])
+                    contains_count += 1
+                    flag_first_arr_found = True
+                elif flag_first_arr_found is False and i == len(pieces) - 1:
+                    assembled_k += u'%s' % piece
+                elif flag_first_arr_found is False:
+                    assembled_k += u'%s__' % piece
+                elif flag_first_arr_found is True:
+                    new_kv = dict()
+                    if piece.endswith('[]'):
+                        new_v = []
+                        new_kv[piece[0:-2]] = new_v
+                    else:
+                        new_v = dict()
+                        new_kv[piece] = new_v
+
+
+                    if last_v is None:
+                        last_v = []
+                        assembled_v = last_v
+
+                    if type(last_v) is list:
+                        last_v.append(new_kv)
+                    elif type(last_v) is dict:
+                        last_kv[last_kv.keys()[0]] = new_kv
+
+                    last_v = new_v
+                    last_kv = new_kv
+                    contains_count += 1
+
+            '''
+            Explicit quotes are kept until this point.
+            Note: we could have totally "ripped" them off earlier when we decided
+            what type to convert the token to.
+            '''
+            if type(v) is unicode and v.startswith('"') and v.endswith('"') and v != u'"null"':
+                v = v[1:-1]
+
+            if contains_count == 0:
+                assembled_v = v
+            elif contains_count == 1:
+                assembled_v = [v]
+            elif contains_count > 1:
+                if type(last_v) is list:
+                    last_v.append(v)
+                if type(last_v) is dict:
+                    last_kv[last_kv.keys()[0]] = v
+
+            return (assembled_k, assembled_v)
+
+        def _extract_key_value(self, t):
+            t_len = len(t)
+
+            k = None
+            v = None
+
+            # key
+            # "something"=
+            v_offset = 2
+            if t_len >= 2 and t[0] == "\"" and t[2] == "\"":
+                k = t[1]
+                v_offset = 4
+            # something=
+            else:
+                k = t[0]
+
+            # value
+            # ="something"
+            if t_len > (v_offset + 2) and t[v_offset] == "\"" and t[v_offset + 2] == "\"":
+                v = u'"' + unicode(t[v_offset + 1]) + u'"'
+                #v = t[v_offset + 1]
+            # empty ""
+            elif t_len > (v_offset + 1):
+                v = u""
+            # no ""
+            else:
+                v = string_to_type(t[v_offset])
+
+            return (k, v)
+
+
+    class BoolBinOp(object):
+        def __init__(self, t):
+            self.result = None
+            i = 2
+            while i < len(t[0]):
+                if not self.result:
+                    self.result = t[0][0].result
+                right = t[0][i].result
+                self.result = self.execute_logic(self.result, right)
+                i += 2
+
+
+    class BoolAnd(BoolBinOp):
+        def execute_logic(self, left, right):
+            return left & right
+
+
+    class BoolOr(BoolBinOp):
+        def execute_logic(self, left, right):
+            return left | right
+
+
+    class BoolNot(object):
+        def __init__(self, t):
+            self.right = t[0][1].result
+
+            self.result = self.execute_logic(self.right)
+
+        def execute_logic(self, right):
+            return ~right
+
+
+    @classmethod
+    def query_from_string(cls, filter_string):
+        '''
+        TODO:
+        * handle values with " via: a.b.c.d="hello\"world"
+        * handle keys with " via: a.\"b.c="yeah"
+        * handle key with __ in it
+
+        '''
+        filter_string_raw = filter_string
+        filter_string = unicode(filter_string)
+
+        atom = CharsNotIn(unicode_spaces_other)
+        atom_inside_quotes = CharsNotIn(u'"')
+        atom_quoted = Literal('"') + Optional(atom_inside_quotes) + Literal('"')
+        EQUAL = Literal('=')
+
+        grammar = ((atom_quoted | atom) + EQUAL + Optional((atom_quoted | atom)))
+        grammar.setParseAction(cls.BoolOperand)
+
+        boolExpr = infixNotation(grammar, [
+            ("not", 1, opAssoc.RIGHT, cls.BoolNot),
+            ("and", 2, opAssoc.LEFT, cls.BoolAnd),
+            ("or",  2, opAssoc.LEFT, cls.BoolOr),
+        ])
+
+        try:
+            res = boolExpr.parseString('(' + filter_string + ')')
+        except Exception:
+            raise RuntimeError(u"Invalid query %s" % filter_string_raw)
+
+        if len(res) > 0:
+            return res[0].result
+
+        raise RuntimeError("Parsing the filter_string %s went terribly wrong" % filter_string)
+
+

awx/main/tests/functional/models/test_inventory.py

@@ -2,7 +2,11 @@ import pytest
 import mock
 
 # AWX
-from awx.main.models import InventorySource, InventoryUpdate
+from awx.main.models import (
+    Inventory,
+    InventorySource,
+    InventoryUpdate,
+)
 
 
 @pytest.mark.django_db
@@ -31,3 +35,31 @@ class TestSCMUpdateFeatures:
             scm_inventory_source.description = "I'm testing this!"
             scm_inventory_source.save()
             assert not mck_update.called
+
+
+@pytest.mark.django_db
+def test_host_objects_manager(organization):
+    dynamic_inventory = Inventory(organization=organization, name='dynamic', host_filter='inventory_sources__source=ec2')
+    dynamic_inventory.save()
+
+    ec2_inv = Inventory(name='test_ec2', organization=organization)
+    ec2_inv.save()
+
+    ec2_source = ec2_inv.inventory_sources.create(name='test_ec2_source', source='ec2')
+    for i in range(2):
+        ec2_host = ec2_inv.hosts.create(name='test_ec2_{0}'.format(i))
+        ec2_host.inventory_sources.add(ec2_source)
+    ec2_inv.save()
+
+    gce_inv = Inventory(name='test_gce', organization=organization)
+    gce_inv.save()
+
+    gce_source = gce_inv.inventory_sources.create(name='test_gce_source', source='gce')
+    gce_host = gce_inv.hosts.create(name='test_gce_host')
+    gce_host.inventory_sources.add(gce_source)
+    gce_inv.save()
+
+    hosts = dynamic_inventory.hosts.all()
+    assert len(hosts) == 2
+    assert hosts[0].inventory_sources.first() == ec2_source
+    assert hosts[1].inventory_sources.first() == ec2_source

awx/main/tests/unit/test_querysets.py

@@ -3,14 +3,13 @@
 import pytest
 
 # AWX
-from awx.main.fields import DynamicFilterField
+from awx.main.querysets import DynamicFilterQuerySet
 
 # Django
 from django.db.models import Q
 
 
-
+class TestDynamicFilterQuerySetQueryFromString():
-class TestDynamicFilterFieldFilterStringToQ():
     @pytest.mark.parametrize("filter_string,q_expected", [
         ('facts__facts__blank=""', Q(**{u"facts__facts__blank": u""})),
         ('"facts__facts__ space "="f"', Q(**{u"facts__facts__ space ": u"f"})),
@@ -24,7 +23,7 @@ class TestDynamicFilterFieldFilterStringToQ():
         #('a__b\"__c="true"', Q(**{u"a__b\"__c": "true"})),
     ])
     def test_query_generated(self, filter_string, q_expected):
-        q = DynamicFilterField.filter_string_to_q(filter_string)
+        q = DynamicFilterQuerySet.query_from_string(filter_string)
         assert unicode(q) == unicode(q_expected)
 
     @pytest.mark.parametrize("filter_string", [
@@ -33,7 +32,7 @@ class TestDynamicFilterFieldFilterStringToQ():
     ])
     def test_invalid_filter_strings(self, filter_string):
         with pytest.raises(RuntimeError) as e:
-            DynamicFilterField.filter_string_to_q(filter_string)
+            DynamicFilterQuerySet.query_from_string(filter_string)
         assert e.value.message == u"Invalid query " + filter_string
 
     @pytest.mark.parametrize("filter_string,q_expected", [
@@ -41,7 +40,7 @@ class TestDynamicFilterFieldFilterStringToQ():
         (u'(ansible_facts__a=abc\u1F5E3def)', Q(**{u"ansible_facts__contains": {u"a": u"abc\u1F5E3def"}})),
     ])
     def test_unicode(self, filter_string, q_expected):
-        q = DynamicFilterField.filter_string_to_q(filter_string)
+        q = DynamicFilterQuerySet.query_from_string(filter_string)
         assert unicode(q) == unicode(q_expected)
 
     @pytest.mark.parametrize("filter_string,q_expected", [
@@ -55,7 +54,7 @@ class TestDynamicFilterFieldFilterStringToQ():
         ('a=b or a=d or a=e or a=z and b=h and b=i and b=j and b=k', Q(**{u"a": u"b"}) | Q(**{u"a": u"d"}) | Q(**{u"a": u"e"}) | Q(**{u"a": u"z"}) & Q(**{u"b": u"h"}) & Q(**{u"b": u"i"}) & Q(**{u"b": u"j"}) & Q(**{u"b": u"k"}))
     ])
     def test_boolean_parenthesis(self, filter_string, q_expected):
-        q = DynamicFilterField.filter_string_to_q(filter_string)
+        q = DynamicFilterQuerySet.query_from_string(filter_string)
         assert unicode(q) == unicode(q_expected)
 
     @pytest.mark.parametrize("filter_string,q_expected", [
@@ -75,7 +74,7 @@ class TestDynamicFilterFieldFilterStringToQ():
         #('a__b\"__c="true"', Q(**{u"a__b\"__c": "true"})),
     ])
     def test_contains_query_generated(self, filter_string, q_expected):
-        q = DynamicFilterField.filter_string_to_q(filter_string)
+        q = DynamicFilterQuerySet.query_from_string(filter_string)
         assert unicode(q) == unicode(q_expected)
 
     @pytest.mark.parametrize("filter_string,q_expected", [
@@ -85,7 +84,7 @@ class TestDynamicFilterFieldFilterStringToQ():
         #('a__b\"__c="true"', Q(**{u"a__b\"__c": "true"})),
     ])
     def test_contains_query_generated_unicode(self, filter_string, q_expected):
-        q = DynamicFilterField.filter_string_to_q(filter_string)
+        q = DynamicFilterQuerySet.query_from_string(filter_string)
         assert unicode(q) == unicode(q_expected)
 
     @pytest.mark.parametrize("filter_string,q_expected", [
@@ -93,7 +92,7 @@ class TestDynamicFilterFieldFilterStringToQ():
         ('ansible_facts__c="null"', Q(**{u"ansible_facts__contains": {u"c": u"\"null\""}})),
     ])
     def test_contains_query_generated_null(self, filter_string, q_expected):
-        q = DynamicFilterField.filter_string_to_q(filter_string)
+        q = DynamicFilterQuerySet.query_from_string(filter_string)
         assert unicode(q) == unicode(q_expected)