Merge pull request #6144 from wwitzel3/issue-5745

Support dynamic Inventory

awx/api/serializers.py

@@ -45,6 +45,8 @@ from awx.main.fields import ImplicitRoleField
 from awx.main.utils import (
     get_type_for_model, get_model_for_type, timestamp_apiformat,
     camelcase_to_underscore, getattrd, parse_yaml_or_json)
+from awx.main.utils.filters import DynamicFilter
+
 from awx.main.validators import vars_validate_or_raise
 
 from awx.conf.license import feature_enabled
@@ -1104,7 +1106,7 @@ class InventorySerializer(BaseSerializerWithVariables):
 
     class Meta:
         model = Inventory
-        fields = ('*', 'organization', 'variables', 'has_active_failures',
+        fields = ('*', 'organization', 'kind', 'host_filter', 'variables', 'has_active_failures',
                   'total_hosts', 'hosts_with_active_failures', 'total_groups',
                   'groups_with_active_failures', 'has_inventory_sources',
                   'total_inventory_sources', 'inventory_sources_with_failures')
@@ -1136,6 +1138,17 @@ class InventorySerializer(BaseSerializerWithVariables):
             ret['organization'] = None
         return ret
 
+    def validate(self, attrs):
+        kind = attrs.get('kind', 'standard')
+        if kind == 'dynamic':
+            host_filter = attrs.get('host_filter')
+            if host_filter is not None:
+                try:
+                    DynamicFilter().query_from_string(host_filter)
+                except RuntimeError, e:
+                    raise models.base.ValidationError(e)
+        return super(InventorySerializer, self).validate(attrs)
+
 
 class InventoryDetailSerializer(InventorySerializer):
 

awx/api/views.py

@@ -72,6 +72,8 @@ from awx.main.utils import * # noqa
 from awx.main.utils import (
     callback_filter_out_ansible_extra_vars
 )
+from awx.main.utils.filters import DynamicFilter
+
 from awx.api.permissions import * # noqa
 from awx.api.renderers import * # noqa
 from awx.api.serializers import * # noqa
@@ -79,7 +81,6 @@ from awx.api.metadata import RoleMetadata
 from awx.main.consumers import emit_channel_notification
 from awx.main.models.unified_jobs import ACTIVE_STATES
 from awx.main.scheduler.tasks import run_job_complete
-from awx.main.fields import DynamicFilterField
 
 logger = logging.getLogger('awx.api.views')
 
@@ -1685,6 +1686,15 @@ class InventoryDetail(RetrieveUpdateDestroyAPIView):
     model = Inventory
     serializer_class = InventoryDetailSerializer
 
+    def update(self, request, *args, **kwargs):
+        obj = self.get_object()
+        kind = self.request.data.get('kind') or kwargs.get('kind')
+
+        # Do not allow changes to an Inventory kind.
+        if kind is not None and obj.kind != kind:
+            return self.http_method_not_allowed(request, *args, **kwargs)
+        return super(InventoryDetail, self).update(request, *args, **kwargs)
+
     def destroy(self, request, *args, **kwargs):
         with ignore_inventory_computed_fields():
             with ignore_inventory_group_removal():
@@ -1767,7 +1777,7 @@ class HostList(ListCreateAPIView):
         qs = super(HostList, self).get_queryset()
         filter_string = self.request.query_params.get('host_filter', None)
         if filter_string:
-            filter_q = DynamicFilterField.filter_string_to_q(filter_string)
+            filter_q = DynamicFilter.query_from_string(filter_string)
             qs = qs.filter(filter_q)
         return qs
 

awx/main/fields.py

@@ -4,10 +4,7 @@
 # Python
 import copy
 import json
-import re
-import sys
 import six
-from pyparsing import infixNotation, opAssoc, Optional, Literal, CharsNotIn
 
 from jinja2 import Environment, StrictUndefined
 from jinja2.exceptions import UndefinedError
@@ -28,7 +25,6 @@ from django.db.models.fields.related import (
     ReverseManyRelatedObjectsDescriptor,
 )
 from django.utils.encoding import smart_text
-from django.db.models import Q
 from django.utils.translation import ugettext_lazy as _
 
 # jsonschema
@@ -39,6 +35,7 @@ from jsonfield import JSONField as upstream_JSONField
 from jsonbfield.fields import JSONField as upstream_JSONBField
 
 # AWX
+from awx.main.utils.filters import DynamicFilter
 from awx.main.models.rbac import batch_role_ancestor_rebuilding, Role
 from awx.main import utils
 
@@ -332,202 +329,17 @@ class ImplicitRoleField(models.ForeignKey):
         Role.rebuild_role_ancestor_list([], child_ids)
 
 
-unicode_spaces = [unichr(c) for c in xrange(sys.maxunicode) if unichr(c).isspace()]
-unicode_spaces_other = unicode_spaces + [u'(', u')', u'=', u'"']
-
-
-def string_to_type(t):
-    if t == u'true':
-        return True
-    elif t == u'false':
-        return False
-
-    if re.search('^[-+]?[0-9]+$',t):
-        return int(t)
-
-    if re.search('^[-+]?[0-9]+\.[0-9]+$',t):
-        return float(t)
-
-    return t
-
-
 class DynamicFilterField(models.TextField):
-    SEARCHABLE_RELATIONSHIP = 'ansible_facts'
+    def get_prep_value(self, value):
-
+        # Change any false value to none.
-    class BoolOperand(object):
+        # https://docs.python.org/2/library/stdtypes.html#truth-value-testing
-        def __init__(self, t):
+        if not value:
-            kwargs = dict()
+            return None
-            k, v = self._extract_key_value(t)
-            k, v = self._json_path_to_contains(k, v)
-            kwargs[k] = v
-            self.result = Q(**kwargs)
-
-        def strip_quotes_traditional_logic(self, v):
-            if type(v) is unicode and v.startswith('"') and v.endswith('"'):
-                return v[1:-1]
-            return v
-
-        def strip_quotes_json_logic(self, v):
-            if type(v) is unicode and v.startswith('"') and v.endswith('"') and v != u'"null"':
-                return v[1:-1]
-            return v
-
-        '''
-        TODO: We should be able to express this in the grammar and let 
-              pyparsing do the heavy lifting.
-        TODO: separate django filter requests from our custom json filter
-              request so we don't process the key any. This could be
-              accomplished using a whitelist or introspecting the
-              relationship refered to to see if it's a jsonb type.
-        '''
-        def _json_path_to_contains(self, k, v):
-            if not k.startswith(DynamicFilterField.SEARCHABLE_RELATIONSHIP):
-                v = self.strip_quotes_traditional_logic(v)
-                return (k, v)
-
-            # Strip off leading relationship key
-            if k.startswith(DynamicFilterField.SEARCHABLE_RELATIONSHIP + '__'):
-                strip_len = len(DynamicFilterField.SEARCHABLE_RELATIONSHIP) + 2
-            else:
-                strip_len = len(DynamicFilterField.SEARCHABLE_RELATIONSHIP)
-            k = k[strip_len:]
-
-            pieces = k.split(u'__')
-
-            assembled_k = u'%s__contains' % (DynamicFilterField.SEARCHABLE_RELATIONSHIP)
-            assembled_v = None
-
-            last_v = None
-            last_kv = None
-
-            for i, piece in enumerate(pieces):
-                new_kv = dict()
-                if piece.endswith(u'[]'):
-                    new_v = []
-                    new_kv[piece[0:-2]] = new_v
-                else:
-                    new_v = dict()
-                    new_kv[piece] = new_v
-
-                if last_kv is None:
-                    assembled_v = new_kv
-                elif type(last_v) is list:
-                    last_v.append(new_kv)
-                elif type(last_v) is dict:
-                    last_kv[last_kv.keys()[0]] = new_kv
-
-                last_v = new_v
-                last_kv = new_kv
-
-            v = self.strip_quotes_json_logic(v)
-
-            if type(last_v) is list:
-                last_v.append(v)
-            elif type(last_v) is dict:
-                last_kv[last_kv.keys()[0]] = v
-
-            return (assembled_k, assembled_v)
-
-        def _extract_key_value(self, t):
-            t_len = len(t)
-
-            k = None
-            v = None
-
-            # key
-            # "something"=
-            v_offset = 2
-            if t_len >= 2 and t[0] == "\"" and t[2] == "\"":
-                k = t[1]
-                v_offset = 4
-            # something=
-            else:
-                k = t[0]
-
-            # value
-            # ="something"
-            if t_len > (v_offset + 2) and t[v_offset] == "\"" and t[v_offset + 2] == "\"":
-                v = u'"' + unicode(t[v_offset + 1]) + u'"'
-                #v = t[v_offset + 1]
-            # empty ""
-            elif t_len > (v_offset + 1):
-                v = u""
-            # no ""
-            else:
-                v = string_to_type(t[v_offset])
-
-            return (k, v)
-
-
-    class BoolBinOp(object):
-        def __init__(self, t):
-            self.result = None
-            i = 2
-            while i < len(t[0]):
-                if not self.result:
-                    self.result = t[0][0].result
-                right = t[0][i].result
-                self.result = self.execute_logic(self.result, right)
-                i += 2
-
-
-    class BoolAnd(BoolBinOp):
-        def execute_logic(self, left, right):
-            return left & right
-
-
-    class BoolOr(BoolBinOp):
-        def execute_logic(self, left, right):
-            return left | right
-
-
-    class BoolNot(object):
-        def __init__(self, t):
-            self.right = t[0][1].result
-
-            self.result = self.execute_logic(self.right)
-
-        def execute_logic(self, right):
-            return ~right
-
-
-    @classmethod
-    def filter_string_to_q(cls, filter_string):
-
-        '''
-        TODO:
-        * handle values with " via: a.b.c.d="hello\"world"
-        * handle keys with " via: a.\"b.c="yeah"
-        * handle key with __ in it
-
-        '''
-        filter_string_raw = filter_string
-        filter_string = unicode(filter_string)
-
-        atom = CharsNotIn(unicode_spaces_other)
-        atom_inside_quotes = CharsNotIn(u'"')
-        atom_quoted = Literal('"') + Optional(atom_inside_quotes) + Literal('"')
-        EQUAL = Literal('=')
-
-        grammar = ((atom_quoted | atom) + EQUAL + Optional((atom_quoted | atom)))
-        grammar.setParseAction(cls.BoolOperand)
-
-        boolExpr = infixNotation(grammar, [
-            ("not", 1, opAssoc.RIGHT, cls.BoolNot),
-            ("and", 2, opAssoc.LEFT, cls.BoolAnd),
-            ("or",  2, opAssoc.LEFT, cls.BoolOr),
-        ])
-
         try:
-            res = boolExpr.parseString('(' + filter_string + ')')
+            DynamicFilter().query_from_string(value)
-        #except ParseException as e:
+        except RuntimeError, e:
-        except Exception:
+            raise models.base.ValidationError(e)
-            raise RuntimeError(u"Invalid query %s" % filter_string_raw)
+        return super(DynamicFilterField, self).get_prep_value(value)
-
-        if len(res) > 0:
-            return res[0].result
-
-        raise RuntimeError("Parsing the filter_string %s went terribly wrong" % filter_string)
 
 
 class JSONSchemaField(JSONBField):

awx/main/managers.py

@@ -9,6 +9,10 @@ from django.utils.timezone import now
 from django.db.models import Sum
 from django.conf import settings
 
+from awx.main.utils.filters import DynamicFilter
+
+___all__ = ['HostManager', 'InstanceManager']
+
 
 class HostManager(models.Manager):
     """Custom manager class for Hosts model."""
@@ -20,6 +24,26 @@ class HostManager(models.Manager):
         except NotImplementedError: # For unit tests only, SQLite doesn't support distinct('name')
             return len(set(self.values_list('name', flat=True)))
 
+    def get_queryset(self):
+        """When the parent instance of the host query set has a `kind` of dynamic and a `host_filter`
+        set. Use the `host_filter` to generate the queryset for the hosts.
+        """
+        qs = super(HostManager, self).get_queryset()
+        if (hasattr(self, 'instance') and
+           hasattr(self.instance, 'host_filter') and
+           hasattr(self.instance, 'kind')):
+            if self.instance.kind == 'dynamic' and self.instance.host_filter is not None:
+                    q = DynamicFilter.query_from_string(self.instance.host_filter)
+                    # If we are using host_filters, disable the core_filters, this allows
+                    # us to access all of the available Host entries, not just the ones associated
+                    # with a specific FK/relation.
+                    #
+                    # If we don't disable this, a filter of {'inventory': self.instance} gets automatically
+                    # injected by the related object mapper.
+                    self.core_filters = {}
+                    return qs.filter(q)
+        return qs
+
 
 class InstanceManager(models.Manager):
     """A custom manager class for the Instance model.

awx/main/migrations/0038_v320_release.py

@@ -36,6 +36,16 @@ class Migration(migrations.Migration):
             name='inventory',
             field=models.ForeignKey(related_name='inventory_sources', default=None, to='main.Inventory', null=True),
         ),
+        migrations.AddField(
+            model_name='inventory',
+            name='host_filter',
+            field=awx.main.fields.DynamicFilterField(default=None, help_text='Filter that will be applied to the hosts of this inventory.', null=True, blank=True),
+        ),
+        migrations.AddField(
+            model_name='inventory',
+            name='kind',
+            field=models.CharField(default=b'standard', help_text='Kind of inventory being represented.', max_length=32, choices=[(b'standard', 'Hosts have a direct link to this inventory.'), (b'dynamic', 'Hosts for inventory generated using the host_filter property.')]),
+        ),
 
         # Facts
         migrations.AlterField(

awx/main/models/inventory.py

@@ -20,7 +20,11 @@ from django.utils.timezone import now
 # AWX
 from awx.api.versioning import reverse
 from awx.main.constants import CLOUD_PROVIDERS
-from awx.main.fields import ImplicitRoleField, JSONBField
+from awx.main.fields import (
+    ImplicitRoleField,
+    JSONBField,
+    DynamicFilterField,
+)
 from awx.main.managers import HostManager
 from awx.main.models.base import * # noqa
 from awx.main.models.unified_jobs import * # noqa
@@ -42,6 +46,11 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin):
     an inventory source contains lists and hosts.
     '''
 
+    KIND_CHOICES = [
+        ('standard', _('Hosts have a direct link to this inventory.')),
+        ('dynamic', _('Hosts for inventory generated using the host_filter property.')),
+    ]
+
     class Meta:
         app_label = 'main'
         verbose_name_plural = _('inventories')
@@ -99,6 +108,20 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin):
         editable=False,
         help_text=_('Number of external inventory sources in this inventory with failures.'),
     )
+    kind = models.CharField(
+        max_length=32,
+        choices=KIND_CHOICES,
+        blank=False,
+        default='standard',
+        help_text=_('Kind of inventory being represented.'),
+    )
+    host_filter = DynamicFilterField(
+        blank=True,
+        null=True,
+        default=None,
+        help_text=_('Filter that will be applied to the hosts of this inventory.'),
+    )
+
     admin_role = ImplicitRoleField(
         parent_role='organization.admin_role',
     )

awx/main/tests/functional/models/test_inventory.py

@@ -2,7 +2,11 @@ import pytest
 import mock
 
 # AWX
-from awx.main.models import InventorySource, InventoryUpdate
+from awx.main.models import (
+    Inventory,
+    InventorySource,
+    InventoryUpdate,
+)
 
 
 @pytest.mark.django_db
@@ -31,3 +35,56 @@ class TestSCMUpdateFeatures:
             scm_inventory_source.description = "I'm testing this!"
             scm_inventory_source.save()
             assert not mck_update.called
+
+
+@pytest.fixture
+def setup_ec2_gce(organization):
+    ec2_inv = Inventory(name='test_ec2', organization=organization)
+    ec2_inv.save()
+
+    ec2_source = ec2_inv.inventory_sources.create(name='test_ec2_source', source='ec2')
+    for i in range(2):
+        ec2_host = ec2_inv.hosts.create(name='test_ec2_{0}'.format(i))
+        ec2_host.inventory_sources.add(ec2_source)
+    ec2_inv.save()
+
+    gce_inv = Inventory(name='test_gce', organization=organization)
+    gce_inv.save()
+
+    gce_source = gce_inv.inventory_sources.create(name='test_gce_source', source='gce')
+    gce_host = gce_inv.hosts.create(name='test_gce_host')
+    gce_host.inventory_sources.add(gce_source)
+    gce_inv.save()
+
+
+@pytest.mark.django_db
+class TestHostManager:
+    def test_host_filter_change(self, setup_ec2_gce, organization):
+        dynamic_inventory = Inventory(name='dynamic',
+                                      kind='dynamic',
+                                      organization=organization,
+                                      host_filter='inventory_sources__source=ec2')
+        dynamic_inventory.save()
+        assert len(dynamic_inventory.hosts.all()) == 2
+
+        dynamic_inventory.host_filter = 'inventory_sources__source=gce'
+        dynamic_inventory.save()
+        assert len(dynamic_inventory.hosts.all()) == 1
+
+    def test_host_filter_not_dynamic(self, setup_ec2_gce, organization):
+        dynamic_inventory = Inventory(name='dynamic',
+                                      organization=organization,
+                                      host_filter='inventory_sources__source=ec2')
+        assert len(dynamic_inventory.hosts.all()) == 0
+
+    def test_host_objects_manager(self, setup_ec2_gce, organization):
+        dynamic_inventory = Inventory(kind='dynamic',
+                                      name='dynamic',
+                                      organization=organization,
+                                      host_filter='inventory_sources__source=ec2')
+        dynamic_inventory.save()
+
+        hosts = dynamic_inventory.hosts.all()
+        assert len(hosts) == 2
+        assert hosts[0].inventory_sources.first().source == 'ec2'
+        assert hosts[1].inventory_sources.first().source == 'ec2'

awx/main/tests/unit/utils/test_filters.py

@@ -3,14 +3,13 @@
 import pytest
 
 # AWX
-from awx.main.fields import DynamicFilterField
+from awx.main.utils.filters import DynamicFilter
 
 # Django
 from django.db.models import Q
 
 
-
+class TestDynamicFilterQueryFromString():
-class TestDynamicFilterFieldFilterStringToQ():
     @pytest.mark.parametrize("filter_string,q_expected", [
         ('facts__facts__blank=""', Q(**{u"facts__facts__blank": u""})),
         ('"facts__facts__ space "="f"', Q(**{u"facts__facts__ space ": u"f"})),
@@ -24,7 +23,7 @@ class TestDynamicFilterFieldFilterStringToQ():
         #('a__b\"__c="true"', Q(**{u"a__b\"__c": "true"})),
     ])
     def test_query_generated(self, filter_string, q_expected):
-        q = DynamicFilterField.filter_string_to_q(filter_string)
+        q = DynamicFilter.query_from_string(filter_string)
         assert unicode(q) == unicode(q_expected)
 
     @pytest.mark.parametrize("filter_string", [
@@ -33,7 +32,7 @@ class TestDynamicFilterFieldFilterStringToQ():
     ])
     def test_invalid_filter_strings(self, filter_string):
         with pytest.raises(RuntimeError) as e:
-            DynamicFilterField.filter_string_to_q(filter_string)
+            DynamicFilter.query_from_string(filter_string)
         assert e.value.message == u"Invalid query " + filter_string
 
     @pytest.mark.parametrize("filter_string,q_expected", [
@@ -41,7 +40,7 @@ class TestDynamicFilterFieldFilterStringToQ():
         (u'(ansible_facts__a=abc\u1F5E3def)', Q(**{u"ansible_facts__contains": {u"a": u"abc\u1F5E3def"}})),
     ])
     def test_unicode(self, filter_string, q_expected):
-        q = DynamicFilterField.filter_string_to_q(filter_string)
+        q = DynamicFilter.query_from_string(filter_string)
         assert unicode(q) == unicode(q_expected)
 
     @pytest.mark.parametrize("filter_string,q_expected", [
@@ -55,7 +54,7 @@ class TestDynamicFilterFieldFilterStringToQ():
         ('a=b or a=d or a=e or a=z and b=h and b=i and b=j and b=k', Q(**{u"a": u"b"}) | Q(**{u"a": u"d"}) | Q(**{u"a": u"e"}) | Q(**{u"a": u"z"}) & Q(**{u"b": u"h"}) & Q(**{u"b": u"i"}) & Q(**{u"b": u"j"}) & Q(**{u"b": u"k"}))
     ])
     def test_boolean_parenthesis(self, filter_string, q_expected):
-        q = DynamicFilterField.filter_string_to_q(filter_string)
+        q = DynamicFilter.query_from_string(filter_string)
         assert unicode(q) == unicode(q_expected)
 
     @pytest.mark.parametrize("filter_string,q_expected", [
@@ -75,7 +74,7 @@ class TestDynamicFilterFieldFilterStringToQ():
         #('a__b\"__c="true"', Q(**{u"a__b\"__c": "true"})),
     ])
     def test_contains_query_generated(self, filter_string, q_expected):
-        q = DynamicFilterField.filter_string_to_q(filter_string)
+        q = DynamicFilter.query_from_string(filter_string)
         assert unicode(q) == unicode(q_expected)
 
     @pytest.mark.parametrize("filter_string,q_expected", [
@@ -85,7 +84,7 @@ class TestDynamicFilterFieldFilterStringToQ():
         #('a__b\"__c="true"', Q(**{u"a__b\"__c": "true"})),
     ])
     def test_contains_query_generated_unicode(self, filter_string, q_expected):
-        q = DynamicFilterField.filter_string_to_q(filter_string)
+        q = DynamicFilter.query_from_string(filter_string)
         assert unicode(q) == unicode(q_expected)
 
     @pytest.mark.parametrize("filter_string,q_expected", [
@@ -93,7 +92,7 @@ class TestDynamicFilterFieldFilterStringToQ():
         ('ansible_facts__c="null"', Q(**{u"ansible_facts__contains": {u"c": u"\"null\""}})),
     ])
     def test_contains_query_generated_null(self, filter_string, q_expected):
-        q = DynamicFilterField.filter_string_to_q(filter_string)
+        q = DynamicFilter.query_from_string(filter_string)
         assert unicode(q) == unicode(q_expected)
 
 

awx/main/utils/filters.py

@@ -0,0 +1,210 @@
+import re
+import sys
+from pyparsing import (
+    infixNotation,
+    opAssoc,
+    Optional,
+    Literal,
+    CharsNotIn,
+)
+
+from django.db import models
+
+
+__all__ = ['DynamicFilter']
+
+unicode_spaces = [unichr(c) for c in xrange(sys.maxunicode) if unichr(c).isspace()]
+unicode_spaces_other = unicode_spaces + [u'(', u')', u'=', u'"']
+
+
+def string_to_type(t):
+    if t == u'true':
+        return True
+    elif t == u'false':
+        return False
+
+    if re.search('^[-+]?[0-9]+$',t):
+        return int(t)
+
+    if re.search('^[-+]?[0-9]+\.[0-9]+$',t):
+        return float(t)
+
+    return t
+
+
+class DynamicFilter(object):
+    SEARCHABLE_RELATIONSHIP = 'ansible_facts'
+
+    class BoolOperand(object):
+        def __init__(self, t):
+            kwargs = dict()
+            k, v = self._extract_key_value(t)
+            k, v = self._json_path_to_contains(k, v)
+            kwargs[k] = v
+            self.result = models.Q(**kwargs)
+
+        def strip_quotes_traditional_logic(self, v):
+            if type(v) is unicode and v.startswith('"') and v.endswith('"'):
+                return v[1:-1]
+            return v
+
+        def strip_quotes_json_logic(self, v):
+            if type(v) is unicode and v.startswith('"') and v.endswith('"') and v != u'"null"':
+                return v[1:-1]
+            return v
+
+        '''
+        TODO: We should be able to express this in the grammar and let
+              pyparsing do the heavy lifting.
+        TODO: separate django filter requests from our custom json filter
+              request so we don't process the key any. This could be
+              accomplished using a whitelist or introspecting the
+              relationship refered to to see if it's a jsonb type.
+        '''
+        def _json_path_to_contains(self, k, v):
+            if not k.startswith(DynamicFilter.SEARCHABLE_RELATIONSHIP):
+                v = self.strip_quotes_traditional_logic(v)
+                return (k, v)
+
+            # Strip off leading relationship key
+            if k.startswith(DynamicFilter.SEARCHABLE_RELATIONSHIP + '__'):
+                strip_len = len(DynamicFilter.SEARCHABLE_RELATIONSHIP) + 2
+            else:
+                strip_len = len(DynamicFilter.SEARCHABLE_RELATIONSHIP)
+            k = k[strip_len:]
+
+            pieces = k.split(u'__')
+
+            assembled_k = u'%s__contains' % (DynamicFilter.SEARCHABLE_RELATIONSHIP)
+            assembled_v = None
+
+            last_v = None
+            last_kv = None
+
+            for i, piece in enumerate(pieces):
+                new_kv = dict()
+                if piece.endswith(u'[]'):
+                    new_v = []
+                    new_kv[piece[0:-2]] = new_v
+                else:
+                    new_v = dict()
+                    new_kv[piece] = new_v
+
+                if last_kv is None:
+                    assembled_v = new_kv
+                elif type(last_v) is list:
+                    last_v.append(new_kv)
+                elif type(last_v) is dict:
+                    last_kv[last_kv.keys()[0]] = new_kv
+
+                last_v = new_v
+                last_kv = new_kv
+
+            v = self.strip_quotes_json_logic(v)
+
+            if type(last_v) is list:
+                last_v.append(v)
+            elif type(last_v) is dict:
+                last_kv[last_kv.keys()[0]] = v
+
+            return (assembled_k, assembled_v)
+
+        def _extract_key_value(self, t):
+            t_len = len(t)
+
+            k = None
+            v = None
+
+            # key
+            # "something"=
+            v_offset = 2
+            if t_len >= 2 and t[0] == "\"" and t[2] == "\"":
+                k = t[1]
+                v_offset = 4
+            # something=
+            else:
+                k = t[0]
+
+            # value
+            # ="something"
+            if t_len > (v_offset + 2) and t[v_offset] == "\"" and t[v_offset + 2] == "\"":
+                v = u'"' + unicode(t[v_offset + 1]) + u'"'
+                #v = t[v_offset + 1]
+            # empty ""
+            elif t_len > (v_offset + 1):
+                v = u""
+            # no ""
+            else:
+                v = string_to_type(t[v_offset])
+
+            return (k, v)
+
+
+    class BoolBinOp(object):
+        def __init__(self, t):
+            self.result = None
+            i = 2
+            while i < len(t[0]):
+                if not self.result:
+                    self.result = t[0][0].result
+                right = t[0][i].result
+                self.result = self.execute_logic(self.result, right)
+                i += 2
+
+
+    class BoolAnd(BoolBinOp):
+        def execute_logic(self, left, right):
+            return left & right
+
+
+    class BoolOr(BoolBinOp):
+        def execute_logic(self, left, right):
+            return left | right
+
+
+    class BoolNot(object):
+        def __init__(self, t):
+            self.right = t[0][1].result
+
+            self.result = self.execute_logic(self.right)
+
+        def execute_logic(self, right):
+            return ~right
+
+
+    @classmethod
+    def query_from_string(cls, filter_string):
+
+        '''
+        TODO:
+        * handle values with " via: a.b.c.d="hello\"world"
+        * handle keys with " via: a.\"b.c="yeah"
+        * handle key with __ in it
+        '''
+        filter_string_raw = filter_string
+        filter_string = unicode(filter_string)
+
+        atom = CharsNotIn(unicode_spaces_other)
+        atom_inside_quotes = CharsNotIn(u'"')
+        atom_quoted = Literal('"') + Optional(atom_inside_quotes) + Literal('"')
+        EQUAL = Literal('=')
+
+        grammar = ((atom_quoted | atom) + EQUAL + Optional((atom_quoted | atom)))
+        grammar.setParseAction(cls.BoolOperand)
+
+        boolExpr = infixNotation(grammar, [
+            ("not", 1, opAssoc.RIGHT, cls.BoolNot),
+            ("and", 2, opAssoc.LEFT, cls.BoolAnd),
+            ("or",  2, opAssoc.LEFT, cls.BoolOr),
+        ])
+
+        try:
+            res = boolExpr.parseString('(' + filter_string + ')')
+        #except ParseException as e:
+        except Exception:
+            raise RuntimeError(u"Invalid query %s" % filter_string_raw)
+
+        if len(res) > 0:
+            return res[0].result
+
+        raise RuntimeError("Parsing the filter_string %s went terribly wrong" % filter_string)