coarse json queries to use gin index

awx/main/fields.py

@@ -343,7 +343,7 @@ def string_to_type(t):
 
 
 class DynamicFilterField(models.TextField):
-
+    SEARCHABLE_RELATIONSHIP = 'ansible_facts'
 
     class BoolOperand(object):
         def __init__(self, t):
@@ -353,6 +353,16 @@ class DynamicFilterField(models.TextField):
             kwargs[k] = v
             self.result = Q(**kwargs)
 
+        def strip_quotes_traditional_logic(self, v):
+            if type(v) is unicode and v.startswith('"') and v.endswith('"'):
+                return v[1:-1]
+            return v
+
+        def strip_quotes_json_logic(self, v):
+            if type(v) is unicode and v.startswith('"') and v.endswith('"') and v != u'"null"':
+                return v[1:-1]
+            return v
+
         '''
         TODO: We should be able to express this in the grammar and let 
               pyparsing do the heavy lifting.
@@ -362,66 +372,50 @@ class DynamicFilterField(models.TextField):
               relationship refered to to see if it's a jsonb type.
         '''
         def _json_path_to_contains(self, k, v):
-            pieces = k.split('__')
+            if not k.startswith(DynamicFilterField.SEARCHABLE_RELATIONSHIP):
+                v = self.strip_quotes_traditional_logic(v)
+                return (k, v)
 
-            flag_first_arr_found = False
+            # Strip off leading relationship key
+            if k.startswith(DynamicFilterField.SEARCHABLE_RELATIONSHIP + '__'):
+                strip_len = len(DynamicFilterField.SEARCHABLE_RELATIONSHIP) + 2
+            else:
+                strip_len = len(DynamicFilterField.SEARCHABLE_RELATIONSHIP)
+            k = k[strip_len:]
 
-            assembled_k = ''
+            pieces = k.split(u'__')
-            assembled_v = v
+
+            assembled_k = u'%s__contains' % (DynamicFilterField.SEARCHABLE_RELATIONSHIP)
+            assembled_v = None
 
-            last_kv = None
             last_v = None
+            last_kv = None
 
-            contains_count = 0
             for i, piece in enumerate(pieces):
-                if flag_first_arr_found is False and piece.endswith('[]'):
+                new_kv = dict()
-                    assembled_k += u'%s__contains' % (piece[0:-2])
+                if piece.endswith(u'[]'):
-                    contains_count += 1
+                    new_v = []
-                    flag_first_arr_found = True
+                    new_kv[piece[0:-2]] = new_v
-                elif flag_first_arr_found is False and i == len(pieces) - 1:
+                else:
-                    assembled_k += u'%s' % piece
+                    new_v = dict()
-                elif flag_first_arr_found is False:
+                    new_kv[piece] = new_v
-                    assembled_k += u'%s__' % piece
-                elif flag_first_arr_found is True:
-                    new_kv = dict()
-                    if piece.endswith('[]'):
-                        new_v = []
-                        new_kv[piece[0:-2]] = new_v
-                    else:
-                        new_v = dict()
-                        new_kv[piece] = new_v
 
+                if last_kv is None:
+                    assembled_v = new_kv
+                elif type(last_v) is list:
+                    last_v.append(new_kv)
+                elif type(last_v) is dict:
+                    last_kv[last_kv.keys()[0]] = new_kv
 
-                    if last_v is None:
+                last_v = new_v
-                        last_v = []
+                last_kv = new_kv
-                        assembled_v = last_v
 
-                    if type(last_v) is list:
+            v = self.strip_quotes_json_logic(v)
-                        last_v.append(new_kv)
-                    elif type(last_v) is dict:
-                        last_kv[last_kv.keys()[0]] = new_kv
 
-                    last_v = new_v
+            if type(last_v) is list:
-                    last_kv = new_kv
+                last_v.append(v)
-                    contains_count += 1
+            elif type(last_v) is dict:
-
+                last_kv[last_kv.keys()[0]] = v
-            '''
-            Explicit quotes are kept until this point.
-            Note: we could have totally "ripped" them off earlier when we decided
-            what type to convert the token to.
-            '''
-            if type(v) is unicode and v.startswith('"') and v.endswith('"') and v != u'"null"':
-                v = v[1:-1]
-
-            if contains_count == 0:
-                assembled_v = v
-            elif contains_count == 1:
-                assembled_v = [v]
-            elif contains_count > 1:
-                if type(last_v) is list:
-                    last_v.append(v)
-                if type(last_v) is dict:
-                    last_kv[last_kv.keys()[0]] = v
 
             return (assembled_k, assembled_v)
 
@@ -517,6 +511,7 @@ class DynamicFilterField(models.TextField):
 
         try:
             res = boolExpr.parseString('(' + filter_string + ')')
+        #except ParseException as e:
         except Exception:
             raise RuntimeError(u"Invalid query %s" % filter_string_raw)
 

awx/main/tests/unit/test_fields.py

@@ -9,6 +9,7 @@ from awx.main.fields import DynamicFilterField
 from django.db.models import Q
 
 
+
 class TestDynamicFilterFieldFilterStringToQ():
     @pytest.mark.parametrize("filter_string,q_expected", [
         ('facts__facts__blank=""', Q(**{u"facts__facts__blank": u""})),
@@ -18,7 +19,7 @@ class TestDynamicFilterFieldFilterStringToQ():
         ('a__b__c=3.14', Q(**{u"a__b__c": 3.14})),
         ('a__b__c=true', Q(**{u"a__b__c": True})),
         ('a__b__c=false', Q(**{u"a__b__c": False})),
-        ('a__b__c="true"', Q(**{u"a__b__c": u"true"})),
+        ('ansible_facts__a="true"', Q(**{u"ansible_facts__contains": {u"a": u"true"}})),
         #('"a__b\"__c"="true"', Q(**{u"a__b\"__c": "true"})),
         #('a__b\"__c="true"', Q(**{u"a__b\"__c": "true"})),
     ])
@@ -27,8 +28,8 @@ class TestDynamicFilterFieldFilterStringToQ():
         assert unicode(q) == unicode(q_expected)
 
     @pytest.mark.parametrize("filter_string", [
-        'facts__facts__blank='
+        'ansible_facts__facts__facts__blank='
-        'a__b__c__ space  =ggg',
+        'ansible_facts__a__b__c__ space  =ggg',
     ])
     def test_invalid_filter_strings(self, filter_string):
         with pytest.raises(RuntimeError) as e:
@@ -37,6 +38,7 @@ class TestDynamicFilterFieldFilterStringToQ():
 
     @pytest.mark.parametrize("filter_string,q_expected", [
         (u'(a=abc\u1F5E3def)', Q(**{u"a": u"abc\u1F5E3def"})),
+        (u'(ansible_facts__a=abc\u1F5E3def)', Q(**{u"ansible_facts__contains": {u"a": u"abc\u1F5E3def"}})),
     ])
     def test_unicode(self, filter_string, q_expected):
         q = DynamicFilterField.filter_string_to_q(filter_string)
@@ -57,18 +59,18 @@ class TestDynamicFilterFieldFilterStringToQ():
         assert unicode(q) == unicode(q_expected)
 
     @pytest.mark.parametrize("filter_string,q_expected", [
-        ('a__b__c[]=3', Q(**{u"a__b__c__contains": [3]})),
+        ('ansible_facts__a__b__c[]=3', Q(**{u"ansible_facts__contains": {u"a": {u"b": {u"c": [3]}}}})),
-        ('a__b__c[]=3.14', Q(**{u"a__b__c__contains": [3.14]})),
+        ('ansible_facts__a__b__c[]=3.14', Q(**{u"ansible_facts__contains": {u"a": {u"b": {u"c": [3.14]}}}})),
-        ('a__b__c[]=true', Q(**{u"a__b__c__contains": [True]})),
+        ('ansible_facts__a__b__c[]=true', Q(**{u"ansible_facts__contains": {u"a": {u"b": {u"c": [True]}}}})),
-        ('a__b__c[]=false', Q(**{u"a__b__c__contains": [False]})),
+        ('ansible_facts__a__b__c[]=false', Q(**{u"ansible_facts__contains": {u"a": {u"b": {u"c": [False]}}}})),
-        ('a__b__c[]="true"', Q(**{u"a__b__c__contains": [u"true"]})),
+        ('ansible_facts__a__b__c[]="true"', Q(**{u"ansible_facts__contains": {u"a": {u"b": {u"c": [u"true"]}}}})),
-        ('a__b__c[]="hello world"', Q(**{u"a__b__c__contains": [u"hello world"]})),
+        ('ansible_facts__a__b__c[]="hello world"', Q(**{u"ansible_facts__contains": {u"a": {u"b": {u"c": [u"hello world"]}}}})),
-        ('a__b__c[]__d[]="foobar"', Q(**{u"a__b__c__contains": [{u"d": [u"foobar"]}]})),
+        ('ansible_facts__a__b__c[]__d[]="foobar"', Q(**{u"ansible_facts__contains": {u"a": {u"b": {u"c": [{u"d": [u"foobar"]}]}}}})),
-        ('a__b__c[]__d="foobar"', Q(**{u"a__b__c__contains": [{u"d": u"foobar"}]})),
+        ('ansible_facts__a__b__c[]__d="foobar"', Q(**{u"ansible_facts__contains": {u"a": {u"b": {u"c": [{u"d": u"foobar"}]}}}})),
-        ('a__b__c[]__d__e="foobar"', Q(**{u"a__b__c__contains": [{u"d": {u"e": u"foobar"}}]})),
+        ('ansible_facts__a__b__c[]__d__e="foobar"', Q(**{u"ansible_facts__contains": {u"a": {u"b": {u"c": [{u"d": {u"e": u"foobar"}}]}}}})),
-        ('a__b__c[]__d__e[]="foobar"', Q(**{u"a__b__c__contains": [{u"d": {u"e": [u"foobar"]}}]})),
+        ('ansible_facts__a__b__c[]__d__e[]="foobar"', Q(**{u"ansible_facts__contains": {u"a": {u"b": {u"c": [{u"d": {u"e": [u"foobar"]}}]}}}})),
-        ('a__b__c[]__d__e__f[]="foobar"', Q(**{u"a__b__c__contains": [{u"d": {u"e": {u"f": [u"foobar"]}}}]})),
+        ('ansible_facts__a__b__c[]__d__e__f[]="foobar"', Q(**{u"ansible_facts__contains": {u"a": {u"b": {u"c": [{u"d": {u"e": {u"f": [u"foobar"]}}}]}}}})),
-        ('(a__b__c[]__d__e__f[]="foobar") and (a__b__c[]__d__e[]="foobar")', Q(**{ u"a__b__c__contains": [{u"d": {u"e": {u"f": [u"foobar"]}}}]}) & Q(**{u"a__b__c__contains": [{u"d": {u"e": [u"foobar"]}}]})),
+        ('(ansible_facts__a__b__c[]__d__e__f[]="foobar") and (ansible_facts__a__b__c[]__d__e[]="foobar")', Q(**{ u"ansible_facts__contains": {u"a": {u"b": {u"c": [{u"d": {u"e": {u"f": [u"foobar"]}}}]}}}}) & Q(**{u"ansible_facts__contains": {u"a": {u"b": {u"c": [{u"d": {u"e": [u"foobar"]}}]}}}})),
         #('"a__b\"__c"="true"', Q(**{u"a__b\"__c": "true"})),
         #('a__b\"__c="true"', Q(**{u"a__b\"__c": "true"})),
     ])
@@ -78,7 +80,7 @@ class TestDynamicFilterFieldFilterStringToQ():
 
     @pytest.mark.parametrize("filter_string,q_expected", [
         #('a__b__c[]="true"', Q(**{u"a__b__c__contains": u"\"true\""})),
-        ('a__b__c="true"', Q(**{u"a__b__c": u"true"})),
+        ('ansible_facts__a="true"', Q(**{u"ansible_facts__contains": {u"a": u"true"}})),
         #('"a__b\"__c"="true"', Q(**{u"a__b\"__c": "true"})),
         #('a__b\"__c="true"', Q(**{u"a__b\"__c": "true"})),
     ])
@@ -87,8 +89,8 @@ class TestDynamicFilterFieldFilterStringToQ():
         assert unicode(q) == unicode(q_expected)
 
     @pytest.mark.parametrize("filter_string,q_expected", [
-        ('a__b__c=null', Q(**{u"a__b__c": u"null"})),
+        ('ansible_facts__a=null', Q(**{u"ansible_facts__contains": {u"a": u"null"}})),
-        ('a__b__c="null"', Q(**{u"a__b__c": u"\"null\""})),
+        ('ansible_facts__c="null"', Q(**{u"ansible_facts__contains": {u"c": u"\"null\""}})),
     ])
     def test_contains_query_generated_null(self, filter_string, q_expected):
         q = DynamicFilterField.filter_string_to_q(filter_string)