Trailing comma rework.

awx/api/parsers.py

@@ -1,6 +1,7 @@
 # Python
 from collections import OrderedDict
 import json
+import yaml
 
 # Django
 from django.conf import settings
@@ -12,24 +13,34 @@ from rest_framework import parsers
 from rest_framework.exceptions import ParseError
 
 
-def _remove_trailing_commas(data):
+class OrderedDictLoader(yaml.SafeLoader):
-    left = 0
+    """
-    right = 0
+    This yaml loader is used to deal with current pyYAML (3.12) not supporting
-    in_string = False
+    custom object pairs hook. Remove it when new version adds that support.
-    ret = []
+    """
-    while left != len(data):
+
-        if data[left] == ',' and not in_string:
+    def construct_mapping(self, node, deep=False):
-            while right != len(data) and data[right] in ',\n\t\r ':
+        if isinstance(node, yaml.nodes.MappingNode):
-                right += 1
+            self.flatten_mapping(node)
-            if right == len(data) or data[right] not in '}]':
-                ret.append(',')
         else:
-            if data[left] == '"' and (left - 1 >= 0 and data[left - 1] != '\\'):
+            raise yaml.constructor.ConstructorError(
-                in_string = not in_string
+                None, None,
-            ret.append(data[left])
+                "expected a mapping node, but found %s" % node.id,
-            right += 1
+                node.start_mark
-        left = right
+            )
-    return ''.join(ret)
+        mapping = OrderedDict()
+        for key_node, value_node in node.value:
+            key = self.construct_object(key_node, deep=deep)
+            try:
+                hash(key)
+            except TypeError, exc:
+                raise yaml.constructor.ConstructorError(
+                    "while constructing a mapping", node.start_mark,
+                    "found unacceptable key (%s)" % exc, key_node.start_mark
+                )
+            value = self.construct_object(value_node, deep=deep)
+            mapping[key] = value
+        return mapping
 
 
 class JSONParser(parsers.JSONParser):
@@ -45,10 +56,15 @@ class JSONParser(parsers.JSONParser):
         encoding = parser_context.get('encoding', settings.DEFAULT_CHARSET)
 
         try:
-            data = _remove_trailing_commas(stream.read().decode(encoding))
+            data = stream.read().decode(encoding)
             obj = json.loads(data, object_pairs_hook=OrderedDict)
             if not isinstance(obj, dict):
                 raise ParseError(_('JSON parse error - not a JSON object'))
             return obj
         except ValueError as exc:
-            raise ParseError(_('JSON parse error - %s') % six.text_type(exc))
+            try:
+                # PyYAML can also parse JSON-style input string, and support more flexible
+                # input grammar like trailing commas.
+                return yaml.load(data, OrderedDictLoader)
+            except Exception:
+                raise ParseError(_('JSON parse error - %s') % six.text_type(exc))

awx/main/tests/unit/api/test_parsers.py

@@ -1,15 +1,31 @@
 import pytest
 
-from awx.api.parsers import _remove_trailing_commas
+import StringIO
+from collections import OrderedDict
+
+from awx.api.parsers import JSONParser
 
 
 @pytest.mark.parametrize('input_, output', [
-    ('{"foo": "bar"}', '{"foo": "bar"}'),
+    ('{"foo": "bar", "alice": "bob"}', OrderedDict([("foo", "bar"), ("alice", "bob")])),
-    ('{"foo": "bar",\n\t\r }', '{"foo": "bar"}'),
+    ('{"foo": "bar", "alice": "bob",\n }', OrderedDict([("foo", "bar"), ("alice", "bob")])),
-    ('{"foo": ["alice", "bob"]}', '{"foo": ["alice","bob"]}'),
+    ('{"foo": ["alice", "bob"]}', {"foo": ["alice","bob"]}),
-    ('{"foo": ["alice", "bob",\n\t\r ]}', '{"foo": ["alice","bob"]}'),
+    ('{"foo": ["alice", "bob",\n ]}', {"foo": ["alice","bob"]}),
-    ('{"foo": "\\"bar,\n\t\r }"}', '{"foo": "\\"bar,\n\t\r }"}'),
+    ('{"foo": "\\"bar, \\n}"}', {"foo": "\"bar, \n}"}),
-    ('{"foo": ["\\"alice,\n\t\r ]", "bob"]}', '{"foo": ["\\"alice,\n\t\r ]","bob"]}'),
+    ('{"foo": ["\\"alice,\\n ]", "bob"]}', {"foo": ["\"alice,\n ]","bob"]}),
 ])
-def test_remove_trailing_commas(input_, output):
+def test_trailing_comma_support(input_, output):
-    assert _remove_trailing_commas(input_) == output
+    input_buffer = StringIO.StringIO()
+    input_buffer.write(input_)
+    input_buffer.seek(0)
+    assert JSONParser().parse(input_buffer) == output
+    input_buffer.close()
+
+
+def test_yaml_load_preserves_input_order():
+    input_ = '{"a": "b", "c": "d", "e": "f"}'
+    output = ('a', 'c', 'e')
+    input_buffer = StringIO.StringIO()
+    input_buffer.write(input_)
+    input_buffer.seek(0)
+    assert tuple(JSONParser().parse(input_buffer)) == output