Repository: falconry/falcon

Issue:
Decoding the query string before parsing it is problematic
Let's say I wanted to encode some JSON, like `{"foo":1,"bar":2}` in a query param. Usually I would encode this value and pass it in the URL like this, `/my/api?skidoo=%7B%22foo%22%3A1%2C%22bar%22%3A1%7D`. The way things currently work, the whole query string will be decoded and then parsed by splitting at commas. This results in a `skidoo` param with two values `'{"foo":1'` and `'"bar":1}'`.

If the query string were parsed and then each key and value decoded the result would be a single `skidoo` param with the value ``'{"foo":1, "bar":1}'`.



Discussion:
That sounds reasonable.  Perhaps the existing behavior can be kept in a method should a user desire this behavior, but default behavior should decode the query string without assuming the values are comma-delimited.

Yeah, so the issue isn't so much that the entire query string is being decoded in a single pass, but that each value is being split by comma. We could use RequestOptions to toggle splitting values on commas, and turn it off by default. That would be a breaking change, but it would be easy for anyone encoding lists into values to turn it back on and be back in business.


Resolving patch:
diff --git a/falcon/request.py b/falcon/request.py
--- a/falcon/request.py
+++ b/falcon/request.py
@@ -220,16 +220,13 @@ def __init__(self, env, options=None):
 
         # PERF(kgriffs): if...in is faster than using env.get(...)
         if 'QUERY_STRING' in env:
-            query_str = env['QUERY_STRING']
+            self.query_string = env['QUERY_STRING']
 
-            if query_str:
-                self.query_string = uri.decode(query_str)
+            if self.query_string:
                 self._params = uri.parse_query_string(
                     self.query_string,
                     keep_blank_qs_values=self.options.keep_blank_qs_values,
                 )
-            else:
-                self.query_string = six.text_type()
 
         else:
             self.query_string = six.text_type()
@@ -915,7 +912,7 @@ def _parse_form_urlencoded(self):
 
         if body:
             extra_params = uri.parse_query_string(
-                uri.decode(body),
+                body,
                 keep_blank_qs_values=self.options.keep_blank_qs_values,
             )
 
diff --git a/falcon/util/uri.py b/falcon/util/uri.py
--- a/falcon/util/uri.py
+++ b/falcon/util/uri.py
@@ -259,6 +259,10 @@ def parse_query_string(query_string, keep_blank_qs_values=False):
         a more compact form in which the param may be given a single time
         but set to a ``list`` of comma-separated elements (e.g., 'foo=a,b,c').
 
+        When using this format, all commas uri-encoded will not be treated by
+        Falcon as a delimiter. If the client wants to send a value as a list,
+        it must not encode the commas with the values.
+
         The two different ways of specifying lists may not be mixed in
         a single query string for the same parameter.
 
@@ -287,14 +291,21 @@ def parse_query_string(query_string, keep_blank_qs_values=False):
         if not (v or keep_blank_qs_values):
             continue
 
+        # Note(steffgrez): Falcon first decode name parameter for handle
+        # utf8 character.
+        k = decode(k)
+
+        # NOTE(steffgrez): Falcon decode value at the last moment. So query
+        # parser won't mix up between percent-encoded comma (as value) and
+        # comma-separated list (as reserved character for sub-delimiter)
         if k in params:
             # The key was present more than once in the POST data.  Convert to
             # a list, or append the next value to the list.
             old_value = params[k]
             if isinstance(old_value, list):
-                old_value.append(v)
+                old_value.append(decode(v))
             else:
-                params[k] = [old_value, v]
+                params[k] = [old_value, decode(v)]
 
         else:
             if ',' in v:
@@ -309,9 +320,11 @@ def parse_query_string(query_string, keep_blank_qs_values=False):
                     # NOTE(kgriffs): Normalize the result in the case that
                     # some elements are empty strings, such that the result
                     # will be the same for 'foo=1,,3' as 'foo=1&foo=&foo=3'.
-                    v = [element for element in v if element]
-
-            params[k] = v
+                    params[k] = [decode(element) for element in v if element]
+                else:
+                    params[k] = [decode(element) for element in v]
+            else:
+                params[k] = decode(v)
 
     return params
 


Test patch:
diff --git a/tests/test_query_params.py b/tests/test_query_params.py
--- a/tests/test_query_params.py
+++ b/tests/test_query_params.py
@@ -49,7 +49,7 @@ def test_simple(self):
         self.assertEqual(store['limit'], '25')
 
     def test_percent_encoded(self):
-        query_string = 'id=23%2c42&q=%e8%b1%86+%e7%93%a3'
+        query_string = 'id=23,42&q=%e8%b1%86+%e7%93%a3'
         self.simulate_request('/', query_string=query_string)
 
         req = self.resource.req
diff --git a/tests/test_utils.py b/tests/test_utils.py
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -185,6 +185,38 @@ def test_prop_uri_decode_models_stdlib_unquote_plus(self):
             actual = uri.decode(case)
             self.assertEqual(expect, actual)
 
+    def test_parse_query_string(self):
+        query_strinq = (
+            "a=http%3A%2F%2Ffalconframework.org%3Ftest%3D1"
+            "&b=%7B%22test1%22%3A%20%22data1%22%"
+            "2C%20%22test2%22%3A%20%22data2%22%7D"
+            "&c=1,2,3"
+            "&d=test"
+            "&e=a,,%26%3D%2C"
+            "&f=a&f=a%3Db"
+            "&%C3%A9=a%3Db"
+        )
+        decoded_url = 'http://falconframework.org?test=1'
+        decoded_json = '{"test1": "data1", "test2": "data2"}'
+
+        result = uri.parse_query_string(query_strinq)
+        self.assertEqual(result['a'], decoded_url)
+        self.assertEqual(result['b'], decoded_json)
+        self.assertEqual(result['c'], ['1', '2', '3'])
+        self.assertEqual(result['d'], 'test')
+        self.assertEqual(result['e'], ['a', '&=,'])
+        self.assertEqual(result['f'], ['a', 'a=b'])
+        self.assertEqual(result[u'é'], 'a=b')
+
+        result = uri.parse_query_string(query_strinq, True)
+        self.assertEqual(result['a'], decoded_url)
+        self.assertEqual(result['b'], decoded_json)
+        self.assertEqual(result['c'], ['1', '2', '3'])
+        self.assertEqual(result['d'], 'test')
+        self.assertEqual(result['e'], ['a', '', '&=,'])
+        self.assertEqual(result['f'], ['a', 'a=b'])
+        self.assertEqual(result[u'é'], 'a=b')
+
     def test_parse_host(self):
         self.assertEqual(uri.parse_host('::1'), ('::1', None))
         self.assertEqual(uri.parse_host('2001:ODB8:AC10:FE01::'),