Repository: falconry/falcon

Issue:
Non-ASCII chars in form posts are not supported
Should check header and act appropriate rather than jut assuming ASCII in falcon.Request



Discussion:
I'll go ahead and take this one.

As it turns out, the bug reported may simply be a result of the request not being encoded correctly.  According to [the spec](http://www.w3.org/TR/html5/forms.html#application/x-www-form-urlencoded-encoding-algorithm), non-ASCII chars are supposed to be percent-encoded.


Resolving patch:
diff --git a/falcon/request.py b/falcon/request.py
--- a/falcon/request.py
+++ b/falcon/request.py
@@ -97,6 +97,14 @@ class Request(object):
                 and merge them into the query string parameters. In this
                 case, the stream will be left at EOF.
 
+                Note also that the character encoding for fields, before
+                percent-encoding non-ASCII bytes, is assumed to be
+                UTF-8. The special "_charset_" field is ignored if present.
+
+                Falcon expects form-encoded request bodies to be
+                encoded according to the standard W3C algorithm (see
+                also http://goo.gl/6rlcux).
+
         date (datetime): Value of the Date header, converted to a
             `datetime.datetime` instance. The header value is assumed to
             conform to RFC 1123.
@@ -216,25 +224,7 @@ def __init__(self, env):
             # covered since the test that does so uses multiprocessing.
             self.stream = helpers.Body(self.stream, self.content_length)
 
-        # PERF(kgriffs): Technically, we should spend a few more
-        # cycles and parse the content type for real, but
-        # this heuristic will work virtually all the time.
-        if (self.content_type and
-                'application/x-www-form-urlencoded' in self.content_type):
-
-            # NOTE(kgriffs): This assumes self.stream has been patched
-            # above in the case of wsgiref, so that self.content_length
-            # is not needed. Normally we just avoid accessing
-            # self.content_length, because it is a little expensive
-            # to call. We could cache self.content_length, but the
-            # overhead to do that won't usually be helpful, since
-            # content length will only ever be read once per
-            # request in most cases.
-            body = self.stream.read()
-            body = body.decode('ascii')
-
-            extra_params = uri.parse_query_string(uri.decode(body))
-            self._params.update(extra_params)
+        self._parse_form_urlencoded()
 
     # ------------------------------------------------------------------------
     # Properties
@@ -760,6 +750,39 @@ def log_error(self, message):  # pragma: no cover
     # Helpers
     # ------------------------------------------------------------------------
 
+    def _parse_form_urlencoded(self):
+        # PERF(kgriffs): Technically, we should spend a few more
+        # cycles and parse the content type for real, but
+        # this heuristic will work virtually all the time.
+        if (self.content_type and
+                'application/x-www-form-urlencoded' in self.content_type):
+
+            # NOTE(kgriffs): This assumes self.stream has been patched
+            # above in the case of wsgiref, so that self.content_length
+            # is not needed. Normally we just avoid accessing
+            # self.content_length, because it is a little expensive
+            # to call. We could cache self.content_length, but the
+            # overhead to do that won't usually be helpful, since
+            # content length will only ever be read once per
+            # request in most cases.
+            body = self.stream.read()
+
+            # NOTE(kgriffs): According to http://goo.gl/6rlcux the
+            # body should be US-ASCII. Enforcing this also helps
+            # catch malicious input.
+            try:
+                body = body.decode('ascii')
+            except UnicodeDecodeError:
+                body = None
+                self.log_error('Non-ASCII characters found in form body '
+                               'with Content-Type of '
+                               'application/x-www-form-urlencoded. Body '
+                               'will be ignored.')
+
+            if body:
+                extra_params = uri.parse_query_string(uri.decode(body))
+                self._params.update(extra_params)
+
     def _get_header_by_wsgi_name(self, name):
         """Looks up a header, assuming name is already UPPERCASE_UNDERSCORE
 


Test patch:
diff --git a/tests/test_query_params.py b/tests/test_query_params.py
--- a/tests/test_query_params.py
+++ b/tests/test_query_params.py
@@ -250,6 +250,14 @@ def simulate_request(self, path, query_string):
         super(PostQueryParams, self).simulate_request(path, body=query_string,
                                                       headers=headers)
 
+    def test_non_ascii(self):
+        value = u'\u8c46\u74e3'
+        query_string = b'q=' + value.encode('utf-8')
+        self.simulate_request('/', query_string=query_string)
+
+        req = self.resource.req
+        self.assertEqual(req.get_param('q'), None)
+
 
 class GetQueryParams(_TestQueryParams):
     def simulate_request(self, path, query_string):