Repository: Kinto/kinto

Issue:
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xc1 in position 10: invalid start byte
With v6.0.5

Related issues #1182 #717 

First issue seems to be this:
```
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xc1 in position 10: invalid start byte
  File "pyramid/tweens.py", line 22, in excview_tween
    response = handler(request)
  File "pyramid_tm/__init__.py", line 119, in tm_tween
    reraise(*exc_info)
  File "pyramid_tm/compat.py", line 15, in reraise
    raise value
  File "pyramid_tm/__init__.py", line 98, in tm_tween
    response = handler(request)
  File "pyramid/router.py", line 76, in handle_request
    has_listeners and notify(NewRequest(request))
  File "pyramid/registry.py", line 110, in notify
    [ _ for _ in self.subscribers(events, None) ]
  File "/usr/local/lib/python3.5/site-packages/zope/interface/registry.py", line 442, in subscribers
    return self.adapters.subscribers(objects, provided)
  File "/usr/local/lib/python3.5/site-packages/zope/interface/adapter.py", line 598, in subscribers
    subscription(*objects)
  File "pyramid/config/adapters.py", line 103, in derived_subscriber
    return subscriber(arg[0])
  File "kinto/core/initialization.py", line 354, in on_new_request
    querystring=dict(request.GET),
  File "webob/request.py", line 833, in GET
    vars = GetDict(data, env)
  File "webob/multidict.py", line 287, in __init__
    MultiDict.__init__(self, data)
  File "webob/multidict.py", line 38, in __init__
    items = list(args[0])
  File "webob/compat.py", line 114, in parse_qsl_text
    yield (name.decode(encoding), value.decode(encoding))
```

which then crashes when trying to log it out:
```
OverflowError: Overlong 2 byte UTF-8 sequence detected when encoding string
  File "pyramid/router.py", line 233, in __call__
    response = self.invoke_subrequest(request, use_tweens=True)
  File "pyramid/router.py", line 208, in invoke_subrequest
    response = handle_request(request)
  File "kinto/core/initialization.py", line 167, in eos_tween
    return handler(request)
  File "pyramid/tweens.py", line 51, in excview_tween
    request_iface=request_iface.combined
  File "pyramid/view.py", line 612, in _call_view
    response = view_callable(context, request)
  File "pyramid/viewderivers.py", line 409, in viewresult_to_response
    result = view(context, request)
  File "kinto/core/views/errors.py", line 121, in error
    logger.error(context, exc_info=True)
  File "structlog/stdlib.py", line 81, in error
    return self._proxy_to_logger('error', event, *args, **kw)
  File "structlog/stdlib.py", line 119, in _proxy_to_logger
    **event_kw)
  File "structlog/_base.py", line 176, in _proxy_to_logger
    args, kw = self._process_event(method_name, event, event_kw)
  File "structlog/_base.py", line 136, in _process_event
    event_dict = proc(self._logger, method_name, event_dict)
  File "kinto/core/logs.py", line 125, in __call__
    return utils.json.dumps(event_dict)

```


Discussion:
Do we have a complete set of request information that triggered this bug?
Unfortunately I couldn't find it. It looks like our logger crashed when trying to output the JSON #DoubleFail

And I couldn't access Kibana either :( 
Refs https://github.com/Pylons/pyramid/pull/2725
Refs https://github.com/Pylons/webob/pull/268
Also
- https://github.com/Pylons/webob/issues/115
- https://github.com/Pylons/webob/issues/161

Resolving patch:
diff --git a/kinto/core/errors.py b/kinto/core/errors.py
--- a/kinto/core/errors.py
+++ b/kinto/core/errors.py
@@ -1,4 +1,5 @@
 import colander
+import logging
 from pyramid import httpexceptions
 from enum import Enum
 
@@ -193,3 +194,20 @@ def send_alert(request, message=None, url=None, code='soft-eol'):
         'message': message,
         'url': url
     })
+
+
+def request_GET(request):
+    """Catches a UnicodeDecode error in request.GET in case a wrong request was received.
+    Fixing a webob long term issue: https://github.com/Pylons/webob/issues/161
+    """
+    try:
+        return request.GET
+    except UnicodeDecodeError as e:
+        querystring = request.environ.get('QUERY_STRING', '')
+        logger = logging.getLogger(__name__)
+        logger.warn('Error decoding QUERY_STRING: %s' % request.environ)
+        raise http_error(
+            httpexceptions.HTTPBadRequest(),
+            errno=ERRORS.INVALID_PARAMETERS,
+            message="A request with an incorrect encoding in the querystring was"
+            "received. Please make sure your requests are encoded in UTF-8: %s" % querystring)
diff --git a/kinto/core/initialization.py b/kinto/core/initialization.py
--- a/kinto/core/initialization.py
+++ b/kinto/core/initialization.py
@@ -336,7 +336,7 @@ def on_new_request(event):
         request.log_context(agent=request.headers.get('User-Agent'),
                             path=request_path,
                             method=request.method,
-                            querystring=dict(request.GET),
+                            querystring=dict(errors.request_GET(request)),
                             lang=request.headers.get('Accept-Language'),
                             uid=None,
                             authn_type=None,
diff --git a/kinto/core/resource/__init__.py b/kinto/core/resource/__init__.py
--- a/kinto/core/resource/__init__.py
+++ b/kinto/core/resource/__init__.py
@@ -12,7 +12,7 @@
                                     HTTPNotFound, HTTPServiceUnavailable)
 
 from kinto.core import Service
-from kinto.core.errors import http_error, raise_invalid, send_alert, ERRORS
+from kinto.core.errors import http_error, raise_invalid, send_alert, ERRORS, request_GET
 from kinto.core.events import ACTIONS
 from kinto.core.storage import exceptions as storage_exceptions, Filter, Sort
 from kinto.core.utils import (
@@ -1062,7 +1062,7 @@ def _next_page_url(self, sorting, limit, last_record, offset):
         """Build the Next-Page header from where we stopped."""
         token = self._build_pagination_token(sorting, last_record, offset)
 
-        params = {**self.request.GET, '_limit': limit, '_token': token}
+        params = {**request_GET(self.request), '_limit': limit, '_token': token}
 
         service = self.request.current_service
         next_page_url = self.request.route_url(service.name, _query=params,


Test patch:
diff --git a/tests/core/resource/test_views.py b/tests/core/resource/test_views.py
--- a/tests/core/resource/test_views.py
+++ b/tests/core/resource/test_views.py
@@ -3,7 +3,8 @@
 
 from kinto.core.storage import exceptions as storage_exceptions
 from kinto.core.errors import ERRORS
-from kinto.core.testing import unittest
+from kinto.core.testing import unittest, FormattedErrorMixin
+
 
 from ..support import BaseWebTest
 
@@ -776,3 +777,14 @@ def test_unknown_fields_are_ignored(self):
         resp = self.app.get(self.collection_url + '?_fields=nationality')
         result = resp.json['data'][0]
         self.assertNotIn('nationality', result)
+
+
+class UnicodeDecodeErrorTest(BaseWebTest, FormattedErrorMixin, unittest.TestCase):
+    collection_url = '/spores'
+
+    def test_wrong_filter_encoding_raise_a_400_bad_request(self):
+        resp = self.app.get(self.collection_url + '?foo\xe2\xfc\xa7bar',
+                            headers=self.headers, status=400)
+        self.assertFormattedError(resp, 400, ERRORS.INVALID_PARAMETERS, "Bad Request",
+                                  "A request with an incorrect encoding in the querystring was"
+                                  "received. Please make sure your requests are encoded in UTF-8")