Repository: pallets/werkzeug

Issue:
Make DynamicCharsetRequestMixin more secure
After #1442 

Currently it only checks if the codec exists, but codecs like "zip" exist and should not be valid. Can look at `codecs.lookup(charset)._is_text_charset` for a better heuristic. However, this is still not completely secure because other Python libraries can add codecs and may not set this attribute appropriately.

The mixin should gain a `is_safe_charset` method that does the above check but can also be overridden to be more strict. A big warning should be added to the docs to tell people to be aware of this before using it.

Alternatively we can just deprecate it completely, but I think it probably is asked about enough that we should have a "right" way to do it even if it's still a footgun.


Discussion:

Resolving patch:
diff --git a/werkzeug/contrib/wrappers.py b/werkzeug/contrib/wrappers.py
--- a/werkzeug/contrib/wrappers.py
+++ b/werkzeug/contrib/wrappers.py
@@ -20,6 +20,7 @@ class Request(RequestBase, JSONRequestMixin):
     :copyright: (c) 2014 by the Werkzeug Team, see AUTHORS for more details.
     :license: BSD, see LICENSE for more details.
 """
+import codecs
 import warnings
 
 from werkzeug.exceptions import BadRequest
@@ -32,13 +33,16 @@ class Request(RequestBase, JSONRequestMixin):
 def is_known_charset(charset):
     """Checks if the given charset is known to Python."""
     warnings.warn(
-        "'werkzeug.contrib.wrappers.is_known_charset' has moved to"
-        " 'werkzeug.wrappers.charset.is_known_charset'. This old import"
-        " will be removed in version 1.0.",
+        "'werkzeug.contrib.wrappers.is_known_charset' is deprecated as"
+        " of version 0.15 and will be removed in version 1.0.",
         DeprecationWarning,
         stacklevel=2,
     )
-    return _charset.is_known_charset(charset)
+    try:
+        codecs.lookup(charset)
+    except LookupError:
+        return False
+    return True
 
 
 class JSONRequestMixin(_JSONMixin):
diff --git a/werkzeug/wrappers/charset.py b/werkzeug/wrappers/charset.py
--- a/werkzeug/wrappers/charset.py
+++ b/werkzeug/wrappers/charset.py
@@ -1,117 +1,129 @@
 import codecs
 
-from werkzeug.http import parse_options_header, dump_options_header
-from werkzeug.utils import cached_property
-
-
-def is_known_charset(charset):
-    """Checks if the given charset is known to Python."""
-    try:
-        codecs.lookup(charset)
-    except LookupError:
-        return False
-    return True
+from ..http import parse_options_header
+from ..utils import cached_property
 
 
 class DynamicCharsetRequestMixin(object):
+    """Take the charset attribute from the the ``Content-Type`` header.
+    Overrides the default behavior of hard-coding UTF-8. This is used
+    when decoding the data in the response body.
 
-    """If this mixin is mixed into a request class it will provide
-    a dynamic `charset` attribute.  This means that if the charset is
-    transmitted in the content type headers it's used from there.
-
-    Because it changes the behavior or :class:`Request` this class has
-    to be mixed in *before* the actual request class::
-
-        class MyRequest(DynamicCharsetRequestMixin, Request):
-            pass
+    .. warning::
+        It may be dangerous to decode with a user-supplied charset
+        without validation in Python. Be absolutely sure you understand
+        the security implications of this before using it.
 
-    By default the request object assumes that the URL charset is the
-    same as the data charset.  If the charset varies on each request
-    based on the transmitted data it's not a good idea to let the URLs
-    change based on that.  Most browsers assume either utf-8 or latin1
-    for the URLs if they have troubles figuring out.  It's strongly
-    recommended to set the URL charset to utf-8::
-
-        class MyRequest(DynamicCharsetRequestMixin, Request):
-            url_charset = 'utf-8'
-
-    .. versionadded:: 0.6
+        See :meth:`is_safe_charset`.
     """
 
-    #: the default charset that is assumed if the content type header
-    #: is missing or does not contain a charset parameter.  The default
-    #: is latin1 which is what HTTP specifies as default charset.
-    #: You may however want to set this to utf-8 to better support
-    #: browsers that do not transmit a charset for incoming data.
-    default_charset = 'latin1'
+    #: The default charset that is assumed if the content type header
+    #: is missing or does not contain a charset parameter.
+    #:
+    #: ..versionchanged:: 0.15
+    #:     Set to "utf-8" rather than "latin1", which matches
+    #:     :class:`~werkzeug.wrappers.BaseRequest`.
+    default_charset = "utf-8"
+
+    #: Use this charset for URL decoding rather than :attr:`charset`,
+    #: since that now refers to the data's charset, not the URL's.
+    #:
+    #: ..versionchanged:: 0.15
+    #:     Set to "utf-8" rather than keeping the base
+    #:     :attr:`~werkzeug.wrappers.BaseRequest.url_charset` behavior.
+    url_charset = "utf-8"
 
     def unknown_charset(self, charset):
-        """Called if a charset was provided but is not supported by
-        the Python codecs module.  By default latin1 is assumed then
-        to not lose any information, you may override this method to
-        change the behavior.
+        """Called if a charset was provided but :meth:`is_safe_charset`
+        returned ``False``.
+
+        :param charset: The unsafe charset.
+        :return: The replacement charset.
 
-        :param charset: the charset that was not found.
-        :return: the replacement charset.
+        .. versionchanged:: 0.15
+            Returns "utf-8" rather than "latin1", matching
+            :attr:`default_charset`.
         """
-        return 'latin1'
+        return "utf-8"
 
     @cached_property
     def charset(self):
         """The charset from the content type."""
-        header = self.environ.get('CONTENT_TYPE')
-        if header:
-            ct, options = parse_options_header(header)
-            charset = options.get('charset')
-            if charset:
-                if is_known_charset(charset):
-                    return charset
-                return self.unknown_charset(charset)
+        charset = self.mimetype_params.get("charset")
+
+        if charset:
+            if self.is_safe_charset(charset):
+                return charset
+
+            return self.unknown_charset(charset)
+
         return self.default_charset
 
+    def is_safe_charset(self, charset):
+        """Whether the given charset is safe to use for decoding request
+        data.
 
-class DynamicCharsetResponseMixin(object):
+        .. warning::
+            Due to the way Python implements codecs, the only 100% safe
+            implementation is checking against a whitelist you maintain.
+
+        By default this calls :func:`codecs.lookup` then checks
+        ``_is_text_encoding`` on the returned info. This will exclude
+        charsets such as "zip" that should not be used to decode
+        untrusted data. However, this check may still be unsafe if other
+        libraries you use register binary codecs and don't set that
+        internal attribute correctly.
+
+        .. versionadded:: 0.15
+            Previously, :attr:`charset` only checked that the charset
+            was known, not that it was a text charset.
+        """
+        try:
+            info = codecs.lookup(charset)
+        except LookupError:
+            return False
 
-    """If this mixin is mixed into a response class it will provide
-    a dynamic `charset` attribute.  This means that if the charset is
-    looked up and stored in the `Content-Type` header and updates
-    itself automatically.  This also means a small performance hit but
-    can be useful if you're working with different charsets on
-    responses.
+        return info._is_text_encoding
 
-    Because the charset attribute is no a property at class-level, the
-    default value is stored in `default_charset`.
 
-    Because it changes the behavior or :class:`Response` this class has
-    to be mixed in *before* the actual response class::
+class DynamicCharsetResponseMixin(object):
+    """Allow setting the charset for a response after creating it.
+    Setting a new charset updates the ``Content-Type`` header.
 
-        class MyResponse(DynamicCharsetResponseMixin, Response):
-            pass
+    If the response already has data set, it is not re-encoded. Text
+    data can be set with the new charset using
+    :meth:`~werkzeug.wrappers.BaseResponse.set_data`.
 
-    .. versionadded:: 0.6
+    This might be useful in some cases, but it's more efficient to
+    pass the data and charset when creating the response.
     """
 
-    #: the default charset.
-    default_charset = 'utf-8'
+    #: The default charset that is assumed if the content type header
+    #: is missing or does not contain a charset parameter.
+    default_charset = "utf-8"
+
+    @property
+    def charset(self):
+        """The charset for the response. It's stored as a parameter of
+        the ``Content-Type`` header.
+        """
+        header = self.headers.get("content-type")
 
-    def _get_charset(self):
-        header = self.headers.get('content-type')
         if header:
-            charset = parse_options_header(header)[1].get('charset')
+            charset = parse_options_header(header)[1].get("charset")
+
             if charset:
                 return charset
+
         return self.default_charset
 
-    def _set_charset(self, charset):
-        header = self.headers.get('content-type')
+    @charset.setter
+    def charset(self, charset):
+        header = self.headers.get("content-type")
         ct, options = parse_options_header(header)
+
         if not ct:
-            raise TypeError('Cannot set charset if Content-Type '
-                            'header is missing.')
-        options['charset'] = charset
-        self.headers['Content-Type'] = dump_options_header(ct, options)
-
-    charset = property(_get_charset, _set_charset, doc="""
-        The charset for the response.  It's stored inside the
-        Content-Type header as a parameter.""")
-    del _get_charset, _set_charset
+            raise TypeError("Cannot set charset if Content-Type header is missing.")
+
+        options["charset"] = charset
+        self.headers.set("content-type", ct, **options)
diff --git a/werkzeug/wrappers/json.py b/werkzeug/wrappers/json.py
--- a/werkzeug/wrappers/json.py
+++ b/werkzeug/wrappers/json.py
@@ -3,9 +3,9 @@
 import datetime
 import uuid
 
-from werkzeug.utils import detect_utf_encoding
 from .._compat import text_type
 from ..exceptions import BadRequest
+from ..utils import detect_utf_encoding
 
 try:
     import simplejson as _json


Test patch:
diff --git a/tests/test_wrappers.py b/tests/test_wrappers.py
--- a/tests/test_wrappers.py
+++ b/tests/test_wrappers.py
@@ -33,7 +33,7 @@
 from werkzeug.datastructures import MultiDict, ImmutableOrderedMultiDict, \
     ImmutableList, ImmutableTypeConversionDict, CharsetAccept, \
     MIMEAccept, LanguageAccept, Accept, CombinedMultiDict
-from werkzeug.test import Client, create_environ, run_wsgi_app
+from werkzeug.test import Client, create_environ, run_wsgi_app, EnvironBuilder
 from werkzeug._compat import implements_iterator, text_type
 
 
@@ -1343,55 +1343,52 @@ def test_cache_disabled(self):
             request.get_json()
 
 
-def test_dynamic_charset_request_mixin():
-    class MyRequest(DynamicCharsetRequestMixin, wrappers.Request):
+class TestDynamicCharsetRequestMixin(object):
+    class Request(DynamicCharsetRequestMixin, wrappers.Request):
         pass
-    env = {'CONTENT_TYPE': 'text/html'}
-    req = MyRequest(env)
-    assert req.charset == 'latin1'
-
-    env = {'CONTENT_TYPE': 'text/html; charset=utf-8'}
-    req = MyRequest(env)
-    assert req.charset == 'utf-8'
-
-    env = {'CONTENT_TYPE': 'application/octet-stream'}
-    req = MyRequest(env)
-    assert req.charset == 'latin1'
-    assert req.url_charset == 'latin1'
-
-    MyRequest.url_charset = 'utf-8'
-    env = {'CONTENT_TYPE': 'application/octet-stream'}
-    req = MyRequest(env)
-    assert req.charset == 'latin1'
-    assert req.url_charset == 'utf-8'
-
-    def return_ascii(x):
-        return "ascii"
-    env = {'CONTENT_TYPE': 'text/plain; charset=x-weird-charset'}
-    req = MyRequest(env)
-    req.unknown_charset = return_ascii
-    assert req.charset == 'ascii'
-    assert req.url_charset == 'utf-8'
-
-
-def test_dynamic_charset_response_mixin():
-    class MyResponse(DynamicCharsetResponseMixin, wrappers.Response):
-        default_charset = 'utf-7'
-    resp = MyResponse(mimetype='text/html')
-    assert resp.charset == 'utf-7'
-    resp.charset = 'utf-8'
-    assert resp.charset == 'utf-8'
-    assert resp.mimetype == 'text/html'
-    assert resp.mimetype_params == {'charset': 'utf-8'}
-    resp.mimetype_params['charset'] = 'iso-8859-15'
-    assert resp.charset == 'iso-8859-15'
-    resp.set_data(u'Hällo Wörld')
-    assert b''.join(resp.iter_encoded()) == \
-           u'Hällo Wörld'.encode('iso-8859-15')
-    del resp.headers['content-type']
-    try:
-        resp.charset = 'utf-8'
-    except TypeError:
+
+    def test_from_header(self):
+        builder = EnvironBuilder(content_type="text/html; charset=latin1")
+        request = builder.get_request(self.Request)
+        assert request.charset == "latin1"
+        assert request.url_charset == "utf-8"
+
+    def test_unknown_charset(self):
+        builder = EnvironBuilder(content_type="text/html; charset=x-weird")
+        request = builder.get_request(self.Request)
+        assert request.charset == "utf-8"
+
+    def test_unsafe_charset(self):
+        builder = EnvironBuilder(content_type="text/html; charset=zip")
+        request = builder.get_request(self.Request)
+        assert request.charset == "utf-8"
+
+
+class TestDynamicCharsetResponseMixin(object):
+    class Response(DynamicCharsetResponseMixin, wrappers.Response):
         pass
-    else:
-        assert False, 'expected type error on charset setting without ct'
+
+    def test_from_header(self):
+        response = self.Response(content_type="text/html; charset=latin1")
+        assert response.charset == "latin1"
+
+    def test_set(self):
+        response = self.Response(content_type="text/html; charset=latin1")
+        response.charset = "ascii"
+        assert response.mimetype == "text/html"
+        assert response.mimetype_params["charset"] == "ascii"
+
+    def test_set_no_content_type(self):
+        response = self.Response()
+        del response.headers["content-type"]
+
+        with pytest.raises(TypeError):
+            response.charset = "ascii"
+
+    def test_set_data(self):
+        value = u"Hällo Wörld"
+        response = self.Response(response=value)
+        response.charset = "latin1"
+        assert response.get_data() == value.encode("utf-8")
+        response.set_data(value)
+        assert response.get_data() == value.encode("latin1")