Repository: pallets/werkzeug

Issue:
Remove charset mixins
@mitsuhiko agreed that we should remove the mixin based on my analysis in #1446.

The request mixin enabled unsafe behavior, because user-provided charsets can't be guaranteed safe in Python. The response mixin doesn't provide a compelling use over the standard response.


Discussion:

Resolving patch:
diff --git a/werkzeug/contrib/wrappers.py b/werkzeug/contrib/wrappers.py
--- a/werkzeug/contrib/wrappers.py
+++ b/werkzeug/contrib/wrappers.py
@@ -25,19 +25,13 @@ class Request(RequestBase, JSONRequestMixin):
 
 from werkzeug.exceptions import BadRequest
 from werkzeug.utils import cached_property
+from werkzeug.http import dump_options_header, parse_options_header
 from werkzeug._compat import wsgi_decoding_dance
 from werkzeug.wrappers.json import JSONMixin as _JSONMixin
-from werkzeug.wrappers import charset as _charset
 
 
 def is_known_charset(charset):
     """Checks if the given charset is known to Python."""
-    warnings.warn(
-        "'werkzeug.contrib.wrappers.is_known_charset' is deprecated as"
-        " of version 0.15 and will be removed in version 1.0.",
-        DeprecationWarning,
-        stacklevel=2,
-    )
     try:
         codecs.lookup(charset)
     except LookupError:
@@ -48,7 +42,7 @@ def is_known_charset(charset):
 class JSONRequestMixin(_JSONMixin):
     """
     .. deprecated:: 0.15
-        Moved to :class:`werkzeug.wrappers.json.JSONMixin`. This old
+        Moved to :class:`werkzeug.wrappers.JSONMixin`. This old
         import will be removed in version 1.0.
     """
 
@@ -73,6 +67,9 @@ class ProtobufRequestMixin(object):
     is not protobuf or if the data itself cannot be parsed property.
 
     .. _protobuf: https://github.com/protocolbuffers/protobuf
+
+    .. deprecated:: 0.15
+        This mixin will be removed in version 1.0.
     """
 
     #: by default the :class:`ProtobufRequestMixin` will raise a
@@ -112,6 +109,9 @@ class RoutingArgsRequestMixin(object):
     `specification`_.
 
     .. _specification: https://wsgi.readthedocs.io/en/latest/specifications/routing_args.html
+
+    .. deprecated:: 0.15
+        This mixin will be removed in version 1.0.
     """
 
     def _get_routing_args(self):
@@ -199,6 +199,9 @@ class MyRequest(ReverseSlashBehaviorRequestMixin, Request):
         +---------------+-------------------+---------------------+
         | `path`        | ``/foo/bar``      | ``foo/bar``         |
         +---------------+-------------------+---------------------+
+
+    .. deprecated:: 0.15
+        This mixin will be removed in version 1.0.
     """
 
     @cached_property
@@ -232,53 +235,132 @@ def script_root(self):
         return path.rstrip('/') + '/'
 
 
-class DynamicCharsetRequestMixin(_charset.DynamicCharsetRequestMixin):
-    """
+class DynamicCharsetRequestMixin(object):
+
+    """"If this mixin is mixed into a request class it will provide
+    a dynamic `charset` attribute.  This means that if the charset is
+    transmitted in the content type headers it's used from there.
+
+    Because it changes the behavior or :class:`Request` this class has
+    to be mixed in *before* the actual request class::
+
+        class MyRequest(DynamicCharsetRequestMixin, Request):
+            pass
+
+    By default the request object assumes that the URL charset is the
+    same as the data charset.  If the charset varies on each request
+    based on the transmitted data it's not a good idea to let the URLs
+    change based on that.  Most browsers assume either utf-8 or latin1
+    for the URLs if they have troubles figuring out.  It's strongly
+    recommended to set the URL charset to utf-8::
+
+        class MyRequest(DynamicCharsetRequestMixin, Request):
+            url_charset = 'utf-8'
+
     .. deprecated:: 0.15
-        Moved to :class:`werkzeug.wrappers.charset.DynamicCharsetRequestMixin`.
-        This old import will be removed in version 1.0.
+        This mixin will be removed in version 1.0.
+
+    .. versionadded:: 0.6
     """
 
-    @property
+    #: the default charset that is assumed if the content type header
+    #: is missing or does not contain a charset parameter.  The default
+    #: is latin1 which is what HTTP specifies as default charset.
+    #: You may however want to set this to utf-8 to better support
+    #: browsers that do not transmit a charset for incoming data.
+    default_charset = 'latin1'
+
+    def unknown_charset(self, charset):
+        """Called if a charset was provided but is not supported by
+        the Python codecs module.  By default latin1 is assumed then
+        to not lose any information, you may override this method to
+        change the behavior.
+
+        :param charset: the charset that was not found.
+        :return: the replacement charset.
+        """
+        return 'latin1'
+
+    @cached_property
     def charset(self):
+        """The charset from the content type."""
         warnings.warn(
-            "'werkzeug.contrib.wrappers.DynamicCharsetRequestMixin' has"
-            " moved to 'werkzeug.wrappers.charset"
-            ".DynamicCharsetRequestMixin'. This old import will be"
-            " removed in version 1.0.",
+            "'werkzeug.contrib.wrappers.DynamicCharsetRequestMixin'"
+            " is deprecated as of version 0.15 and will be removed in"
+            " version 1.0.",
             DeprecationWarning,
             stacklevel=2,
         )
-        return super(DynamicCharsetRequestMixin, self).charset
+        header = self.environ.get('CONTENT_TYPE')
+        if header:
+            ct, options = parse_options_header(header)
+            charset = options.get('charset')
+            if charset:
+                if is_known_charset(charset):
+                    return charset
+                return self.unknown_charset(charset)
+        return self.default_charset
 
 
-class DynamicCharsetResponseMixin(_charset.DynamicCharsetResponseMixin):
-    """
+class DynamicCharsetResponseMixin(object):
+
+    """If this mixin is mixed into a response class it will provide
+    a dynamic `charset` attribute.  This means that if the charset is
+    looked up and stored in the `Content-Type` header and updates
+    itself automatically.  This also means a small performance hit but
+    can be useful if you're working with different charsets on
+    responses.
+
+    Because the charset attribute is no a property at class-level, the
+    default value is stored in `default_charset`.
+
+    Because it changes the behavior or :class:`Response` this class has
+    to be mixed in *before* the actual response class::
+
+        class MyResponse(DynamicCharsetResponseMixin, Response):
+            pass
+
     .. deprecated:: 0.15
-        Moved to :class:`werkzeug.wrappers.charset.DynamicCharsetResponseMixin`.
-        This old import will be removed in version 1.0.
+        This mixin will be removed in version 1.0.
+
+    .. versionadded:: 0.6
     """
 
-    @property
-    def charset(self):
+    #: the default charset.
+    default_charset = 'utf-8'
+
+    def _get_charset(self):
         warnings.warn(
             "'werkzeug.contrib.wrappers.DynamicCharsetResponseMixin'"
-            " has moved to 'werkzeug.wrappers.charset"
-            ".DynamicCharsetResponseMixin'. This old import will be"
-            " removed in version 1.0.",
+            " is deprecated as of version 0.15 and will be removed in"
+            " version 1.0.",
             DeprecationWarning,
             stacklevel=2,
         )
-        return super(DynamicCharsetResponseMixin, self).charset
-
-    @charset.setter
-    def charset(self, value):
+        header = self.headers.get('content-type')
+        if header:
+            charset = parse_options_header(header)[1].get('charset')
+            if charset:
+                return charset
+        return self.default_charset
+
+    def _set_charset(self, charset):
         warnings.warn(
             "'werkzeug.contrib.wrappers.DynamicCharsetResponseMixin'"
-            " has moved to 'werkzeug.wrappers.charset"
-            ".DynamicCharsetResponseMixin'. This old import will be"
-            " removed in version 1.0.",
+            " is deprecated as of version 0.15 and will be removed in"
+            " version 1.0.",
             DeprecationWarning,
             stacklevel=2,
         )
-        super(DynamicCharsetResponseMixin, type(self)).charset.fset(self, value)
+        header = self.headers.get('content-type')
+        ct, options = parse_options_header(header)
+        if not ct:
+            raise TypeError('Cannot set charset if Content-Type '
+                            'header is missing.')
+        options['charset'] = charset
+        self.headers['Content-Type'] = dump_options_header(ct, options)
+
+    charset = property(_get_charset, _set_charset, doc="""
+        The charset for the response.  It's stored inside the
+        Content-Type header as a parameter.""")
+    del _get_charset, _set_charset
diff --git a/werkzeug/wrappers/charset.py b/werkzeug/wrappers/charset.py
deleted file mode 100644
--- a/werkzeug/wrappers/charset.py
+++ /dev/null
@@ -1,129 +0,0 @@
-import codecs
-
-from ..http import parse_options_header
-from ..utils import cached_property
-
-
-class DynamicCharsetRequestMixin(object):
-    """Take the charset attribute from the the ``Content-Type`` header.
-    Overrides the default behavior of hard-coding UTF-8. This is used
-    when decoding the data in the response body.
-
-    .. warning::
-        It may be dangerous to decode with a user-supplied charset
-        without validation in Python. Be absolutely sure you understand
-        the security implications of this before using it.
-
-        See :meth:`is_safe_charset`.
-    """
-
-    #: The default charset that is assumed if the content type header
-    #: is missing or does not contain a charset parameter.
-    #:
-    #: ..versionchanged:: 0.15
-    #:     Set to "utf-8" rather than "latin1", which matches
-    #:     :class:`~werkzeug.wrappers.BaseRequest`.
-    default_charset = "utf-8"
-
-    #: Use this charset for URL decoding rather than :attr:`charset`,
-    #: since that now refers to the data's charset, not the URL's.
-    #:
-    #: ..versionchanged:: 0.15
-    #:     Set to "utf-8" rather than keeping the base
-    #:     :attr:`~werkzeug.wrappers.BaseRequest.url_charset` behavior.
-    url_charset = "utf-8"
-
-    def unknown_charset(self, charset):
-        """Called if a charset was provided but :meth:`is_safe_charset`
-        returned ``False``.
-
-        :param charset: The unsafe charset.
-        :return: The replacement charset.
-
-        .. versionchanged:: 0.15
-            Returns "utf-8" rather than "latin1", matching
-            :attr:`default_charset`.
-        """
-        return "utf-8"
-
-    @cached_property
-    def charset(self):
-        """The charset from the content type."""
-        charset = self.mimetype_params.get("charset")
-
-        if charset:
-            if self.is_safe_charset(charset):
-                return charset
-
-            return self.unknown_charset(charset)
-
-        return self.default_charset
-
-    def is_safe_charset(self, charset):
-        """Whether the given charset is safe to use for decoding request
-        data.
-
-        .. warning::
-            Due to the way Python implements codecs, the only 100% safe
-            implementation is checking against a whitelist you maintain.
-
-        By default this calls :func:`codecs.lookup` then checks
-        ``_is_text_encoding`` on the returned info. This will exclude
-        charsets such as "zip" that should not be used to decode
-        untrusted data. However, this check may still be unsafe if other
-        libraries you use register binary codecs and don't set that
-        internal attribute correctly.
-
-        .. versionadded:: 0.15
-            Previously, :attr:`charset` only checked that the charset
-            was known, not that it was a text charset.
-        """
-        try:
-            info = codecs.lookup(charset)
-        except LookupError:
-            return False
-
-        return info._is_text_encoding
-
-
-class DynamicCharsetResponseMixin(object):
-    """Allow setting the charset for a response after creating it.
-    Setting a new charset updates the ``Content-Type`` header.
-
-    If the response already has data set, it is not re-encoded. Text
-    data can be set with the new charset using
-    :meth:`~werkzeug.wrappers.BaseResponse.set_data`.
-
-    This might be useful in some cases, but it's more efficient to
-    pass the data and charset when creating the response.
-    """
-
-    #: The default charset that is assumed if the content type header
-    #: is missing or does not contain a charset parameter.
-    default_charset = "utf-8"
-
-    @property
-    def charset(self):
-        """The charset for the response. It's stored as a parameter of
-        the ``Content-Type`` header.
-        """
-        header = self.headers.get("content-type")
-
-        if header:
-            charset = parse_options_header(header)[1].get("charset")
-
-            if charset:
-                return charset
-
-        return self.default_charset
-
-    @charset.setter
-    def charset(self, charset):
-        header = self.headers.get("content-type")
-        ct, options = parse_options_header(header)
-
-        if not ct:
-            raise TypeError("Cannot set charset if Content-Type header is missing.")
-
-        options["charset"] = charset
-        self.headers.set("content-type", ct, **options)
diff --git a/werkzeug/wrappers/json.py b/werkzeug/wrappers/json.py
--- a/werkzeug/wrappers/json.py
+++ b/werkzeug/wrappers/json.py
@@ -3,9 +3,9 @@
 import datetime
 import uuid
 
+from werkzeug.utils import detect_utf_encoding
 from .._compat import text_type
 from ..exceptions import BadRequest
-from ..utils import detect_utf_encoding
 
 try:
     import simplejson as _json


Test patch:
diff --git a/tests/contrib/test_wrappers.py b/tests/contrib/test_wrappers.py
--- a/tests/contrib/test_wrappers.py
+++ b/tests/contrib/test_wrappers.py
@@ -11,7 +11,7 @@
 
 from werkzeug.contrib import wrappers
 from werkzeug import routing
-from werkzeug.wrappers import Request
+from werkzeug.wrappers import Request, Response
 
 
 def test_reverse_slash_behavior():
@@ -29,3 +29,57 @@ class MyRequest(wrappers.ReverseSlashBehaviorRequestMixin, Request):
     assert adapter.match() == ('foo', {})
     adapter = map.bind(req.host, req.script_root)
     assert adapter.match(req.path) == ('foo', {})
+
+
+def test_dynamic_charset_request_mixin():
+    class MyRequest(wrappers.DynamicCharsetRequestMixin, Request):
+        pass
+    env = {'CONTENT_TYPE': 'text/html'}
+    req = MyRequest(env)
+    assert req.charset == 'latin1'
+
+    env = {'CONTENT_TYPE': 'text/html; charset=utf-8'}
+    req = MyRequest(env)
+    assert req.charset == 'utf-8'
+
+    env = {'CONTENT_TYPE': 'application/octet-stream'}
+    req = MyRequest(env)
+    assert req.charset == 'latin1'
+    assert req.url_charset == 'latin1'
+
+    MyRequest.url_charset = 'utf-8'
+    env = {'CONTENT_TYPE': 'application/octet-stream'}
+    req = MyRequest(env)
+    assert req.charset == 'latin1'
+    assert req.url_charset == 'utf-8'
+
+    def return_ascii(x):
+        return "ascii"
+    env = {'CONTENT_TYPE': 'text/plain; charset=x-weird-charset'}
+    req = MyRequest(env)
+    req.unknown_charset = return_ascii
+    assert req.charset == 'ascii'
+    assert req.url_charset == 'utf-8'
+
+
+def test_dynamic_charset_response_mixin():
+    class MyResponse(wrappers.DynamicCharsetResponseMixin, Response):
+        default_charset = 'utf-7'
+    resp = MyResponse(mimetype='text/html')
+    assert resp.charset == 'utf-7'
+    resp.charset = 'utf-8'
+    assert resp.charset == 'utf-8'
+    assert resp.mimetype == 'text/html'
+    assert resp.mimetype_params == {'charset': 'utf-8'}
+    resp.mimetype_params['charset'] = 'iso-8859-15'
+    assert resp.charset == 'iso-8859-15'
+    resp.set_data(u'Hällo Wörld')
+    assert b''.join(resp.iter_encoded()) == \
+           u'Hällo Wörld'.encode('iso-8859-15')
+    del resp.headers['content-type']
+    try:
+        resp.charset = 'utf-8'
+    except TypeError:
+        pass
+    else:
+        assert False, 'expected type error on charset setting without ct'
diff --git a/tests/test_wrappers.py b/tests/test_wrappers.py
--- a/tests/test_wrappers.py
+++ b/tests/test_wrappers.py
@@ -24,16 +24,12 @@
 from werkzeug import wrappers
 from werkzeug.exceptions import SecurityError, RequestedRangeNotSatisfiable, \
     BadRequest
-from werkzeug.wrappers.charset import (
-    DynamicCharsetRequestMixin,
-    DynamicCharsetResponseMixin,
-)
 from werkzeug.wrappers.json import JSONMixin
 from werkzeug.wsgi import LimitedStream, wrap_file
 from werkzeug.datastructures import MultiDict, ImmutableOrderedMultiDict, \
     ImmutableList, ImmutableTypeConversionDict, CharsetAccept, \
     MIMEAccept, LanguageAccept, Accept, CombinedMultiDict
-from werkzeug.test import Client, create_environ, run_wsgi_app, EnvironBuilder
+from werkzeug.test import Client, create_environ, run_wsgi_app
 from werkzeug._compat import implements_iterator, text_type
 
 
@@ -1341,54 +1337,3 @@ def test_cache_disabled(self):
 
         with pytest.raises(BadRequest):
             request.get_json()
-
-
-class TestDynamicCharsetRequestMixin(object):
-    class Request(DynamicCharsetRequestMixin, wrappers.Request):
-        pass
-
-    def test_from_header(self):
-        builder = EnvironBuilder(content_type="text/html; charset=latin1")
-        request = builder.get_request(self.Request)
-        assert request.charset == "latin1"
-        assert request.url_charset == "utf-8"
-
-    def test_unknown_charset(self):
-        builder = EnvironBuilder(content_type="text/html; charset=x-weird")
-        request = builder.get_request(self.Request)
-        assert request.charset == "utf-8"
-
-    def test_unsafe_charset(self):
-        builder = EnvironBuilder(content_type="text/html; charset=zip")
-        request = builder.get_request(self.Request)
-        assert request.charset == "utf-8"
-
-
-class TestDynamicCharsetResponseMixin(object):
-    class Response(DynamicCharsetResponseMixin, wrappers.Response):
-        pass
-
-    def test_from_header(self):
-        response = self.Response(content_type="text/html; charset=latin1")
-        assert response.charset == "latin1"
-
-    def test_set(self):
-        response = self.Response(content_type="text/html; charset=latin1")
-        response.charset = "ascii"
-        assert response.mimetype == "text/html"
-        assert response.mimetype_params["charset"] == "ascii"
-
-    def test_set_no_content_type(self):
-        response = self.Response()
-        del response.headers["content-type"]
-
-        with pytest.raises(TypeError):
-            response.charset = "ascii"
-
-    def test_set_data(self):
-        value = u"Hällo Wörld"
-        response = self.Response(response=value)
-        response.charset = "latin1"
-        assert response.get_data() == value.encode("utf-8")
-        response.set_data(value)
-        assert response.get_data() == value.encode("latin1")