Repository: urllib3/urllib3

Issue:
decode_content parameter to Response.stream() is not respected for chunked bodies
Originally kennethreitz/requests#2561.

It seems that if you hit an endpoint that sets both Content-Encoding: gzip and Transfer-Encoding: chunked, and then use `Response.stream()` with `decode_content=True`, we don't decompress the content. That represents an API change, and I'd argue a regression.

Should be a fairly simple fix though.



Discussion:

Resolving patch:
diff --git a/urllib3/response.py b/urllib3/response.py
--- a/urllib3/response.py
+++ b/urllib3/response.py
@@ -172,6 +172,36 @@ def tell(self):
         """
         return self._fp_bytes_read
 
+    def _init_decoder(self):
+        """
+        Set-up the _decoder attribute if necessar.
+        """
+        # Note: content-encoding value should be case-insensitive, per RFC 7230
+        # Section 3.2
+        content_encoding = self.headers.get('content-encoding', '').lower()
+        if self._decoder is None:
+            if content_encoding in self.CONTENT_DECODERS:
+                self._decoder = _get_decoder(content_encoding)
+
+    def _decode(self, data, decode_content, flush_decoder):
+        """
+        Decode the data passed in and potentially flush the decoder.
+        """
+        try:
+            if decode_content and self._decoder:
+                data = self._decoder.decompress(data)
+        except (IOError, zlib.error) as e:
+            content_encoding = self.headers.get('content-encoding', '').lower()
+            raise DecodeError(
+                "Received response with content-encoding: %s, but "
+                "failed to decode it." % content_encoding, e)
+
+        if flush_decoder and decode_content and self._decoder:
+            buf = self._decoder.decompress(binary_type())
+            data += buf + self._decoder.flush()
+
+        return data
+
     def read(self, amt=None, decode_content=None, cache_content=False):
         """
         Similar to :meth:`httplib.HTTPResponse.read`, but with two additional
@@ -193,12 +223,7 @@ def read(self, amt=None, decode_content=None, cache_content=False):
             after having ``.read()`` the file object. (Overridden if ``amt`` is
             set.)
         """
-        # Note: content-encoding value should be case-insensitive, per RFC 7230
-        # Section 3.2
-        content_encoding = self.headers.get('content-encoding', '').lower()
-        if self._decoder is None:
-            if content_encoding in self.CONTENT_DECODERS:
-                self._decoder = _get_decoder(content_encoding)
+        self._init_decoder()
         if decode_content is None:
             decode_content = self.decode_content
 
@@ -247,17 +272,7 @@ def read(self, amt=None, decode_content=None, cache_content=False):
 
             self._fp_bytes_read += len(data)
 
-            try:
-                if decode_content and self._decoder:
-                    data = self._decoder.decompress(data)
-            except (IOError, zlib.error) as e:
-                raise DecodeError(
-                    "Received response with content-encoding: %s, but "
-                    "failed to decode it." % content_encoding, e)
-
-            if flush_decoder and decode_content and self._decoder:
-                buf = self._decoder.decompress(binary_type())
-                data += buf + self._decoder.flush()
+            data = self._decode(data, decode_content, flush_decoder)
 
             if cache_content:
                 self._body = data
@@ -284,9 +299,10 @@ def stream(self, amt=2**16, decode_content=None):
             If True, will attempt to decode the body based on the
             'content-encoding' header.
         """
+        self._init_decoder()
         if self.chunked:
             for line in self.read_chunked(amt):
-                yield line
+                yield self._decode(line, decode_content, True)
         else:
             while not is_fp_closed(self._fp):
                 data = self.read(amt=amt, decode_content=decode_content)


Test patch:
diff --git a/test/test_response.py b/test/test_response.py
--- a/test/test_response.py
+++ b/test/test_response.py
@@ -418,6 +418,29 @@ def test_mock_transfer_encoding_chunked(self):
             self.assertEqual(c, stream[i])
             i += 1
 
+    def test_mock_gzipped_transfer_encoding_chunked_decoded(self):
+        """Show that we can decode the gizpped and chunked body."""
+        def stream():
+            # Set up a generator to chunk the gzipped body
+            import zlib
+            compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
+            data = compress.compress(b'foobar')
+            data += compress.flush()
+            for i in range(0, len(data), 2):
+                yield data[i:i+2]
+
+        fp = MockChunkedEncodingResponse(list(stream()))
+        r = httplib.HTTPResponse(MockSock)
+        r.fp = fp
+        headers = {'transfer-encoding': 'chunked', 'content-encoding': 'gzip'}
+        resp = HTTPResponse(r, preload_content=False, headers=headers)
+
+        data = b''
+        for c in resp.stream(decode_content=True):
+            data += c
+
+        self.assertEqual(b'foobar', data)
+
     def test_mock_transfer_encoding_chunked_custom_read(self):
         stream = [b"foooo", b"bbbbaaaaar"]
         fp = MockChunkedEncodingResponse(stream)
@@ -517,7 +540,9 @@ def __init__(self, content):
 
     @staticmethod
     def _encode_chunk(chunk):
-        return '%X\r\n%s\r\n' % (len(chunk), chunk.decode())
+        # In the general case, we can't decode the chunk to unicode
+        length = '%X\r\n' % len(chunk)
+        return length.encode() + chunk + b'\r\n'
 
     def _pop_new_chunk(self):
         if self.chunks_exhausted:
@@ -529,8 +554,10 @@ def _pop_new_chunk(self):
             self.chunks_exhausted = True
         else:
             self.index += 1
-        encoded_chunk = self._encode_chunk(chunk)
-        return encoded_chunk.encode()
+        chunk = self._encode_chunk(chunk)
+        if not isinstance(chunk, bytes):
+            chunk = chunk.encode()
+        return chunk
 
     def pop_current_chunk(self, amt=-1, till_crlf=False):
         if amt > 0 and till_crlf: