Repository: urllib3/urllib3

Issue:
Maybe it should be explicitly documented that iterating over an HTTPResponse is bad
HTTPResponse inherits from io.IOBase, meaning it's iterable. By python's documentation this means it iterates over lines in the response.

The problem is that it calls IOBase's readline, which in turn **calls the object's read method with size of 1, until it gets a newline.**

While discovering this when sending a huge payload and figuring out why its slow, this might not be obvious to people using the library and by accident calling:
for chunk in response:
  ...
instead of
for chunk in response.stream(sane_size):
  ...
Maybe iterating over the response should be disallowed, or documented that it's bad to do so.


Discussion:

Resolving patch:
diff --git a/src/urllib3/response.py b/src/urllib3/response.py
--- a/src/urllib3/response.py
+++ b/src/urllib3/response.py
@@ -705,3 +705,20 @@ def geturl(self):
             return self.retries.history[-1].redirect_location
         else:
             return self._request_url
+
+    def __iter__(self):
+        buffer = [b""]
+        for chunk in self.stream(decode_content=True):
+            if b"\n" in chunk:
+                chunk = chunk.split(b"\n")
+                yield b"".join(buffer) + chunk[0] + b"\n"
+                for x in chunk[1:-1]:
+                    yield x + b"\n"
+                if chunk[-1]:
+                    buffer = [chunk[-1]]
+                else:
+                    buffer = []
+            else:
+                buffer.append(chunk)
+        if buffer:
+            yield b"".join(buffer)


Test patch:
diff --git a/test/test_response.py b/test/test_response.py
--- a/test/test_response.py
+++ b/test/test_response.py
@@ -708,6 +708,41 @@ def test_geturl_retries(self):
         resp = HTTPResponse(fp, retries=retry)
         assert resp.geturl() == 'https://www.example.com'
 
+    @pytest.mark.parametrize(
+        ["payload", "expected_stream"],
+        [(b"", [b""]),
+         (b"\n", [b"\n"]),
+         (b"abc\ndef", [b"abc\n", b"def"]),
+         (b"Hello\nworld\n\n\n!", [b"Hello\n", b"world\n", b"\n", b"\n", b"!"])]
+    )
+    def test__iter__(self, payload, expected_stream):
+        actual_stream = []
+        for chunk in HTTPResponse(BytesIO(payload), preload_content=False):
+            actual_stream.append(chunk)
+
+        assert actual_stream == expected_stream
+
+    def test__iter__decode_content(self):
+        def stream():
+            # Set up a generator to chunk the gzipped body
+            compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
+            data = compress.compress(b'foo\nbar')
+            data += compress.flush()
+            for i in range(0, len(data), 2):
+                yield data[i:i + 2]
+
+        fp = MockChunkedEncodingResponse(list(stream()))
+        r = httplib.HTTPResponse(MockSock)
+        r.fp = fp
+        headers = {'transfer-encoding': 'chunked', 'content-encoding': 'gzip'}
+        resp = HTTPResponse(r, preload_content=False, headers=headers)
+
+        data = b''
+        for c in resp:
+            data += c
+
+        assert b'foo\nbar' == data
+
 
 class MockChunkedEncodingResponse(object):
 
diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py
--- a/test/with_dummyserver/test_connectionpool.py
+++ b/test/with_dummyserver/test_connectionpool.py
@@ -512,6 +512,23 @@ def test_post_with_multipart(self):
 
             self.assertEqual(body[i], expected_body[i])
 
+    def test_post_with_multipart__iter__(self):
+        data = {'hello': 'world'}
+        r = self.pool.request('POST', '/echo',
+                              fields=data,
+                              preload_content=False,
+                              multipart_boundary="boundary",
+                              encode_multipart=True)
+
+        chunks = [chunk for chunk in r]
+        assert chunks == [
+            b"--boundary\r\n",
+            b'Content-Disposition: form-data; name="hello"\r\n',
+            b'\r\n',
+            b'world\r\n',
+            b"--boundary--\r\n"
+        ]
+
     def test_check_gzip(self):
         r = self.pool.request('GET', '/encodingrequest',
                               headers={'accept-encoding': 'gzip'})