Repository: urllib3/urllib3

Issue:
This seems to fix the IncompleteRead errors I was getting
I'm sometimes getting IncompleteRead errors when using twython which would bring down the connection.  Found a fix online and it seems to have solved the issue so far.

Fix came from:
http://bobrochel.blogspot.co.nz/2010/11/bad-servers-chunked-encoding-and.html



Discussion:
Thanks @beettlle, can you find a way to write a test for this? Check out some of our tests using dummyserver. I think specifically the socket-level tests would work for this. For example: https://github.com/shazow/urllib3/blob/master/test/with_dummyserver/test_socketlevel.py

I've looked though the examples and can't seem to figure out how to make the server send a partial message.  Is there anyone that might give me some pointers?

I imagine we want something like this...

``` python
   ...

   def test_incomplete_response(self):
        body = 'Response'
        partial_body = body[:2]

        def socket_handler(listener):
            sock = listener.accept()[0]

            # Consume request
            buf = b''
            while not buf.endswith(b'\r\n\r\n'):
                buf = sock.recv(65536)

            # Send partial response and close socket.
            sock.send(
                'HTTP/1.1 200 OK\r\n'
                'Content-Type: text/plain\r\n'
                'Content-Length: %d\r\n'
                '\r\n'
                '%s' % (len(body), partial_body)).encode('utf-8')
            )
            sock.close()

        self._start_server(socket_handler)
        pool = HTTPConnectionPool(self.host, self.port)

        response = pool.request('GET', '/', retries=0)
        self.assertEqual(response.data, partial_body)
```

Seems a bit dangerous to not raise an error in this scenario though. :/ Maybe an exception should be raised with the partial data attached to it?

I know that was just a quick little example and not serious code, but there is a bug here that should definitely be considered before any real implementation:

```
buf = b''
while not buf.endswith(b'\r\n\r\n'):
    buf = sock.recv(65536)
```

If `sock.recv()` consistently returns an empty string at any point (either immediately or before `'\r\n\r\n'` is reached), this will be caught in an infinite loop.

@Anorov Would you be willing to make a PR with a test that shows this behaviour and this fix?


Resolving patch:
diff --git a/urllib3/response.py b/urllib3/response.py
--- a/urllib3/response.py
+++ b/urllib3/response.py
@@ -10,9 +10,10 @@
 from socket import timeout as SocketTimeout
 
 from ._collections import HTTPHeaderDict
-from .exceptions import DecodeError, ReadTimeoutError
+from .exceptions import ConnectionError, DecodeError, ReadTimeoutError
 from .packages.six import string_types as basestring, binary_type
 from .util import is_fp_closed
+from .connection import HTTPException
 
 
 class DeflateDecoder(object):
@@ -202,6 +203,10 @@ def read(self, amt=None, decode_content=None, cache_content=False):
                 # there is yet no clean way to get at it from this context.
                 raise ReadTimeoutError(self._pool, None, 'Read timed out.')
 
+            except HTTPException as e:
+                # This includes IncompleteRead.
+                raise ConnectionError('Connection failed: %r' % e, e)
+
             self._fp_bytes_read += len(data)
 
             try:


Test patch:
diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py
--- a/test/with_dummyserver/test_https.py
+++ b/test/with_dummyserver/test_https.py
@@ -135,11 +135,11 @@ def test_ssl_verified_with_platform_ca_certs(self):
         try:
             import urllib3.contrib.pyopenssl
         except ImportError:
-            raise SkipTest('This test needs pyopenssl support')
+            raise SkipTest('Test requires PyOpenSSL')
         if (urllib3.connection.ssl_wrap_socket is
                 urllib3.contrib.pyopenssl.orig_connection_ssl_wrap_socket):
             # Not patched
-            raise SkipTest('This test should only be run after pyopenssl '
+            raise SkipTest('Test should only be run after PyOpenSSL '
                            'monkey patching')
 
         https_pool = HTTPSConnectionPool('httpbin.org', 443,
diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py
--- a/test/with_dummyserver/test_socketlevel.py
+++ b/test/with_dummyserver/test_socketlevel.py
@@ -2,6 +2,7 @@
 from urllib3 import HTTPConnectionPool, HTTPSConnectionPool
 from urllib3.poolmanager import proxy_from_url
 from urllib3.exceptions import (
+        ConnectionError,
         MaxRetryError,
         ProxyError,
         ReadTimeoutError,
@@ -16,7 +17,6 @@
 from nose.plugins.skip import SkipTest
 from threading import Event
 import socket
-import time
 import ssl
 
 
@@ -196,6 +196,34 @@ def socket_handler(listener):
         self.assertRaises(ReadTimeoutError, response.read)
         timed_out.set()
 
+    def test_incomplete_response(self):
+        body = 'Response'
+        partial_body = body[:2]
+
+        def socket_handler(listener):
+            sock = listener.accept()[0]
+
+            # Consume request
+            buf = b''
+            while not buf.endswith(b'\r\n\r\n'):
+                buf = sock.recv(65536)
+
+            # Send partial response and close socket.
+            sock.send((
+                'HTTP/1.1 200 OK\r\n'
+                'Content-Type: text/plain\r\n'
+                'Content-Length: %d\r\n'
+                '\r\n'
+                '%s' % (len(body), partial_body)).encode('utf-8')
+            )
+            sock.close()
+
+        self._start_server(socket_handler)
+        pool = HTTPConnectionPool(self.host, self.port)
+
+        response = pool.request('GET', '/', retries=0, preload_content=False)
+        self.assertRaises(ConnectionError, response.read)
+
 
 class TestProxyManager(SocketDummyServerTestCase):