Repository: urllib3/urllib3

Issue:
Likely bug in retry/release-conn code, need tests
In this change: https://github.com/shazow/urllib3/pull/647/files#diff-211abff6a07837e4a89a8663a89d2c84R582

@jlatherfold writes:

> For the (TimeoutError, HTTPException, SocketError, ConnectionError) exception handler you don't raise (presumably because there may be retries) and you will go on to release the conn and make another call to urlopen (if retries). But you call it with release_conn=True (since you've just set it to true) - what happens if that call succeeds? You'll release the conn back to the pool before the response is read. At least that's what it looks like to me.



Discussion:
I will try and confirm this next week sometime, but writing a test to force it might not be so easy.

Take a look at the socketlevel tests in #647, should be some good examples.

Hi,

I ran a few trial runs with your changes and set read timeout low enough to force the issue. I set retries=1 and got a lot of retries that succeeded so the problem that I thought would occur didn't - I couldn't confirm that this is an issue. You could probably be certain if you did something like this (ie. use a local varaible instead of release_conn):

put_conn_on_error = False
try:
    # make call...
except:
    conn = conn and conn.close()
    put_conn_on_error = True
finally:
    if release_conn or put_conn_on_error:
        self._put_conn(conn)

if retries:
    # try again...

Anyway, it seems ok as it is and I won't be doing any further tests on this for now.

Maybe, though not as certain as actually having a test in the suite which exercises this scenario. :) We'll see if we can add that later, thanks for trying!


Resolving patch:
diff --git a/urllib3/connectionpool.py b/urllib3/connectionpool.py
--- a/urllib3/connectionpool.py
+++ b/urllib3/connectionpool.py
@@ -550,6 +550,17 @@ def urlopen(self, method, url, body=None, headers=None, retries=None,
 
         conn = None
 
+        # Track whether `conn` needs to be released before
+        # returning/raising/recursing. Update this variable if necessary, and
+        # leave `release_conn` constant throughout the function. That way, if
+        # the function recurses, the original value of `release_conn` will be
+        # passed down into the recursive call, and its value will be respected.
+        #
+        # See issue #651 [1] for details.
+        #
+        # [1] <https://github.com/shazow/urllib3/issues/651>
+        release_this_conn = release_conn
+
         # Merge the proxy headers. Only do this in HTTP. We have to copy the
         # headers dict so we can safely change it without those changes being
         # reflected in anyone else's copy.
@@ -638,9 +649,9 @@ def urlopen(self, method, url, body=None, headers=None, retries=None,
                 # Close the connection, set the variable to None, and make sure
                 # we put the None back in the pool to avoid leaking it.
                 conn = conn and conn.close()
-                release_conn = True
+                release_this_conn = True
 
-            if release_conn:
+            if release_this_conn:
                 # Put the connection back to be reused. If the connection is
                 # expired then it will be None, which will get replaced with a
                 # fresh connection during _get_conn.
diff --git a/urllib3/response.py b/urllib3/response.py
--- a/urllib3/response.py
+++ b/urllib3/response.py
@@ -165,6 +165,10 @@ def data(self):
         if self._fp:
             return self.read(cache_content=True)
 
+    @property
+    def connection(self):
+        return self._connection
+
     def tell(self):
         """
         Obtain the number of bytes pulled over the wire so far. May differ from


Test patch:
diff --git a/test/test_connectionpool.py b/test/test_connectionpool.py
--- a/test/test_connectionpool.py
+++ b/test/test_connectionpool.py
@@ -1,3 +1,5 @@
+from __future__ import absolute_import
+
 import unittest
 
 from urllib3.connectionpool import (
@@ -6,6 +8,7 @@
     HTTPConnectionPool,
     HTTPSConnectionPool,
 )
+from urllib3.response import httplib
 from urllib3.util.timeout import Timeout
 from urllib3.packages.ssl_match_hostname import CertificateError
 from urllib3.exceptions import (
@@ -16,7 +19,10 @@
     MaxRetryError,
     ProtocolError,
     SSLError,
+    TimeoutError,
 )
+from urllib3._collections import HTTPHeaderDict
+from .test_response import MockChunkedEncodingResponse, MockSock
 
 from socket import error as SocketError
 from ssl import SSLError as BaseSSLError
@@ -309,6 +315,58 @@ def kaboom(*args, **kwargs):
         new_pool_size = c.pool.qsize()
         self.assertEqual(initial_pool_size, new_pool_size)
 
+    def test_release_conn_param_is_respected_after_http_error_retry(self):
+        """For successful ```urlopen(release_conn=False)```, the connection isn't released, even after a retry.
+
+        This is a regression test for issue #651 [1], where the connection
+        would be released if the initial request failed, even if a retry
+        succeeded.
+
+        [1] <https://github.com/shazow/urllib3/issues/651>
+        """
+
+        class _raise_once_make_request_function(object):
+            """Callable that can mimic `_make_request()`.
+
+            Raises the given exception on its first call, but returns a
+            successful response on subsequent calls.
+            """
+            def __init__(self, ex):
+                super(_raise_once_make_request_function, self).__init__()
+                self._ex = ex
+
+            def __call__(self, *args, **kwargs):
+                if self._ex:
+                    ex, self._ex = self._ex, None
+                    raise ex()
+                response = httplib.HTTPResponse(MockSock)
+                response.fp = MockChunkedEncodingResponse([b'f', b'o', b'o'])
+                response.headers = response.msg = HTTPHeaderDict()
+                return response
+
+        def _test(exception):
+            pool = HTTPConnectionPool(host='localhost', maxsize=1, block=True)
+
+            # Verify that the request succeeds after two attempts, and that the
+            # connection is left on the response object, instead of being
+            # released back into the pool.
+            pool._make_request = _raise_once_make_request_function(exception)
+            response = pool.urlopen('GET', '/', retries=1,
+                                    release_conn=False, preload_content=False,
+                                    chunked=True)
+            self.assertEqual(pool.pool.qsize(), 0)
+            self.assertEqual(pool.num_connections, 2)
+            self.assertTrue(response.connection is not None)
+
+            response.release_conn()
+            self.assertEqual(pool.pool.qsize(), 1)
+            self.assertTrue(response.connection is None)
+
+        # Run the test case for all the retriable exceptions.
+        _test(TimeoutError)
+        _test(HTTPException)
+        _test(SocketError)
+        _test(ProtocolError)
 
 
 if __name__ == '__main__':
diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py
--- a/test/with_dummyserver/test_socketlevel.py
+++ b/test/with_dummyserver/test_socketlevel.py
@@ -17,7 +17,7 @@
 from urllib3.util.retry import Retry
 from urllib3._collections import HTTPHeaderDict, OrderedDict
 
-from dummyserver.testcase import SocketDummyServerTestCase
+from dummyserver.testcase import SocketDummyServerTestCase, consume_socket
 from dummyserver.server import (
     DEFAULT_CERTS, DEFAULT_CA, get_unreachable_address)
 
@@ -507,6 +507,66 @@ def socket_handler(listener):
         if not successful:
             self.fail("Timed out waiting for connection close")
 
+    def test_release_conn_param_is_respected_after_timeout_retry(self):
+        """For successful ```urlopen(release_conn=False)```, the connection isn't released, even after a retry.
+
+        This test allows a retry: one request fails, the next request succeeds.
+
+        This is a regression test for issue #651 [1], where the connection
+        would be released if the initial request failed, even if a retry
+        succeeded.
+
+        [1] <https://github.com/shazow/urllib3/issues/651>
+        """
+        def socket_handler(listener):
+            sock = listener.accept()[0]
+            consume_socket(sock)
+
+            # Close the connection, without sending any response (not even the
+            # HTTP status line). This will trigger a `Timeout` on the client,
+            # inside `urlopen()`.
+            sock.close()
+
+            # Expect a new request. Because we don't want to hang this thread,
+            # we actually use select.select to confirm that a new request is
+            # coming in: this lets us time the thread out.
+            rlist, _, _ = select.select([listener], [], [], 5)
+            assert rlist
+            sock = listener.accept()[0]
+            consume_socket(sock)
+
+            # Send complete chunked response.
+            sock.send((
+                'HTTP/1.1 200 OK\r\n'
+                'Content-Type: text/plain\r\n'
+                'Transfer-Encoding: chunked\r\n'
+                '\r\n'
+                '8\r\n'
+                '12345678\r\n'
+                '0\r\n\r\n').encode('utf-8')
+            )
+
+            sock.close()
+
+        self._start_server(socket_handler)
+        with HTTPConnectionPool(self.host, self.port, maxsize=1) as pool:
+            # First request should fail, but the timeout and `retries=1` should
+            # save it.
+            response = pool.urlopen('GET', '/', retries=1,
+                                    release_conn=False, preload_content=False,
+                                    timeout=Timeout(connect=1, read=0.001))
+
+            # The connection should still be on the response object, and none
+            # should be in the pool. We opened two though.
+            self.assertEqual(pool.num_connections, 2)
+            self.assertEqual(pool.pool.qsize(), 0)
+            self.assertTrue(response.connection is not None)
+
+            # Consume the data. This should put the connection back.
+            response.read()
+            self.assertEqual(pool.pool.qsize(), 1)
+            self.assertTrue(response.connection is None)
+
 
 class TestProxyManager(SocketDummyServerTestCase):