Repository: aio-libs/aiohttp

Issue:
Stall when connector hits limit from several connections to same bad hostname
## Long story short

When a session's connector has a connection limit, multiple requests for a URL with a bad hostname never complete.
## Expected behaviour

Script should run to completion
## Actual behaviour

The second connection neither completes nor throws an error, and the script hangs.
## Steps to reproduce

``` python
import aiohttp
import asyncio
import sys
import traceback

@asyncio.coroutine                                                                                  
def stall(session):
    for c in asyncio.as_completed([session.get('http://badhost.example/') for i in range(2)]):
        print(c)
        try:
            resp = yield from c
        except:
            for line in traceback.format_exception_only(*sys.exc_info()[0:2]):
                print(line, end='', file=sys.stderr)
        else:
            yield from resp.release()

loop = asyncio.get_event_loop()
with aiohttp.ClientSession(connector=aiohttp.TCPConnector(limit=1)) as session:
    loop.run_until_complete(stall(session))
loop.close()
```
## Your environment

python 3.4.2 on Debian jessie
aiohttp 0.21.4



Discussion:
I don't think it is only bad hostname which leads to this situation. With the following code:

```
import asyncio
import logging
import aiohttp

LOG_FORMAT = ('%(asctime)s %(message)s')
logging.basicConfig(format=LOG_FORMAT)
log = logging.getLogger('root')  # pylint: disable=I0011,C0103
log.setLevel('DEBUG')


async def fetch(session, url):
    "single fetch"
    log.info('trying %s', url)
    try:
        with aiohttp.Timeout(5):
            async with session.get(url, compress=True) as response:
                if response.status == 200:
                    rs =  await response.text()
                    log.info('status 200  %s response size %s', url, len(rs))
                else:
                    log.error('received %s for %s', response.status, url)
    except aiohttp.errors.ClientError as exc:
        log.error(exc)
    except asyncio.TimeoutError as exc:
        log.error('fetching data timeout for %s', url)

async def fetch_all(session, urls, loop):
    """This a delegating routine."""
    futures = [loop.create_task(fetch(session, url)) for url in urls]
    for future in asyncio.as_completed(futures):
        result = await future
        if result is not None:
            pass


def main():
    """main code"""
    ssl_conn = aiohttp.TCPConnector(conn_timeout=2, limit=1, verify_ssl=False)
    sites = ['google.com', 'yahoo.com', 'facebook.com', 'cnn.com']
    urls = ['https://{}:443/'.format(site) for site in sites]

    loop = asyncio.get_event_loop()
    with aiohttp.ClientSession(loop=loop, connector=ssl_conn) as session:
        loop.run_until_complete(fetch_all(session, urls, loop))

# This is the standard boilerplate that calls the main() function.
if __name__ == '__main__':
    main()
```

I see that request to yahoo receives  timeout exception (catched by  try block) but in the tcpdump trace I see actual response from yahoo (it responses with 301).

```
2016-03-27 12:08:58,578 trying https://google.com:443/
2016-03-27 12:08:58,582 trying https://yahoo.com:443/
2016-03-27 12:08:58,583 trying https://facebook.com:443/
2016-03-27 12:08:58,584 trying https://cnn.com:443/
2016-03-27 12:08:58,771 received 400 for https://google.com:443/
2016-03-27 12:08:58,859 [Errno None] Cannot connect to host cnn.com:443 ssl:True [Can not connect to cnn.com:443 [None]]
2016-03-27 12:08:59,856 status 200  https://facebook.com:443/ response size 69800
2016-03-27 12:09:03,587 fetching data timeout for https://yahoo.com:443/
```

When I remove cnn from the url list and remove also 443 then I see:

```
2016-03-27 12:21:41,845 trying https://google.com/
2016-03-27 12:21:41,853 trying https://yahoo.com/
2016-03-27 12:21:41,856 trying https://facebook.com/
2016-03-27 12:21:42,032 received 400 for https://google.com/
2016-03-27 12:21:43,038 status 200  https://facebook.com/ response size 69651
2016-03-27 12:21:46,857 fetching data timeout for https://yahoo.com/
```

running curl -vv   https://yahoo.com:443/ or curl -vv   https://yahoo.com returns 301 to www.yahoo.com.


Resolving patch:
diff --git a/aiohttp/connector.py b/aiohttp/connector.py
--- a/aiohttp/connector.py
+++ b/aiohttp/connector.py
@@ -297,27 +297,33 @@ def connect(self, req):
             # This connection will now count towards the limit.
             waiters.append(fut)
 
-            yield from fut
-
-        transport, proto = self._get(key)
-        if transport is None:
-            try:
-                if self._conn_timeout:
-                    transport, proto = yield from asyncio.wait_for(
-                        self._create_connection(req),
-                        self._conn_timeout, loop=self._loop)
-                else:
-                    transport, proto = yield from self._create_connection(req)
-
-            except asyncio.TimeoutError as exc:
-                raise ClientTimeoutError(
-                    'Connection timeout to host {0[0]}:{0[1]} ssl:{0[2]}'
-                    .format(key)) from exc
-            except OSError as exc:
-                raise ClientOSError(
-                    exc.errno,
-                    'Cannot connect to host {0[0]}:{0[1]} ssl:{0[2]} [{1}]'
-                    .format(key, exc.strerror)) from exc
+        try:
+            if limit is not None:
+                yield from fut
+
+            transport, proto = self._get(key)
+            if transport is None:
+                try:
+                    if self._conn_timeout:
+                        transport, proto = yield from asyncio.wait_for(
+                            self._create_connection(req),
+                            self._conn_timeout, loop=self._loop)
+                    else:
+                        transport, proto = \
+                            yield from self._create_connection(req)
+
+                except asyncio.TimeoutError as exc:
+                    raise ClientTimeoutError(
+                        'Connection timeout to host {0[0]}:{0[1]} ssl:{0[2]}'
+                        .format(key)) from exc
+                except OSError as exc:
+                    raise ClientOSError(
+                        exc.errno,
+                        'Cannot connect to host {0[0]}:{0[1]} ssl:{0[2]} [{1}]'
+                        .format(key, exc.strerror)) from exc
+        except:
+            self._release_waiter(key)
+            raise
 
         self._acquired[key].add(transport)
         conn = Connection(self, key, req, transport, proto, self._loop)
@@ -344,6 +350,14 @@ def _get(self, key):
         del self._conns[key]
         return None, None
 
+    def _release_waiter(self, key):
+        waiters = self._waiters[key]
+        while waiters:
+            waiter = waiters.pop(0)
+            if not waiter.done():
+                waiter.set_result(None)
+                break
+
     def _release(self, key, req, transport, protocol, *, should_close=False):
         if self._closed:
             # acquired connection is already released on connector closing
@@ -358,12 +372,7 @@ def _release(self, key, req, transport, protocol, *, should_close=False):
             pass
         else:
             if self._limit is not None and len(acquired) < self._limit:
-                waiters = self._waiters[key]
-                while waiters:
-                    waiter = waiters.pop(0)
-                    if not waiter.done():
-                        waiter.set_result(None)
-                        break
+                self._release_waiter(key)
 
         resp = req.response
 


Test patch:
diff --git a/tests/test_connector.py b/tests/test_connector.py
--- a/tests/test_connector.py
+++ b/tests/test_connector.py
@@ -560,11 +560,28 @@ class Req:
                 # limit exhausted
                 yield from asyncio.wait_for(conn.connect(Req), 0.01,
                                             loop=self.loop)
-
             connection.close()
-
         self.loop.run_until_complete(go())
 
+    def test_connect_with_limit_release_waiters(self):
+
+        def check_with_exc(err):
+            conn = aiohttp.BaseConnector(limit=1, loop=self.loop)
+            conn._create_connection = unittest.mock.Mock()
+            conn._create_connection.return_value = \
+                asyncio.Future(loop=self.loop)
+            conn._create_connection.return_value.set_exception(err)
+
+            with self.assertRaises(Exception):
+                req = unittest.mock.Mock()
+                self.loop.run_until_complete(conn.connect(req))
+            key = (req.host, req.port, req.ssl)
+            self.assertFalse(conn._waiters[key])
+
+        check_with_exc(OSError(1, 'permission error'))
+        check_with_exc(RuntimeError())
+        check_with_exc(asyncio.TimeoutError())
+
     def test_connect_with_limit_concurrent(self):
 
         @asyncio.coroutine