Repository: aio-libs/aiohttp

Issue:
Tests for new proxy connectot
1. we need tests
2. we should think about connection keep-alive for proxy.
3. probably we should remove use_proxy from ClientRequest and modify path in connector's connect() method.



Discussion:

Resolving patch:
diff --git a/aiohttp/client.py b/aiohttp/client.py
--- a/aiohttp/client.py
+++ b/aiohttp/client.py
@@ -94,12 +94,11 @@ def request(method, url, *,
         connector = aiohttp.TCPConnector(loop=loop)
 
     while True:
-        using_proxy = isinstance(connector, aiohttp.ProxyConnector)
         req = request_class(
             method, url, params=params, headers=headers, data=data,
             cookies=cookies, files=files, auth=auth, encoding=encoding,
             version=version, compress=compress, chunked=chunked,
-            loop=loop, expect100=expect100, using_proxy=using_proxy,
+            loop=loop, expect100=expect100,
             response_class=response_class)
 
         try:
@@ -174,8 +173,7 @@ def __init__(self, method, url, *,
                  params=None, headers=None, data=None, cookies=None,
                  files=None, auth=None, encoding='utf-8', version=(1, 1),
                  compress=None, chunked=None, expect100=False,
-                 verify_ssl=True, loop=None, using_proxy=False,
-                 response_class=None):
+                 verify_ssl=True, loop=None, response_class=None):
         self.url = url
         self.method = method.upper()
         self.encoding = encoding
@@ -183,7 +181,6 @@ def __init__(self, method, url, *,
         self.compress = compress
         self.verify_ssl = verify_ssl
         self.loop = loop
-        self.using_proxy = using_proxy
         self.response_class = response_class or ClientResponse
 
         self.update_version(version)
@@ -281,13 +278,8 @@ def update_path(self, params, data):
             else:
                 query = params
 
-        if not self.using_proxy:
-            scheme = ''
-            netloc = ''
-
         self.path = urllib.parse.urlunsplit(
-            (scheme, netloc,
-             urllib.parse.quote(path, safe='/%'), query, fragment))
+            ('', '', urllib.parse.quote(path, safe='/%'), query, fragment))
 
     def update_headers(self, headers):
         """Update request headers."""
diff --git a/aiohttp/connector.py b/aiohttp/connector.py
--- a/aiohttp/connector.py
+++ b/aiohttp/connector.py
@@ -261,12 +261,20 @@ def __init__(self, proxy, *args, **kwargs):
     @asyncio.coroutine
     def _create_connection(self, req, **kwargs):
         proxy_req = ClientRequest('GET', self.proxy,
-                                  headers={'Host': req.host})
-
+                                  headers={'Host': req.host},
+                                  loop=self._loop)
         try:
             transport, proto = yield from super()._create_connection(proxy_req)
         except OSError:
             raise ProxyConnectionError()
+        req.path = '{scheme}://{host}{path}'.format(scheme=req.scheme,
+                                                    host=req.host,
+                                                    path=req.path)
+        if proxy_req.auth:
+            auth = proxy_req.headers['AUTHORIZATION']
+            del proxy_req.headers['AUTHORIZATION']
+            req.headers['PROXY-AUTHORIZATION'] = auth
+            proxy_req.headers['PROXY-AUTHORIZATION'] = auth
 
         if req.ssl:
             # For HTTPS requests over HTTP proxy
diff --git a/aiohttp/errors.py b/aiohttp/errors.py
--- a/aiohttp/errors.py
+++ b/aiohttp/errors.py
@@ -4,7 +4,7 @@
            'HttpBadRequest', 'HttpMethodNotAllowed',
            'IncompleteRead', 'BadStatusLine', 'LineTooLong', 'InvalidHeader',
            'ConnectionError', 'OsConnectionError', 'ClientConnectionError',
-           'TimeoutError', 'ProxyConnectionError']
+           'TimeoutError', 'ProxyConnectionError', 'HttpProxyError']
 
 from asyncio import TimeoutError
 


Test patch:
diff --git a/tests/test_client.py b/tests/test_client.py
--- a/tests/test_client.py
+++ b/tests/test_client.py
@@ -204,10 +204,6 @@ def test_no_path(self):
         req = ClientRequest('get', 'http://python.org')
         self.assertEqual('/', req.path)
 
-    def test_path_with_proxy(self):
-        req = ClientRequest('get', 'http://python.org', using_proxy=True)
-        self.assertEqual('http://python.org/', req.path)
-
     def test_basic_auth(self):
         req = ClientRequest('get', 'http://python.org', auth=('nkim', '1234'))
         self.assertIn('AUTHORIZATION', req.headers)
diff --git a/tests/test_connector.py b/tests/test_connector.py
--- a/tests/test_connector.py
+++ b/tests/test_connector.py
@@ -292,7 +292,206 @@ def tearDown(self):
         self.loop.close()
         gc.collect()
 
+    def _fake_coroutine(self, mock, return_value):
+
+        def coro(*args, **kw):
+            if isinstance(return_value, Exception):
+                raise return_value
+            return return_value
+            yield
+        mock.side_effect = coro
+
     def test_ctor(self):
         with self.assertRaises(AssertionError):
-            aiohttp.connector.ProxyConnector('https://localhost:8118',
-                                             loop=self.loop)
+            aiohttp.ProxyConnector('https://localhost:8118', loop=self.loop)
+
+    @unittest.mock.patch('aiohttp.connector.ClientRequest')
+    def test_connect(self, ClientRequestMock):
+        req = ClientRequest('GET', 'http://www.python.org')
+        self.assertEqual(req.path, '/')
+
+        loop_mock = unittest.mock.Mock()
+        connector = aiohttp.ProxyConnector('http://proxy.example.com',
+                                           loop=loop_mock)
+        self.assertIs(loop_mock, connector._loop)
+
+        resolve_host = unittest.mock.Mock()
+        self._fake_coroutine(resolve_host, [unittest.mock.MagicMock()])
+        connector._resolve_host = resolve_host
+
+        tr, proto = unittest.mock.Mock(), unittest.mock.Mock()
+        self._fake_coroutine(loop_mock.create_connection, (tr, proto))
+        conn = self.loop.run_until_complete(connector.connect(req))
+        self.assertEqual(req.path, 'http://www.python.org/')
+        self.assertIs(conn._transport, tr)
+        self.assertIs(conn._protocol, proto)
+
+        # resolve_host.assert_called_once_with('proxy.example.com', 80)
+        self.assertEqual(tr.mock_calls, [])
+
+        ClientRequestMock.assert_called_with(
+            'GET', 'http://proxy.example.com',
+            headers={'Host': 'www.python.org'},
+            loop=loop_mock)
+
+    def test_proxy_connection_error(self):
+        connector = aiohttp.ProxyConnector('http://proxy.example.com',
+                                           loop=self.loop)
+        connector._resolve_host = resolve_mock = unittest.mock.Mock()
+        self._fake_coroutine(resolve_mock, OSError('dont take it serious'))
+
+        req = ClientRequest('GET', 'http://www.python.org')
+        expected_headers = dict(req.headers)
+        with self.assertRaises(aiohttp.ProxyConnectionError):
+            self.loop.run_until_complete(connector.connect(req))
+        self.assertEqual(req.path, '/')
+        self.assertEqual(dict(req.headers), expected_headers)
+
+    @unittest.mock.patch('aiohttp.connector.ClientRequest')
+    def test_auth(self, ClientRequestMock):
+        proxy_req = ClientRequest('GET', 'http://user:pass@proxy.example.com')
+        ClientRequestMock.return_value = proxy_req
+        self.assertIn('AUTHORIZATION', proxy_req.headers)
+        self.assertNotIn('PROXY-AUTHORIZATION', proxy_req.headers)
+
+        loop_mock = unittest.mock.Mock()
+        connector = aiohttp.ProxyConnector(
+            'http://user:pass@proxy.example.com', loop=loop_mock)
+        connector._resolve_host = resolve_mock = unittest.mock.Mock()
+        self._fake_coroutine(resolve_mock, [unittest.mock.MagicMock()])
+
+        tr, proto = unittest.mock.Mock(), unittest.mock.Mock()
+        self._fake_coroutine(loop_mock.create_connection, (tr, proto))
+
+        req = ClientRequest('GET', 'http://www.python.org')
+        self.assertNotIn('AUTHORIZATION', req.headers)
+        self.assertNotIn('PROXY-AUTHORIZATION', req.headers)
+        self.loop.run_until_complete(connector.connect(req))
+
+        self.assertEqual(req.path, 'http://www.python.org/')
+        self.assertNotIn('AUTHORIZATION', req.headers)
+        self.assertIn('PROXY-AUTHORIZATION', req.headers)
+        self.assertNotIn('AUTHORIZATION', proxy_req.headers)
+        self.assertIn('PROXY-AUTHORIZATION', proxy_req.headers)
+
+    @unittest.mock.patch('aiohttp.connector.ClientRequest')
+    def test_auth__not_modifying_request(self, ClientRequestMock):
+        proxy_req = ClientRequest('GET', 'http://user:pass@proxy.example.com')
+        ClientRequestMock.return_value = proxy_req
+        proxy_req_headers = dict(proxy_req.headers)
+
+        loop_mock = unittest.mock.Mock()
+        connector = aiohttp.ProxyConnector(
+            'http://user:pass@proxy.example.com', loop=loop_mock)
+        connector._resolve_host = resolve_mock = unittest.mock.Mock()
+        self._fake_coroutine(resolve_mock, OSError('nothing personal'))
+
+        req = ClientRequest('GET', 'http://www.python.org')
+        req_headers = dict(req.headers)
+        with self.assertRaises(aiohttp.ProxyConnectionError):
+            self.loop.run_until_complete(connector.connect(req))
+        self.assertEqual(req.headers, req_headers)
+        self.assertEqual(req.path, '/')
+        self.assertEqual(proxy_req.headers, proxy_req_headers)
+
+    @unittest.mock.patch('aiohttp.connector.ClientRequest')
+    def test_https_connect(self, ClientRequestMock):
+        loop_mock = unittest.mock.Mock()
+        proxy_req = ClientRequest('GET', 'http://proxy.example.com',
+                                  loop=loop_mock)
+        ClientRequestMock.return_value = proxy_req
+
+        proxy_resp = ClientResponse('get', 'http://proxy.example.com')
+        proxy_req.send = send_mock = unittest.mock.Mock()
+        send_mock.return_value = proxy_resp
+        proxy_resp.start = start_mock = unittest.mock.Mock()
+        self._fake_coroutine(start_mock, unittest.mock.Mock(status=200))
+
+        connector = aiohttp.ProxyConnector(
+            'http://proxy.example.com', loop=loop_mock)
+
+        tr, proto = unittest.mock.Mock(), unittest.mock.Mock()
+        self._fake_coroutine(loop_mock.create_connection, (tr, proto))
+
+        req = ClientRequest('GET', 'https://www.python.org')
+        self.loop.run_until_complete(connector._create_connection(req))
+
+        self.assertEqual(proxy_req.method, 'CONNECT')
+        self.assertEqual(proxy_req.path, 'www.python.org:443')
+        tr.pause_reading.assert_called_once_with()
+        tr.get_extra_info.assert_called_once_with('socket', default=None)
+
+    @unittest.mock.patch('aiohttp.connector.ClientRequest')
+    def test_https_connect_runtime_error(self, ClientRequestMock):
+        loop_mock = unittest.mock.Mock()
+        proxy_req = ClientRequest('GET', 'http://proxy.example.com',
+                                  loop=loop_mock)
+        ClientRequestMock.return_value = proxy_req
+
+        proxy_resp = ClientResponse('get', 'http://proxy.example.com')
+        proxy_req.send = send_mock = unittest.mock.Mock()
+        send_mock.return_value = proxy_resp
+        proxy_resp.start = start_mock = unittest.mock.Mock()
+        self._fake_coroutine(start_mock, unittest.mock.Mock(status=200))
+
+        connector = aiohttp.ProxyConnector(
+            'http://proxy.example.com', loop=loop_mock)
+
+        tr, proto = unittest.mock.Mock(), unittest.mock.Mock()
+        tr.get_extra_info.return_value = None
+        self._fake_coroutine(loop_mock.create_connection, (tr, proto))
+
+        req = ClientRequest('GET', 'https://www.python.org')
+        with self.assertRaisesRegex(RuntimeError,
+                "Transport does not expose socket instance"):
+            self.loop.run_until_complete(connector._create_connection(req))
+
+    @unittest.mock.patch('aiohttp.connector.ClientRequest')
+    def test_https_connect_http_proxy_error(self, ClientRequestMock):
+        loop_mock = unittest.mock.Mock()
+        proxy_req = ClientRequest('GET', 'http://proxy.example.com',
+                                  loop=loop_mock)
+        ClientRequestMock.return_value = proxy_req
+
+        proxy_resp = ClientResponse('get', 'http://proxy.example.com')
+        proxy_req.send = send_mock = unittest.mock.Mock()
+        send_mock.return_value = proxy_resp
+        proxy_resp.start = start_mock = unittest.mock.Mock()
+        self._fake_coroutine(
+            start_mock, unittest.mock.Mock(status=400, reason='bad request'))
+
+        connector = aiohttp.ProxyConnector(
+            'http://proxy.example.com', loop=loop_mock)
+
+        tr, proto = unittest.mock.Mock(), unittest.mock.Mock()
+        tr.get_extra_info.return_value = None
+        self._fake_coroutine(loop_mock.create_connection, (tr, proto))
+
+        req = ClientRequest('GET', 'https://www.python.org')
+        with self.assertRaisesRegex(
+                aiohttp.HttpProxyError, "(400, 'bad request')"):
+            self.loop.run_until_complete(connector._create_connection(req))
+
+    @unittest.mock.patch('aiohttp.connector.ClientRequest')
+    def test_https_connect_resp_start_error(self, ClientRequestMock):
+        loop_mock = unittest.mock.Mock()
+        proxy_req = ClientRequest('GET', 'http://proxy.example.com',
+                                  loop=loop_mock)
+        ClientRequestMock.return_value = proxy_req
+
+        proxy_resp = ClientResponse('get', 'http://proxy.example.com')
+        proxy_req.send = send_mock = unittest.mock.Mock()
+        send_mock.return_value = proxy_resp
+        proxy_resp.start = start_mock = unittest.mock.Mock()
+        self._fake_coroutine(start_mock, OSError("error message"))
+
+        connector = aiohttp.ProxyConnector(
+            'http://proxy.example.com', loop=loop_mock)
+
+        tr, proto = unittest.mock.Mock(), unittest.mock.Mock()
+        tr.get_extra_info.return_value = None
+        self._fake_coroutine(loop_mock.create_connection, (tr, proto))
+
+        req = ClientRequest('GET', 'https://www.python.org')
+        with self.assertRaisesRegex(OSError, "error message"):
+            self.loop.run_until_complete(connector._create_connection(req))