Repository: aio-libs/aiohttp

Issue:
Inconsistent support of IDNA hostnames in Client
## Long story short

aiohttp's client handle IDNA hostnames in a way that seems inconsistent: the ``Host`` header always contains a dedcoded utf-8 value which seems problematic.

For instance:
- ``session.get("http://éé.com/")`` makes a request with ``Host: éé.com``
- ``session.get("http://xn--9caa.com/")`` also makes a request with ``Host: éé.com``.

While it's unclear to me if an unicode hostname should always be IDNA encoded (see bellow), it should at least not be decoded when explicitly encoded by the caller.

### IDNA or not?
The newest HTTP/1 RFCs doesn't specify the encoding of the headers, but recommend to handle them as US-ASCII characters only for security reasons (see: https://tools.ietf.org/html/rfc7230#section-3, especially the last paragraph of 3.2.4).
Most of the resources I read from the W3C or the IETF (normative or not) tells that the hostname should always be encoded, for instance, https://www.w3.org/International/articles/idn-and-iri/#resolvedomain says:

> Finally the user agent sends the request for the page. Since punycode contains no characters outside those normally allowed for protocols such as HTTP, there is no issue with the transmission of the address. This should simply match against a registered domain name.

Browsers I tested (Firefox, Chromium) always encode the hostname in IDNA.

I made some tests on a random hostname with unicode characters served by nginx. Nginx doesn't care about the encoding and applies the virtual host rules matching the exact string. Ie: with ``xn--9caa.com`` I see the right website, while ``éé.com`` returns a 404 probably because only the IDNA encoded version is specified in the configuration.
 
## Expected behaviour

- ``session.get("http://xn--9caa.com/")`` must make a request with ``Host: xn--9caa.com`` (encoded host).
- ``session.get("http://éé.com/")`` should make a request with ``Host: xn--9caa.com`` (encoded host)

## Actual behaviour

- ``session.get("http://xn--9caa.com/")`` makes a request with a decoded host: ``Host: éé.com`` (UTF-8 encoded host).
- ``session.get("http://éé.com/")`` makes a request with ``Host: éé.com`` too.

## Suggested fix
It seems that ``self.url.raw_host`` should be used rather than ``self.url.host`` in ``ClientRequest``:
https://github.com/KeepSafe/aiohttp/blob/master/aiohttp/client_reqrep.py#L168
(according to my quick test, yarl.URL.raw_host is always return the idna-encoded version, regardless of the encoding of the input url).


Discussion:
Nice catch!
I agree with `self.url.raw_host` usage.
@Martiusweb would you make a Pull Request?

Resolving patch:
diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py
--- a/aiohttp/client_reqrep.py
+++ b/aiohttp/client_reqrep.py
@@ -165,7 +165,7 @@ def update_auto_headers(self, skip_auto_headers):
 
         # add host
         if hdrs.HOST not in used_headers:
-            netloc = self.url.host
+            netloc = self.url.raw_host
             if not self.url.is_default_port():
                 netloc += ':' + str(self.url.port)
             self.headers[hdrs.HOST] = netloc
@@ -437,7 +437,7 @@ def send(self, writer, reader):
         # - not CONNECT proxy must send absolute form URI
         # - most common is origin form URI
         if self.method == hdrs.METH_CONNECT:
-            path = '{}:{}'.format(self.url.host, self.url.port)
+            path = '{}:{}'.format(self.url.raw_host, self.url.port)
         elif self.proxy and not self.ssl:
             path = str(self.url)
         else:
diff --git a/aiohttp/cookiejar.py b/aiohttp/cookiejar.py
--- a/aiohttp/cookiejar.py
+++ b/aiohttp/cookiejar.py
@@ -91,7 +91,7 @@ def _expire_cookie(self, when, domain, name):
 
     def update_cookies(self, cookies, response_url=URL()):
         """Update cookies."""
-        hostname = response_url.host
+        hostname = response_url.raw_host
 
         if not self._unsafe and is_ip_address(hostname):
             # Don't accept cookies from IPs
@@ -168,7 +168,7 @@ def filter_cookies(self, request_url=URL()):
         """Returns this jar's cookies filtered by their attributes."""
         self._do_expiration()
         filtered = SimpleCookie()
-        hostname = request_url.host or ""
+        hostname = request_url.raw_host or ""
         is_not_secure = request_url.scheme not in ("https", "wss")
 
         for cookie in self:


Test patch:
diff --git a/tests/test_client_request.py b/tests/test_client_request.py
--- a/tests/test_client_request.py
+++ b/tests/test_client_request.py
@@ -143,6 +143,16 @@ def test_host_header_host_with_nondefault_port(make_request):
     assert req.headers['HOST'] == 'python.org:99'
 
 
+def test_host_header_host_idna_encode(make_request):
+    req = make_request('get', 'http://xn--9caa.com')
+    assert req.headers['HOST'] == 'xn--9caa.com'
+
+
+def test_host_header_host_unicode(make_request):
+    req = make_request('get', 'http://éé.com')
+    assert req.headers['HOST'] == 'xn--9caa.com'
+
+
 def test_host_header_explicit_host(make_request):
     req = make_request('get', 'http://python.org/',
                        headers={'host': 'example.com'})
diff --git a/tests/test_cookiejar.py b/tests/test_cookiejar.py
--- a/tests/test_cookiejar.py
+++ b/tests/test_cookiejar.py
@@ -162,6 +162,32 @@ def test_save_load(loop, cookies_to_send, cookies_to_receive):
     assert jar_test == cookies_to_receive
 
 
+def test_update_cookie_with_unicode_domain(loop):
+    cookies = (
+        "idna-domain-first=first; Domain=xn--9caa.com; Path=/;",
+        "idna-domain-second=second; Domain=xn--9caa.com; Path=/;",
+    )
+
+    jar = CookieJar(loop=loop)
+    jar.update_cookies(SimpleCookie(cookies[0]), URL("http://éé.com/"))
+    jar.update_cookies(SimpleCookie(cookies[1]), URL("http://xn--9caa.com/"))
+
+    jar_test = SimpleCookie()
+    for cookie in jar:
+        jar_test[cookie.key] = cookie
+
+    assert jar_test == SimpleCookie(" ".join(cookies))
+
+
+def test_filter_cookie_with_unicode_domain(loop):
+    jar = CookieJar(loop=loop)
+    jar.update_cookies(SimpleCookie(
+        "idna-domain-first=first; Domain=xn--9caa.com; Path=/; "
+    ))
+    assert len(jar.filter_cookies(URL("http://éé.com"))) == 1
+    assert len(jar.filter_cookies(URL("http://xn--9caa.com"))) == 1
+
+
 def test_ctor_ith_default_loop(loop):
     asyncio.set_event_loop(loop)
     jar = CookieJar()
diff --git a/tests/test_proxy.py b/tests/test_proxy.py
--- a/tests/test_proxy.py
+++ b/tests/test_proxy.py
@@ -62,6 +62,21 @@ def test_proxy_auth(self):
             "proxy_auth must be None or BasicAuth() tuple",
         )
 
+    @mock.patch('aiohttp.Request')
+    def test_connect_request_with_unicode_host(self, Request_mock):
+        loop = mock.Mock()
+        request = ClientRequest("CONNECT", URL("http://éé.com/"),
+                                loop=loop)
+
+        request.response_class = mock.Mock()
+        request.write_bytes = mock.Mock()
+        request.write_bytes.return_value = asyncio.Future(loop=loop)
+        request.write_bytes.return_value.set_result(None)
+        request.send(mock.Mock(), mock.Mock())
+
+        Request_mock.assert_called_with(mock.ANY, mock.ANY, "xn--9caa.com:80",
+                                        mock.ANY)
+
     def test_proxy_connection_error(self):
         connector = aiohttp.TCPConnector(loop=self.loop)
         connector._resolve_host = make_mocked_coro(