Repository: encode/httpx
Pull request: Update IDNA encoding to 2008 spec
Resolves #150 and adds tests based on deviations from 2003 spec: https://unicode.org/reports/tr46/#Deviations

This also enables Unicode® Technical Standard #46 to normalize capital letters and such, and a test based on the example given in the [module documentation](https://github.com/kjd/idna#compatibility-mapping-uts-46).

Since the idna encode() function chokes on colons, I've split and rejoined the authority string and added to the tests to make sure the port stays as expected.

Problem statement:
Use IDNA 2008 instead of IDNA 2003
Using `str.encode("idna")` uses IDNA 2003 which isn't recommended for modern use. We should be using IDNA 2008 provided by the `idna` module (which is a dependency of `httpx` but I don't think we're using it anywhere?)

Hints from issue discussion:

Commit message: Add test for IRI object

Patch to review:
diff --git a/httpx/models.py b/httpx/models.py
index 2f29be5f9f..c8634a5807 100644
--- a/httpx/models.py
+++ b/httpx/models.py
@@ -87,17 +87,14 @@ def __init__(
     ) -> None:
         if isinstance(url, rfc3986.uri.URIReference):
             self.components = url
+        elif isinstance(url, rfc3986.iri.IRIReference):
+            self.components = url.encode()
         elif isinstance(url, str):
-            self.components = rfc3986.api.uri_reference(url)
+            # Handle IDNA domain names.
+            self.components = rfc3986.api.iri_reference(url).encode()
         else:
             self.components = url.components
 
-        # Handle IDNA domain names.
-        if self.components.authority:
-            idna_authority = self.components.authority.encode("idna").decode("ascii")
-            if idna_authority != self.components.authority:
-                self.components = self.components.copy_with(authority=idna_authority)
-
         # Normalize scheme and domain name.
         if self.is_absolute_url:
             self.components = self.components.normalize()
diff --git a/tests/models/test_url.py b/tests/models/test_url.py
index 7c865f5a66..f20ef87cb3 100644
--- a/tests/models/test_url.py
+++ b/tests/models/test_url.py
@@ -1,13 +1,65 @@
 import pytest
+import rfc3986
 
 from httpx import URL
 from httpx.exceptions import InvalidURL
 
 
-def test_idna_url():
-    url = URL("http://中国.icom.museum:80/")
-    assert url == URL("http://xn--fiqs8s.icom.museum:80/")
-    assert url.host == "xn--fiqs8s.icom.museum"
+@pytest.mark.parametrize(
+    "given,idna,host,scheme,port",
+    [
+        (
+            "http://中国.icom.museum:80/",
+            "http://xn--fiqs8s.icom.museum:80/",
+            "xn--fiqs8s.icom.museum",
+            "http",
+            80,
+        ),
+        (
+            "http://Königsgäßchen.de",
+            "http://xn--knigsgchen-b4a3dun.de",
+            "xn--knigsgchen-b4a3dun.de",
+            "http",
+            80,
+        ),
+        ("https://faß.de", "https://xn--fa-hia.de", "xn--fa-hia.de", "https", 443),
+        (
+            "https://βόλος.com:443",
+            "https://xn--nxasmm1c.com:443",
+            "xn--nxasmm1c.com",
+            "https",
+            443,
+        ),
+        (
+            "http://ශ්‍රී.com:444",
+            "http://xn--10cl1a0b660p.com:444",
+            "xn--10cl1a0b660p.com",
+            "http",
+            444,
+        ),
+        (
+            "https://نامه‌ای.com:4433",
+            "https://xn--mgba3gch31f060k.com:4433",
+            "xn--mgba3gch31f060k.com",
+            "https",
+            4433,
+        ),
+    ],
+    ids=[
+        "http_with_port",
+        "unicode_tr46_compat",
+        "https_without_port",
+        "https_with_port",
+        "http_with_custom_port",
+        "https_with_custom_port",
+    ],
+)
+def test_idna_url(given, idna, host, scheme, port):
+    url = URL(given)
+    assert url == URL(idna)
+    assert url.host == host
+    assert url.scheme == scheme
+    assert url.port == port
 
 
 def test_url():
@@ -27,6 +79,17 @@ def test_url():
     assert new == URL("http://example.org:123/path/to/somewhere?abc=123#anchor")
     assert new.scheme == "http"
 
+def test_iri():
+    iri = rfc3986.iri.IRIReference.from_string("https://example.org:123/path/to/somewhere?abc=123#anchor")
+    url = URL(iri)
+    assert url.scheme == "https"
+    assert url.host == "example.org"
+    assert url.port == 123
+    assert url.authority == "example.org:123"
+    assert url.path == "/path/to/somewhere"
+    assert url.query == "abc=123"
+    assert url.fragment == "anchor"
+
 
 def test_url_eq_str():
     url = URL("https://example.org:123/path/to/somewhere?abc=123#anchor")


Code review comments:
diff_hunk: @@ -87,17 +87,14 @@ def __init__(
     ) -> None:
         if isinstance(url, rfc3986.uri.URIReference):
             self.components = url
+        elif isinstance(url, rfc3986.iri.IRIReference):
path: httpx/models.py
text: @user1:
I don't think we should be exposing extra types as part of this pull request.

(In fact I don't think we should be exposing `rfc3986.uri.URIReference` as part of our public interface in the first place - it'd be much better if we could keep that as an internal implementation detail, in order to allow us to completely switch implementations if needed.)

@user1:
See #170