Repository: encode/httpx

Issue:
Use IDNA 2008 instead of IDNA 2003
Using `str.encode("idna")` uses IDNA 2003 which isn't recommended for modern use. We should be using IDNA 2008 provided by the `idna` module (which is a dependency of `httpx` but I don't think we're using it anywhere?)

Discussion:
StephenBrown2: Looks like rfc3986.IRIReference will in fact do exactly what we want, including handling colons in the authority. I've rebased on master to resolve the conflict with hsts... and switched to using `rfc3986.api.iri_reference(url).encode()` which will handle the IDNA Domain and return a URLReference object as expected.

Ref: convenience function - https://github.com/python-hyper/rfc3986/blob/df21c4b09078580c1cb10c728dc18c3111fcf603/src/rfc3986/api.py#L41
Relevant bit of IRIReference.encode() - https://github.com/python-hyper/rfc3986/blob/df21c4b09078580c1cb10c728dc18c3111fcf603/src/rfc3986/iri.py#L97-L129
tomchristie: I guess we *might* also choose to switch our dependency listing to `rfc3986[idna]`, instead of listing the two packages seperately?

That'd stop devs from having to do a "wait where are you actually using the `idna` package?" double take.
tomchristie: Yas - good stuff!


Patch:
diff --git a/httpx/models.py b/httpx/models.py
index df5d071..32e412f 100644
--- a/httpx/models.py
+++ b/httpx/models.py
@@ -89,16 +89,10 @@ class URL:
         params: QueryParamTypes = None,
     ) -> None:
         if isinstance(url, str):
-            self._uri_reference = rfc3986.api.uri_reference(url)
+            self._uri_reference = rfc3986.api.iri_reference(url).encode()
         else:
             self._uri_reference = url._uri_reference
 
-        # Handle IDNA domain names.
-        if self._uri_reference.authority:
-            idna_authority = self._uri_reference.authority.encode("idna").decode("ascii")
-            if idna_authority != self._uri_reference.authority:
-                self._uri_reference = self._uri_reference.copy_with(authority=idna_authority)
-
         # Normalize scheme and domain name.
         if self.is_absolute_url:
             self._uri_reference = self._uri_reference.normalize()


Test patch:
diff --git a/tests/models/test_url.py b/tests/models/test_url.py
index 7c865f5..a556ed8 100644
--- a/tests/models/test_url.py
+++ b/tests/models/test_url.py
@@ -1,13 +1,65 @@
 import pytest
+import rfc3986
 
 from httpx import URL
 from httpx.exceptions import InvalidURL
 
 
-def test_idna_url():
-    url = URL("http://中国.icom.museum:80/")
-    assert url == URL("http://xn--fiqs8s.icom.museum:80/")
-    assert url.host == "xn--fiqs8s.icom.museum"
+@pytest.mark.parametrize(
+    "given,idna,host,scheme,port",
+    [
+        (
+            "http://中国.icom.museum:80/",
+            "http://xn--fiqs8s.icom.museum:80/",
+            "xn--fiqs8s.icom.museum",
+            "http",
+            80,
+        ),
+        (
+            "http://Königsgäßchen.de",
+            "http://xn--knigsgchen-b4a3dun.de",
+            "xn--knigsgchen-b4a3dun.de",
+            "http",
+            80,
+        ),
+        ("https://faß.de", "https://xn--fa-hia.de", "xn--fa-hia.de", "https", 443),
+        (
+            "https://βόλος.com:443",
+            "https://xn--nxasmm1c.com:443",
+            "xn--nxasmm1c.com",
+            "https",
+            443,
+        ),
+        (
+            "http://ශ්‍රී.com:444",
+            "http://xn--10cl1a0b660p.com:444",
+            "xn--10cl1a0b660p.com",
+            "http",
+            444,
+        ),
+        (
+            "https://نامه‌ای.com:4433",
+            "https://xn--mgba3gch31f060k.com:4433",
+            "xn--mgba3gch31f060k.com",
+            "https",
+            4433,
+        ),
+    ],
+    ids=[
+        "http_with_port",
+        "unicode_tr46_compat",
+        "https_without_port",
+        "https_with_port",
+        "http_with_custom_port",
+        "https_with_custom_port",
+    ],
+)
+def test_idna_url(given, idna, host, scheme, port):
+    url = URL(given)
+    assert url == URL(idna)
+    assert url.host == host
+    assert url.scheme == scheme
+    assert url.port == port
 
 
 def test_url():