Repository: pydantic/pydantic

Issue:
Support unicode and punycode when validating TLDs
# Feature Request

Output of `python -c "import pydantic.utils; print(pydantic.utils.version_info())"`:
```
pydantic version: 1.3
            pydantic compiled: True
                 install path: /usr/local/lib/python3.8/site-packages/pydantic
               python version: 3.8.1 (default, Jan  3 2020, 22:55:55)  [GCC 8.3.0]
                     platform: Linux-4.9.184-linuxkit-x86_64-with-glibc2.2.5
     optional deps. installed: ['typing-extensions']
```

The IANA has approved many TLDs that are not matched by the [TLD domain ending regex](https://github.com/samuelcolvin/pydantic/blob/07271ca5bc89aff88957274d2de1a06b28dd070c/pydantic/networks.py#L49) used for `HttpUrl` validation. There are currently ~152 such TLDs: see the entries in the [authoritative list of TLDs](http://data.iana.org/TLD/tlds-alpha-by-domain.txt) containing the ASCII Compatible Encoding prefix `xn--`.

One approach to adding compatibility for such TLDs would be to modify the domain ending regex pattern to allow for Unicode characters, as well as the corresponding internationalized ASCII strings. For example:

 ```python
_domain_ending = r"(?P<tld>(\.[^\W\d_]{2,63})|(\.(?:xn--)[_0-9a-z-]{2,63}))?\.?"
```

Such a change would allow for the following to run successfully

```python
from pydantic import BaseModel, HttpUrl, ValidationError 


class Domain(BaseModel):
    domain: HttpUrl
        

ascii_domains = ["https://example.com"]

idna_domains = [
    "https://example.xn--p1ai",
    "https://example.xn--vermgensberatung-pwb",
    "https://example.xn--zfr164b",
]

unicode_domains = [str.encode(domain).decode("idna") for domain in idna_domains]
        
valid_domains = ascii_domains + idna_domains + unicode_domains

invalid_domains = ["https://example.123", "https://example.ab34"]

for domain in valid_domains:
    Domain(domain=domain)
    
for invalid_domain in invalid_domains:
    try:
        Domain(domain=invalid_domain)
    except ValidationError:
        pass
```

Would you accept a PR for this?


Comments:
Yes, I would.

One concern: you might need to wrap the regex in a function to avoid increasing module import time, same as we do for other regexes.

Patch:
diff --git a/docs/examples/types_url_punycode.py b/docs/examples/types_url_punycode.py
--- a/docs/examples/types_url_punycode.py
+++ b/docs/examples/types_url_punycode.py
@@ -9,3 +9,6 @@ class MyModel(BaseModel):
 m2 = MyModel(url='https://www.аррӏе.com/')
 print(m2.url)
 print(m2.url.host_type)
+m3 = MyModel(url='https://www.example.珠宝/')
+print(m3.url)
+print(m3.url.host_type)
diff --git a/pydantic/networks.py b/pydantic/networks.py
--- a/pydantic/networks.py
+++ b/pydantic/networks.py
@@ -46,7 +46,6 @@
 _url_regex_cache = None
 _ascii_domain_regex_cache = None
 _int_domain_regex_cache = None
-_domain_ending = r'(?P<tld>\.[a-z]{2,63})?\.?'
 
 
 def url_regex() -> Pattern[str]:
@@ -73,7 +72,10 @@ def ascii_domain_regex() -> Pattern[str]:
     global _ascii_domain_regex_cache
     if _ascii_domain_regex_cache is None:
         ascii_chunk = r'[_0-9a-z](?:[-_0-9a-z]{0,61}[_0-9a-z])?'
-        _ascii_domain_regex_cache = re.compile(fr'(?:{ascii_chunk}\.)*?{ascii_chunk}{_domain_ending}', re.IGNORECASE)
+        ascii_domain_ending = r'(?P<tld>\.[a-z]{2,63})?\.?'
+        _ascii_domain_regex_cache = re.compile(
+            fr'(?:{ascii_chunk}\.)*?{ascii_chunk}{ascii_domain_ending}', re.IGNORECASE
+        )
     return _ascii_domain_regex_cache
 
 
@@ -81,7 +83,8 @@ def int_domain_regex() -> Pattern[str]:
     global _int_domain_regex_cache
     if _int_domain_regex_cache is None:
         int_chunk = r'[_0-9a-\U00040000](?:[-_0-9a-\U00040000]{0,61}[_0-9a-\U00040000])?'
-        _int_domain_regex_cache = re.compile(fr'(?:{int_chunk}\.)*?{int_chunk}{_domain_ending}', re.IGNORECASE)
+        int_domain_ending = r'(?P<tld>(\.[^\W\d_]{2,63})|(\.(?:xn--)[_0-9a-z-]{2,63}))?\.?'
+        _int_domain_regex_cache = re.compile(fr'(?:{int_chunk}\.)*?{int_chunk}{int_domain_ending}', re.IGNORECASE)
     return _int_domain_regex_cache
 
 
@@ -220,20 +223,32 @@ def validate_host(cls, parts: Dict[str, str]) -> Tuple[str, Optional[str], str,
         if host is None:
             raise errors.UrlHostError()
         elif host_type == 'domain':
+            is_international = False
             d = ascii_domain_regex().fullmatch(host)
             if d is None:
                 d = int_domain_regex().fullmatch(host)
-                if not d:
+                if d is None:
                     raise errors.UrlHostError()
-                host_type = 'int_domain'
-                rebuild = True
-                host = host.encode('idna').decode('ascii')
+                is_international = True
 
             tld = d.group('tld')
+            if tld is None and not is_international:
+                d = int_domain_regex().fullmatch(host)
+                tld = d.group('tld')
+                is_international = True
+
             if tld is not None:
                 tld = tld[1:]
             elif cls.tld_required:
                 raise errors.UrlHostTldError()
+
+            if is_international:
+                host_type = 'int_domain'
+                rebuild = True
+                host = host.encode('idna').decode('ascii')
+                if tld is not None:
+                    tld = tld.encode('idna').decode('ascii')
+
         return host, tld, host_type, rebuild  # type: ignore
 
     def __repr__(self) -> str:


Test patch:
diff --git a/tests/test_networks.py b/tests/test_networks.py
--- a/tests/test_networks.py
+++ b/tests/test_networks.py
@@ -211,6 +211,9 @@ def test_at_in_path():
         'https://example.org#a=3;b=3',
         'https://foo_bar.example.com/',
         'https://exam_ple.com/',  # should perhaps fail? I think it's contrary to the RFC but chrome allows it
+        'https://example.xn--p1ai',
+        'https://example.xn--vermgensberatung-pwb',
+        'https://example.xn--zfr164b',
     ],
 )
 def test_http_url_success(value):
@@ -231,6 +234,8 @@ class Model(BaseModel):
         ),
         ('http://foobar/', 'value_error.url.host', 'URL host invalid, top level domain required', None),
         ('http://localhost/', 'value_error.url.host', 'URL host invalid, top level domain required', None),
+        ('https://example.123', 'value_error.url.host', 'URL host invalid, top level domain required', None),
+        ('https://example.ab123', 'value_error.url.host', 'URL host invalid, top level domain required', None),
         (
             'x' * 2084,
             'value_error.any_str.max_length',
@@ -260,6 +265,10 @@ class Model(BaseModel):
         # https://www.xudongz.com/blog/2017/idn-phishing/ accepted but converted
         ('https://www.аррӏе.com/', 'https://www.xn--80ak6aa92e.com/'),
         ('https://exampl£e.org', 'https://xn--example-gia.org'),
+        ('https://example.珠宝', 'https://example.xn--pbt977c'),
+        ('https://example.vermögensberatung', 'https://example.xn--vermgensberatung-pwb'),
+        ('https://example.рф', 'https://example.xn--p1ai'),
+        ('https://exampl£e.珠宝', 'https://xn--example-gia.xn--pbt977c'),
     ],
 )
 def test_coerse_url(input, output):
@@ -269,6 +278,26 @@ class Model(BaseModel):
     assert Model(v=input).v == output
 
 
+@pytest.mark.parametrize(
+    'input,output',
+    [
+        ('  https://www.example.com \n', 'com'),
+        (b'https://www.example.com', 'com'),
+        ('https://www.example.com?param=value', 'com'),
+        ('https://example.珠宝', 'xn--pbt977c'),
+        ('https://exampl£e.珠宝', 'xn--pbt977c'),
+        ('https://example.vermögensberatung', 'xn--vermgensberatung-pwb'),
+        ('https://example.рф', 'xn--p1ai'),
+        ('https://example.рф?param=value', 'xn--p1ai'),
+    ],
+)
+def test_parses_tld(input, output):
+    class Model(BaseModel):
+        v: HttpUrl
+
+    assert Model(v=input).v.tld == output
+
+
 def test_postgres_dsns():
     class Model(BaseModel):
         a: PostgresDsn