Repository: pydantic/pydantic
Pull request: fix(PostgresDsn): Add support for multiple hosts in PostgresDsn #3337
## Change Summary

Add support for multiple hosts validation in `PostgresDsn` class.

Postgres supports multiple hosts in the DSN like: `postgres://user:pass@host1.db.net:4321,host2.db.net:6432/app`.
The change keeps old PostgreDsn working for single host as it was before and introduces new attribute `hosts` to keep track of all added hosts with validation.
I have also separated some regex to be able to reuse them in the new matching regexes. 

## Related issue number

Fixes: #3337

## Checklist

* [x] Unit tests for the changes exist
* [x] Tests pass on CI and coverage remains at 100%
* [x] Documentation reflects the changes where applicable
* [x] `changes/<pull request or issue id>-<github username>.md` file added describing change
  (see [changes/README.md](https://github.com/samuelcolvin/pydantic/blob/master/changes/README.md) for details)
* [x] My PR is ready to review, **please add a comment including the phrase "please review" to assign reviewers**


Problem statement:
PostgresDsn fails for multiple hosts
Output of `python -c "import pydantic.utils; print(pydantic.utils.version_info())"`:
```
pydantic version: 1.8.2
pydantic compiled: False
install path: /home/stupidhobbit/arc/arcadia/contrib/python/pydantic/pydantic
python version: 3.9.7 (default, Oct 13 2021, 18:49:29)  [Clang 12.0.1 ]
platform: Linux-5.4.0-66-generic-x86_64-with-glibc2.27
optional deps. installed: ['typing-extensions']
```

```py
from pydantic import BaseSettings, PostgresDsn


class PGSettings(BaseSettings):
    dsn: PostgresDsn = 'postgres://user:pass@host1.db.net,host2.db.net:6432/db'
```
This code gives you this error:

```
pydantic.error_wrappers.ValidationError: 1 validation error for PGSettings
dsn
  URL host invalid (type=value_error.url.host)
```
But this is a valid dsn for postgres and it works just fine with psycopg2

Hints from issue discussion:

Commit message: Reuse _host_regex in postgres_url_regex

Patch to review:
diff --git a/changes/3337-rglsk.md b/changes/3337-rglsk.md
new file mode 100644
index 00000000000..c0f2da37208
--- /dev/null
+++ b/changes/3337-rglsk.md
@@ -0,0 +1 @@
+Support multi hosts validation in `PostgresDsn`.
\ No newline at end of file
diff --git a/pydantic/networks.py b/pydantic/networks.py
index 2be1ac80d56..dbd77721805 100644
--- a/pydantic/networks.py
+++ b/pydantic/networks.py
@@ -15,6 +15,7 @@
     Collection,
     Dict,
     Generator,
+    List,
     Optional,
     Pattern,
     Set,
@@ -51,6 +52,13 @@ class Parts(TypedDict, total=False):
         query: Optional[str]
         fragment: Optional[str]
 
+    class HostParts(TypedDict, total=False):
+        host: str
+        tld: Optional[str]
+        host_type: Optional[str]
+        port: Optional[str]
+
+
 else:
     email_validator = None
 
@@ -74,28 +82,56 @@ class Parts(TypedDict, total=False):
 ]
 
 _url_regex_cache = None
+_postgres_url_regex_cache = None
 _ascii_domain_regex_cache = None
 _int_domain_regex_cache = None
+_host_regex_cache = None
+
+_host_regex = (
+    r'(?:'
+    r'(?P<ipv4>(?:\d{1,3}\.){3}\d{1,3})(?=$|[/:#?])|'  # ipv4
+    r'(?P<ipv6>\[[A-F0-9]*:[A-F0-9:]+\])(?=$|[/:#?])|'  # ipv6
+    r'(?P<domain>[^\s/:?#]+)'  # domain, validation occurs later
+    r')?'
+    r'(?::(?P<port>\d+))?'  # port
+)
+_scheme_regex = r'(?:(?P<scheme>[a-z][a-z0-9+\-.]+)://)?'  # scheme https://tools.ietf.org/html/rfc3986#appendix-A
+_user_info_regex = r'(?:(?P<user>[^\s:/]*)(?::(?P<password>[^\s/]*))?@)?'
+_path_regex = r'(?P<path>/[^\s?#]*)?'
+_query_regex = r'(?:\?(?P<query>[^\s#]*))?'
+_fragment_regex = r'(?:#(?P<fragment>[^\s#]*))?'
 
 
 def url_regex() -> Pattern[str]:
     global _url_regex_cache
     if _url_regex_cache is None:
         _url_regex_cache = re.compile(
-            r'(?:(?P<scheme>[a-z][a-z0-9+\-.]+)://)?'  # scheme https://tools.ietf.org/html/rfc3986#appendix-A
-            r'(?:(?P<user>[^\s:/]*)(?::(?P<password>[^\s/]*))?@)?'  # user info
+            rf'{_scheme_regex}{_user_info_regex}{_host_regex}{_path_regex}{_query_regex}{_fragment_regex}',
+            re.IGNORECASE,
+        )
+    return _url_regex_cache
+
+
+def postgres_url_regex() -> Pattern[str]:
+    """
+    Compiled postgres regex to match database urls.
+
+    Additionally to `url_regex` it allows to match multiple hosts.
+    E.g. host1.db.net,host2.db.net
+    """
+    global _postgres_url_regex_cache
+    if _postgres_url_regex_cache is None:
+        _postgres_url_regex_cache = re.compile(
+            rf'{_scheme_regex}{_user_info_regex}'
             r'(?:'
-            r'(?P<ipv4>(?:\d{1,3}\.){3}\d{1,3})(?=$|[/:#?])|'  # ipv4
-            r'(?P<ipv6>\[[A-F0-9]*:[A-F0-9:]+\])(?=$|[/:#?])|'  # ipv6
-            r'(?P<domain>[^\s/:?#]+)'  # domain, validation occurs later
+            r'(?P<hosts>('  # hosts, validation occurs later
+            rf'{_host_regex}'
+            r',?)+)'
             r')?'
-            r'(?::(?P<port>\d+))?'  # port
-            r'(?P<path>/[^\s?#]*)?'  # path
-            r'(?:\?(?P<query>[^\s#]*))?'  # query
-            r'(?:#(?P<fragment>[^\s#]*))?',  # fragment
+            rf'{_path_regex}{_query_regex}{_fragment_regex}',
             re.IGNORECASE,
         )
-    return _url_regex_cache
+    return _postgres_url_regex_cache
 
 
 def ascii_domain_regex() -> Pattern[str]:
@@ -118,6 +154,16 @@ def int_domain_regex() -> Pattern[str]:
     return _int_domain_regex_cache
 
 
+def host_regex() -> Pattern[str]:
+    global _host_regex_cache
+    if _host_regex_cache is None:
+        _host_regex_cache = re.compile(
+            _host_regex,
+            re.IGNORECASE,
+        )
+    return _host_regex_cache
+
+
 class AnyUrl(str):
     strip_whitespace = True
     min_length = 1
@@ -238,7 +284,7 @@ def validate(cls, value: Any, field: 'ModelField', config: 'BaseConfig') -> 'Any
         )
 
     @classmethod
-    def validate_parts(cls, parts: 'Parts') -> 'Parts':
+    def validate_parts(cls, parts: 'Parts', validate_port: bool = True) -> 'Parts':
         """
         A method used to validate parts of an URL.
         Could be overridden to set default values for parts if missing
@@ -250,9 +296,10 @@ def validate_parts(cls, parts: 'Parts') -> 'Parts':
         if cls.allowed_schemes and scheme.lower() not in cls.allowed_schemes:
             raise errors.UrlSchemePermittedError(set(cls.allowed_schemes))
 
-        port = parts['port']
-        if port is not None and int(port) > 65_535:
-            raise errors.UrlPortError()
+        if validate_port:
+            port = parts['port']
+            if port is not None and int(port) > 65_535:
+                raise errors.UrlPortError()
 
         user = parts['user']
         if cls.user_required and user is None:
@@ -350,6 +397,98 @@ class PostgresDsn(AnyUrl):
     }
     user_required = True
 
+    __slots__ = AnyUrl.__slots__ + ('hosts',)
+
+    def __init__(self, *args, hosts: Optional[List['HostParts']] = None, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.hosts = hosts
+
+    @classmethod
+    def validate_host_parts(cls, parts: 'HostParts') -> 'HostParts':
+        """
+        A method used to validate host parts of an URL.
+        """
+        port = parts['port']
+        if port is not None and int(port) > 65_535:
+            raise errors.UrlPortError()
+
+        return parts
+
+    @classmethod
+    def validate_multi_host(cls, hosts: List[str]) -> List['HostParts']:
+        hosts_parts: List['HostParts'] = []
+        for host in hosts:
+            hm = host_regex().match(host)
+            original_parts = cast('Parts', hm.groupdict())
+            host, tld, host_type, rebuild = cls.validate_host(original_parts)
+            host_parts = cast(
+                'HostParts',
+                {
+                    'host': host,
+                    'host_type': host_type,
+                    'tld': tld,
+                    'rebuild': rebuild,
+                    'port': original_parts.get('port'),
+                },
+            )
+            host_parts = cls.validate_host_parts(host_parts)
+            hosts_parts.append(host_parts)
+        return hosts_parts
+
+    @classmethod
+    def validate(cls, value: Any, field: 'ModelField', config: 'BaseConfig') -> 'PostgresDsn':
+        if value.__class__ == cls:
+            return value
+        value = str_validator(value)
+        if cls.strip_whitespace:
+            value = value.strip()
+        url: str = cast(str, constr_length_validator(value, field, config))
+
+        m = postgres_url_regex().match(url)
+        # the regex should always match, if it doesn't please report with details of the URL tried
+        assert m, 'URL regex failed unexpectedly'
+
+        original_parts = cast('Parts', m.groupdict())
+        parts = cls.validate_parts(original_parts, validate_port=False)
+
+        hosts = m.groupdict()['hosts']
+        if hosts is None and cls.host_required:
+            raise errors.UrlHostError()
+
+        hosts_parts = cls.validate_multi_host(hosts.split(','))
+
+        if m.end() != len(url):
+            raise errors.UrlExtraError(extra=url[m.end() :])
+
+        if len(hosts_parts) > 1:
+            return cls(
+                None if any([hp['rebuild'] for hp in hosts_parts]) else url,
+                scheme=parts['scheme'],
+                user=parts['user'],
+                password=parts['password'],
+                path=parts['path'],
+                query=parts['query'],
+                fragment=parts['fragment'],
+                host_type=None,
+                hosts=hosts_parts,
+            )
+
+        # Keep the back compatibility with single host
+        _host_part = hosts_parts[0]
+        return cls(
+            None if _host_part['rebuild'] else url,
+            scheme=parts['scheme'],
+            user=parts['user'],
+            password=parts['password'],
+            host=_host_part['host'],
+            tld=_host_part['tld'],
+            host_type=_host_part['host_type'],
+            port=_host_part.get('port'),
+            path=parts['path'],
+            query=parts['query'],
+            fragment=parts['fragment'],
+        )
+
 
 class RedisDsn(AnyUrl):
     allowed_schemes = {'redis', 'rediss'}
diff --git a/tests/test_networks.py b/tests/test_networks.py
index 4e7fe29fd0b..2db104f382a 100644
--- a/tests/test_networks.py
+++ b/tests/test_networks.py
@@ -426,6 +426,25 @@ class Model(BaseModel):
         Model(a='postgresql+asyncpg://user:pass@localhost:5432/app').a
         == 'postgresql+asyncpg://user:pass@localhost:5432/app'
     )
+    assert (
+        Model(a='postgres://user:pass@host1.db.net,host2.db.net:6432/app').a
+        == 'postgres://user:pass@host1.db.net,host2.db.net:6432/app'
+    )
+
+    with pytest.raises(ValidationError) as exc_info:
+        Model(a='postgres://user:pass@host1.db.net:4321,/foo/bar:5432/app')
+    error = exc_info.value.errors()[0]
+    assert error == {'loc': ('a',), 'msg': 'URL host invalid', 'type': 'value_error.url.host'}
+
+    with pytest.raises(ValidationError) as exc_info:
+        Model(a='postgres://user:pass@host1.db.net,/app')
+    error = exc_info.value.errors()[0]
+    assert error == {'loc': ('a',), 'msg': 'URL host invalid', 'type': 'value_error.url.host'}
+
+    with pytest.raises(ValidationError) as exc_info:
+        Model(a='postgres://user:pass@/foo/bar:5432,host1.db.net:4321/app')
+    error = exc_info.value.errors()[0]
+    assert error == {'loc': ('a',), 'msg': 'URL host invalid', 'type': 'value_error.url.host'}
 
     with pytest.raises(ValidationError) as exc_info:
         Model(a='http://example.org')
@@ -443,6 +462,34 @@ class Model(BaseModel):
     assert error == {'loc': ('a',), 'msg': 'URL host invalid', 'type': 'value_error.url.host'}
 
 
+def test_multihost_postgres_dsns():
+    class Model(BaseModel):
+        a: PostgresDsn
+
+    any_multihost_url = Model(a='postgres://user:pass@host1.db.net:4321,host2.db.net:6432/app').a
+    assert any_multihost_url == 'postgres://user:pass@host1.db.net:4321,host2.db.net:6432/app'
+    assert any_multihost_url.scheme == 'postgres'
+    assert any_multihost_url.host is None
+    assert any_multihost_url.host_type is None
+    assert any_multihost_url.tld is None
+    assert any_multihost_url.port is None
+    assert any_multihost_url.path == '/app'
+    assert any_multihost_url.hosts == [
+        {'host': 'host1.db.net', 'port': '4321', 'tld': 'net', 'host_type': 'domain', 'rebuild': False},
+        {'host': 'host2.db.net', 'port': '6432', 'tld': 'net', 'host_type': 'domain', 'rebuild': False},
+    ]
+
+    any_multihost_url = Model(a='postgres://user:pass@host.db.net:4321/app').a
+    assert any_multihost_url.scheme == 'postgres'
+    assert any_multihost_url == 'postgres://user:pass@host.db.net:4321/app'
+    assert any_multihost_url.host == 'host.db.net'
+    assert any_multihost_url.host_type == 'domain'
+    assert any_multihost_url.tld == 'net'
+    assert any_multihost_url.port == '4321'
+    assert any_multihost_url.path == '/app'
+    assert any_multihost_url.hosts is None
+
+
 def test_redis_dsns():
     class Model(BaseModel):
         a: RedisDsn


Code review comments:
diff_hunk: @@ -350,6 +397,98 @@ class PostgresDsn(AnyUrl):
     }
     user_required = True
 
+    __slots__ = AnyUrl.__slots__ + ('hosts',)
+
+    def __init__(self, *args, hosts: Optional[List['HostParts']] = None, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.hosts = hosts
+
+    @classmethod
+    def validate_host_parts(cls, parts: 'HostParts') -> 'HostParts':
path: pydantic/networks.py
text: @user1:
move this to the parent, and call it from there and here to avoid duplication.

diff_hunk: @@ -350,6 +397,98 @@ class PostgresDsn(AnyUrl):
     }
     user_required = True
 
+    __slots__ = AnyUrl.__slots__ + ('hosts',)
+
+    def __init__(self, *args, hosts: Optional[List['HostParts']] = None, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.hosts = hosts
+
+    @classmethod
+    def validate_host_parts(cls, parts: 'HostParts') -> 'HostParts':
+        """
+        A method used to validate host parts of an URL.
+        """
+        port = parts['port']
+        if port is not None and int(port) > 65_535:
+            raise errors.UrlPortError()
+
+        return parts
+
+    @classmethod
+    def validate_multi_host(cls, hosts: List[str]) -> List['HostParts']:
+        hosts_parts: List['HostParts'] = []
+        for host in hosts:
+            hm = host_regex().match(host)
+            original_parts = cast('Parts', hm.groupdict())
+            host, tld, host_type, rebuild = cls.validate_host(original_parts)
+            host_parts = cast(
+                'HostParts',
+                {
+                    'host': host,
+                    'host_type': host_type,
+                    'tld': tld,
+                    'rebuild': rebuild,
+                    'port': original_parts.get('port'),
+                },
+            )
+            host_parts = cls.validate_host_parts(host_parts)
+            hosts_parts.append(host_parts)
+        return hosts_parts
+
+    @classmethod
+    def validate(cls, value: Any, field: 'ModelField', config: 'BaseConfig') -> 'PostgresDsn':
path: pydantic/networks.py
text: @user1:
There's still to much duplication here.

There are a number of ways to work around this while maintaining performance and not altering the external interface to `AnyUrl`.

This class should only override `validate_host`, or perhaps a new method `validate_netloc`.

@author:
This part is the most tricky.

It is hard to override it since we need to build a new objects with new field `hosts`. We could map `host` and other fields and join them by `,` but the problem would be that we will lost which port relies to which host ect.

I will think more what would be the best way to merge it.

@user1 Please let me know if you have already something in mind.

diff_hunk: @@ -350,6 +397,98 @@ class PostgresDsn(AnyUrl):
     }
     user_required = True
 
+    __slots__ = AnyUrl.__slots__ + ('hosts',)
+
+    def __init__(self, *args, hosts: Optional[List['HostParts']] = None, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.hosts = hosts
+
+    @classmethod
+    def validate_host_parts(cls, parts: 'HostParts') -> 'HostParts':
+        """
+        A method used to validate host parts of an URL.
+        """
+        port = parts['port']
+        if port is not None and int(port) > 65_535:
+            raise errors.UrlPortError()
+
+        return parts
+
+    @classmethod
+    def validate_multi_host(cls, hosts: List[str]) -> List['HostParts']:
+        hosts_parts: List['HostParts'] = []
+        for host in hosts:
+            hm = host_regex().match(host)
path: pydantic/networks.py
text: @user1:
with this we're re-running the host regex. Instead of running this bit of the regex twice, surely have just `r'(?P<hosts>.+?)` in `_postgres_url_regex_cache` above, then do the validation here?

@author:
Makes sense 💪

diff_hunk: @@ -350,6 +397,98 @@ class PostgresDsn(AnyUrl):
     }
     user_required = True
 
+    __slots__ = AnyUrl.__slots__ + ('hosts',)
+
+    def __init__(self, *args, hosts: Optional[List['HostParts']] = None, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.hosts = hosts
+
+    @classmethod
+    def validate_host_parts(cls, parts: 'HostParts') -> 'HostParts':
+        """
+        A method used to validate host parts of an URL.
+        """
+        port = parts['port']
+        if port is not None and int(port) > 65_535:
+            raise errors.UrlPortError()
+
+        return parts
+
+    @classmethod
+    def validate_multi_host(cls, hosts: List[str]) -> List['HostParts']:
+        hosts_parts: List['HostParts'] = []
+        for host in hosts:
+            hm = host_regex().match(host)
+            original_parts = cast('Parts', hm.groupdict())
+            host, tld, host_type, rebuild = cls.validate_host(original_parts)
+            host_parts = cast(
+                'HostParts',
+                {
+                    'host': host,
+                    'host_type': host_type,
+                    'tld': tld,
+                    'rebuild': rebuild,
path: pydantic/networks.py
text: @user1:
This is not correct. There's no point in using types if you're just going to ignore them.

diff_hunk: @@ -74,28 +82,56 @@ class Parts(TypedDict, total=False):
 ]
 
 _url_regex_cache = None
+_postgres_url_regex_cache = None
 _ascii_domain_regex_cache = None
 _int_domain_regex_cache = None
+_host_regex_cache = None
+
+_host_regex = (
+    r'(?:'
+    r'(?P<ipv4>(?:\d{1,3}\.){3}\d{1,3})(?=$|[/:#?])|'  # ipv4
+    r'(?P<ipv6>\[[A-F0-9]*:[A-F0-9:]+\])(?=$|[/:#?])|'  # ipv6
+    r'(?P<domain>[^\s/:?#]+)'  # domain, validation occurs later
+    r')?'
+    r'(?::(?P<port>\d+))?'  # port
+)
+_scheme_regex = r'(?:(?P<scheme>[a-z][a-z0-9+\-.]+)://)?'  # scheme https://tools.ietf.org/html/rfc3986#appendix-A
+_user_info_regex = r'(?:(?P<user>[^\s:/]*)(?::(?P<password>[^\s/]*))?@)?'
+_path_regex = r'(?P<path>/[^\s?#]*)?'
+_query_regex = r'(?:\?(?P<query>[^\s#]*))?'
+_fragment_regex = r'(?:#(?P<fragment>[^\s#]*))?'
 
 
 def url_regex() -> Pattern[str]:
     global _url_regex_cache
     if _url_regex_cache is None:
         _url_regex_cache = re.compile(
-            r'(?:(?P<scheme>[a-z][a-z0-9+\-.]+)://)?'  # scheme https://tools.ietf.org/html/rfc3986#appendix-A
-            r'(?:(?P<user>[^\s:/]*)(?::(?P<password>[^\s/]*))?@)?'  # user info
+            rf'{_scheme_regex}{_user_info_regex}{_host_regex}{_path_regex}{_query_regex}{_fragment_regex}',
+            re.IGNORECASE,
+        )
+    return _url_regex_cache
+
+
+def postgres_url_regex() -> Pattern[str]:
+    """
+    Compiled postgres regex to match database urls.
+
+    Additionally to `url_regex` it allows to match multiple hosts.
+    E.g. host1.db.net,host2.db.net
+    """
+    global _postgres_url_regex_cache
+    if _postgres_url_regex_cache is None:
+        _postgres_url_regex_cache = re.compile(
+            rf'{_scheme_regex}{_user_info_regex}'
             r'(?:'
path: pydantic/networks.py
text: @user1:
Surely you don't need this group?

diff_hunk: @@ -426,6 +426,25 @@ class Model(BaseModel):
         Model(a='postgresql+asyncpg://user:pass@localhost:5432/app').a
         == 'postgresql+asyncpg://user:pass@localhost:5432/app'
     )
+    assert (
+        Model(a='postgres://user:pass@host1.db.net,host2.db.net:6432/app').a
+        == 'postgres://user:pass@host1.db.net,host2.db.net:6432/app'
+    )
+
+    with pytest.raises(ValidationError) as exc_info:
+        Model(a='postgres://user:pass@host1.db.net:4321,/foo/bar:5432/app')
+    error = exc_info.value.errors()[0]
+    assert error == {'loc': ('a',), 'msg': 'URL host invalid', 'type': 'value_error.url.host'}
+
+    with pytest.raises(ValidationError) as exc_info:
+        Model(a='postgres://user:pass@host1.db.net,/app')
+    error = exc_info.value.errors()[0]
+    assert error == {'loc': ('a',), 'msg': 'URL host invalid', 'type': 'value_error.url.host'}
+
+    with pytest.raises(ValidationError) as exc_info:
+        Model(a='postgres://user:pass@/foo/bar:5432,host1.db.net:4321/app')
+    error = exc_info.value.errors()[0]
+    assert error == {'loc': ('a',), 'msg': 'URL host invalid', 'type': 'value_error.url.host'}
path: tests/test_networks.py
text: @user1:
these can be parameterised.