Repository: coverahealth/dataspec

Issue:
Create URL string validator factory
In #8, we will add a `s.str(format_="url")` string format to check that a string contains something which can be considered a URL. However, nearly any string can be considered a valid URL, so this is not terribly useful outside of limited cases. I think it would be more useful to add a full `s.url_str(...)` factory, which can check any of the useful components of a URL (e.g. asserting that URLs are using a certain scheme or are constrained to a certain subset of hosts).

Discussion:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5751826..8f7b79b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add ISO time string format (#4)
 - Allow type-checking specs to be created by passing a type directly to `s` (#12)
 - Add email address string format (#6)
+- Add URL string format factory `s.url` (#16)
 
 ### Fixed
 - Guard against `inspect.signature` failures with Python builtins (#18)
diff --git a/src/dataspec/api.py b/src/dataspec/api.py
index 924f027..5673c9e 100644
--- a/src/dataspec/api.py
+++ b/src/dataspec/api.py
@@ -23,6 +23,7 @@ from dataspec.factories import (
     opt_key,
     str_spec,
     time_spec,
+    url_str_spec,
     uuid_spec,
 )
 
@@ -97,6 +98,7 @@ class SpecAPI:
     obj = staticmethod(obj_spec)
     str = staticmethod(str_spec)
     time = staticmethod(time_spec)
+    url = staticmethod(url_str_spec)
     uuid = staticmethod(uuid_spec)
 
     # Builtin pre-baked specs
diff --git a/src/dataspec/factories.py b/src/dataspec/factories.py
index 35e5e0a..3092dac 100644
--- a/src/dataspec/factories.py
+++ b/src/dataspec/factories.py
@@ -19,6 +19,7 @@ from typing import (
     Union,
     cast,
 )
+from urllib.parse import ParseResult, parse_qs, urlparse
 
 import attr
 
@@ -381,7 +382,7 @@ def _str_is_email_address(s: str) -> Iterator[ErrorDetails]:
         Address(addr_spec=s)
     except (TypeError, ValueError) as e:
         yield ErrorDetails(
-            f"String does not contain a valid email address: {e}",
+            message=f"String does not contain a valid email address: {e}",
             pred=_str_is_email_address,
             value=s,
         )
@@ -554,6 +555,224 @@ def str_spec(  # noqa: MC0001  # pylint: disable=too-many-arguments
     return ValidatorSpec.from_validators(tag or "str", *validators, conformer=conformer)
 
 
+_IGNORE_URL_PARAM = object()
+_URL_RESULT_FIELDS = frozenset(
+    {
+        "scheme",
+        "netloc",
+        "path",
+        "params",
+        "fragment",
+        "username",
+        "password",
+        "hostname",
+        "port",
+    }
+)
+
+
+def _url_attr_validator(
+    attr: str, exact_attr: Any, regex_attr: Any, in_attr: Any
+) -> Optional[ValidatorFn]:
+    """Return a urllib.parse.ParseResult attribute validator function for the given
+    attribute based on the values of the three rule types."""
+
+    def get_url_attr(v: Any, attr: str = attr) -> Any:
+        return getattr(v, attr)
+
+    if exact_attr is not _IGNORE_URL_PARAM:
+
+        @pred_to_validator(
+            f"URL attribute '{attr}' value '{{value}}' is not '{exact_attr}'",
+            complement=True,
+            convert_value=get_url_attr,
+        )
+        def url_attr_equals(v: Any) -> bool:
+            return get_url_attr(v) == exact_attr
+
+        return url_attr_equals
+
+    elif regex_attr is not _IGNORE_URL_PARAM:
+
+        if attr == "port":
+            raise ValueError(f"Cannot define regex spec for URL attribute 'port'")
+
+        if not isinstance(regex_attr, str):
+            raise TypeError(f"URL attribute '{attr}_regex' must be a string value")
+
+        pattern = re.compile(regex_attr)
+
+        @pred_to_validator(
+            f"URL attribute '{attr}' value '{{value}}' does not "
+            f"match regex '{regex_attr}'",
+            complement=True,
+            convert_value=get_url_attr,
+        )
+        def url_attr_matches_regex(v: ParseResult) -> bool:
+            return bool(re.fullmatch(pattern, get_url_attr(v)))
+
+        return url_attr_matches_regex
+
+    elif in_attr is not _IGNORE_URL_PARAM:
+
+        if not isinstance(in_attr, (frozenset, set)):
+            raise TypeError(f"URL attribute '{attr}_in' must be set or frozenset")
+
+        @pred_to_validator(
+            f"URL attribute '{attr}' value '{{value}}' not in {in_attr}",
+            complement=True,
+            convert_value=get_url_attr,
+        )
+        def url_attr_is_allowed_value(v: ParseResult) -> bool:
+            return get_url_attr(v) in in_attr
+
+        return url_attr_is_allowed_value
+    else:
+        return None
+
+
+def url_str_spec(
+    tag: Optional[Tag] = None,
+    query: Optional[SpecPredicate] = None,
+    conformer: Optional[Conformer] = None,
+    **kwargs,
+) -> Spec:
+    """
+    Return a spec that can validate URLs against common rules.
+
+    URL string specs always verify that input values are strings and that they can
+    be successfully parsed by :py:func:`urllib.parse.urlparse`.
+
+    URL specs can specify a new or existing Spec or spec predicate value to validate
+    the query string value produced by calling :py:func:`urllib.parse.parse_qs` on the
+    :py:attr:`urllib.parse.ParseResult.query` attribute of the parsed URL result.
+
+    Other restrictions can be applied by passing any one of three different keyword
+    arguments for any of the fields (excluding :py:attr:`urllib.parse.ParseResult.query`)
+    of :py:class:`urllib.parse.ParseResult`. For example, to specify restrictions on the
+    ``hostname`` field, you could use the following keywords:
+
+     * ``hostname`` accepts any value (including :py:obj:`None`) and checks for an
+       exact match of the keyword argument value
+     * ``hostname_in`` takes a :py:class:``set`` or :py:class:``frozenset`` and
+       validates that the `hostname`` field is an exact match with one of the
+       elements of the set
+     * ``hostname_regex`` takes a :py:class:``str``, creates a Regex pattern from
+       that string, and validates that ``hostname`` is a match (by
+       :py:func:`re.fullmatch`) with the given pattern
+
+    The value :py:obj:`None` can be used for comparison in all cases. Note that default
+    the values for fields of :py:class:`urllib.parse.ParseResult` vary by field, so
+    using None may produce unexpected results.
+
+    At most only one restriction can be applied to any given field for the
+    :py:class:`urllib.parse.ParseResult`. Specifying more than one restriction for
+    a field will produce a :py:exc:`ValueError`.
+
+    At least one restriction must be specified to create a URL string Spec.
+    Attempting to create a URL Spec without specifying a restriction will produce a
+    :py:exc:`ValueError`.
+
+    :param tag: an optional tag for the resulting spec
+    :param query: an optional spec for the :py:class:`dict` created by calling
+        :py:func:`urllib.parse.parse_qs` on the :py:attr:`urllib.parse.ParseResult.query`
+        attribute of the parsed URL
+    :param scheme: if specified, require an exact match for ``scheme``
+    :param scheme_in: if specified, require ``scheme`` to match at least one value in the set
+    :param schema_regex: if specified, require ``scheme`` to match the regex pattern
+    :param netloc: if specified, require an exact match for ``netloc``
+    :param netloc_in: if specified, require ``netloc`` to match at least one value in the set
+    :param netloc_regex: if specified, require ``netloc`` to match the regex pattern
+    :param path: if specified, require an exact match for ``path``
+    :param path_in: if specified, require ``path`` to match at least one value in the set
+    :param path_regex: if specified, require ``path`` to match the regex pattern
+    :param params: if specified, require an exact match for ``params``
+    :param params_in: if specified, require ``params`` to match at least one value in the set
+    :param params_regex: if specified, require ``params`` to match the regex pattern
+    :param fragment: if specified, require an exact match for ``fragment``
+    :param fragment_in: if specified, require ``fragment`` to match at least one value in the set
+    :param fragment_regex: if specified, require ``fragment`` to match the regex pattern
+    :param username: if specified, require an exact match for ``username``
+    :param username_in: if specified, require ``username`` to match at least one value in the set
+    :param username_regex: if specified, require ``username`` to match the regex pattern
+    :param password: if specified, require an exact match for ``password``
+    :param password_in: if specified, require ``password`` to match at least one value in the set
+    :param password_regex: if specified, require ``password`` to match the regex pattern
+    :param hostname: if specified, require an exact match for ``hostname``
+    :param hostname_in: if specified, require ``hostname`` to match at least one value in the set
+    :param hostname_regex: if specified, require ``hostname`` to match the regex pattern
+    :param port: if specified, require an exact match for ``port``
+    :param port_in: if specified, require ``port`` to match at least one value in the set
+    :param conformer: an optional conformer for the value
+    :return: a Spec which can validate that a string contains a URL
+    """
+
+    @pred_to_validator(f"Value '{{value}}' is not a string", complement=True)
+    def is_str(s: Any) -> bool:
+        return isinstance(s, str)
+
+    validators: List[Union[ValidatorFn, ValidatorSpec]] = [is_str]
+
+    child_validators = []
+    for url_attr in _URL_RESULT_FIELDS:
+        in_attr = kwargs.pop(f"{url_attr}_in", _IGNORE_URL_PARAM)
+        regex_attr = kwargs.pop(f"{url_attr}_regex", _IGNORE_URL_PARAM)
+        exact_attr = kwargs.pop(f"{url_attr}", _IGNORE_URL_PARAM)
+
+        if (
+            sum(
+                int(v is not _IGNORE_URL_PARAM)
+                for v in [in_attr, regex_attr, exact_attr]
+            )
+            > 1
+        ):
+            raise ValueError(
+                f"URL specs may only specify one of {url_attr}, "
+                f"{url_attr}_in, and {url_attr}_regex for any URL attribute"
+            )
+
+        attr_validator = _url_attr_validator(url_attr, exact_attr, regex_attr, in_attr)
+        if attr_validator is not None:
+            child_validators.append(attr_validator)
+
+    if query is not None:
+        query_spec: Optional[Spec] = make_spec(query)
+    else:
+        query_spec = None
+
+    if kwargs:
+        raise ValueError(f"Unused keyword arguments: {kwargs}")
+
+    if query_spec is None and not child_validators:
+        raise ValueError(f"URL specs must include at least one validation rule")
+
+    def validate_parse_result(v: ParseResult) -> Iterator[ErrorDetails]:
+        for validate in child_validators:
+            yield from validate(v)
+
+        if query_spec is not None:
+            query_dict = parse_qs(v.query)
+            yield from query_spec.validate(query_dict)
+
+    def validate_url(s: str) -> Iterator[ErrorDetails]:
+        try:
+            url = urlparse(s)
+        except ValueError as e:
+            yield ErrorDetails(
+                message=f"String does not contain a valid URL: {e}",
+                pred=validate_url,
+                value=s,
+            )
+        else:
+            yield from validate_parse_result(url)
+
+    validators.append(validate_url)
+
+    return ValidatorSpec.from_validators(
+        tag or "url_str", *validators, conformer=conformer
+    )
+
+
 def uuid_spec(
     tag: Optional[Tag] = None,
     versions: Optional[Set[int]] = None,


Test patch:
diff --git a/tests/test_factories.py b/tests/test_factories.py
index 3f368a8..dc141ce 100644
--- a/tests/test_factories.py
+++ b/tests/test_factories.py
@@ -904,6 +904,132 @@ class TestStringFormatValidation:
             assert not conforming_uuid_spec.is_valid(v)
 
 
+class TestURLSpecValidation:
+    @pytest.mark.parametrize(
+        "spec_kwargs",
+        [
+            {},
+            {"host": "coverahealth.com"},
+            {
+                "hostname": "coverahealth.com",
+                "hostname_regex": r"(api\.)?coverahealth.com",
+            },
+            {"port_regex": r"80|443"},
+        ],
+    )
+    def test_invalid_url_specs(self, spec_kwargs):
+        with pytest.raises(ValueError):
+            s.url(**spec_kwargs)
+
+    @pytest.mark.parametrize(
+        "spec_kwargs", [{"hostname_regex": b"coverahealth.com"}, {"port_in": [80, 443]}]
+    )
+    def test_invalid_url_spec_argument_types(self, spec_kwargs):
+        with pytest.raises(TypeError):
+            s.url(**spec_kwargs)
+
+    @pytest.mark.parametrize(
+        "v", [None, 25, 3.14, {}, [], set(), (), "", "//[coverahealth.com"]
+    )
+    def test_invalid_url(self, v):
+        assert not s.url(hostname_regex=r".*").is_valid(v)
+
+    @pytest.fixture
+    def urlstr(self) -> str:
+        return (
+            "https://jdoe:securepass@api.coverahealth.com:80/v1/patients"
+            "?last-name=Smith&last-name=Doe&birth-year=1985#user-profile"
+        )
+
+    def test_valid_query_str(self, urlstr):
+        assert s.url(
+            query={
+                "last-name": [s.all(s.str(), str.istitle)],
+                "birth-year": [s.str(regex=r"\d{4}")],
+            }
+        ).is_valid(urlstr)
+
+    def test_invalid_query_str(self, urlstr):
+        assert not s.url(
+            query={
+                "last-name": [s.str(), {"count": 1}],
+                "birth-year": [s.str(regex=r"\d{4}")],
+            }
+        ).is_valid(urlstr)
+
+    @pytest.mark.parametrize(
+        "spec_kwargs",
+        [
+            {"scheme": "https"},
+            {"scheme_in": {"https", "http"}},
+            {"scheme_regex": r"https?"},
+            {"netloc": "jdoe:securepass@api.coverahealth.com:80"},
+            {
+                "netloc_in": {
+                    "jdoe:securepass@api.coverahealth.com:80",
+                    "jdoe:securepass@api.coverahealth.com:443",
+                }
+            },
+            {"netloc_regex": r"jdoe:securepass@api\.coverahealth\.com:(80|443)"},
+            {"path": "/v1/patients"},
+            {"path_in": {"/v1/patients", "/v2/patients"}},
+            {"path_regex": r"\/(v1|v2)\/patients"},
+            {"fragment": "user-profile"},
+            {"fragment_in": {"user-profile", "user-addresses"}},
+            {"fragment_regex": r"user\-(profile|addresses)"},
+            {"username": "jdoe"},
+            {"username_in": {"jdoe", "doej"}},
+            {"username_regex": r"jdoe"},
+            {"password": "securepass"},
+            {"password_in": {"securepass", "insecurepass"}},
+            {"password_regex": r"(in)?securepass"},
+            {"hostname": "api.coverahealth.com"},
+            {"hostname_in": {"coverahealth.com", "api.coverahealth.com"}},
+            {"hostname_regex": r"(api\.)?coverahealth\.com"},
+            {"port": 80},
+            {"port_in": {80, 443}},
+        ],
+    )
+    def test_is_url_str(self, urlstr: str, spec_kwargs):
+        assert s.url(**spec_kwargs).is_valid(urlstr)
+
+    @pytest.mark.parametrize(
+        "spec_kwargs",
+        [
+            {"scheme": "http"},
+            {"scheme_in": {"http", "ftp"}},
+            {"scheme_regex": r"(ht|f)tp"},
+            {"netloc": "carl:insecurepass@api.coverahealth.com:80"},
+            {
+                "netloc_in": {
+                    "carl:insecurepass@api.coverahealth.com:80",
+                    "carl:insecurepass@api.coverahealth.com:443",
+                }
+            },
+            {"netloc_regex": r"carl:insecurepass@api\.coverahealth\.com:(80|443)"},
+            {"path": "/v2/patients"},
+            {"path_in": {"/v2/patients", "/v3/patients"}},
+            {"path_regex": r"\/(v2|v3)\/patients"},
+            {"fragment": "user-addresses"},
+            {"fragment_in": {"user-phone-numbers", "user-addresses"}},
+            {"fragment_regex": r"user\-(phone\-numbers|addresses)"},
+            {"username": "carl"},
+            {"username_in": {"carl", "vic"}},
+            {"username_regex": r"(carl|vic)"},
+            {"password": "insecurepass"},
+            {"password_in": {"rlysecurepass", "insecurepass"}},
+            {"password_regex": r"(rly|in)securepass"},
+            {"hostname": "coverahealth.com"},
+            {"hostname_in": {"coverahealth.com", "data.coverahealth.com"}},
+            {"hostname_regex": r"(data\.)?coverahealth\.com"},
+            {"port": 21},
+            {"port_in": {21, 443}},
+        ],
+    )
+    def test_is_not_url_str(self, urlstr: str, spec_kwargs):
+        assert not s.url(**spec_kwargs).is_valid(urlstr)
+
+
 class TestUUIDSpecValidation:
     @pytest.mark.parametrize(
         "v",