Repository: pydantic/pydantic

Issue:
V2: Support forward references in recursive types
Example:

```python
JSON = Union[str, int, float, bool, None, Mapping[str, 'JSON'], List['JSON']]
```

Given how popular this feature request was for MyPy ([https://github.com/python/typing/issues/182](https://github.com/python/typing/issues/182)), I think it would be nice if it "just worked" for Pydantic. Since `pydantic-core` already supports recursive types, it's just a matter of properly handling the recursive forward references.

@dmontagu I wonder if we could use similar work to what you did with generics for these sorts of recursive references? Ultimately I think we need to convert this into a `DefinitionReference` core schema, but I suspect we'll have to do a bit of smart stuff like your generics but with the forward references to detect the recursive forward reference and not just try to "resolve" it.


Discussion:
Even if we don't support this in an initial release, we almost certainly need a better error message:
```python
from typing import List, Mapping, Union

from pydantic import BaseModel

JSON = Union[str, int, float, bool, None, Mapping[str, 'JSON'], List['JSON']]


class A(BaseModel):
    x: JSON
    model_config = dict(arbitrary_types_allowed=True)
```

```
Traceback (most recent call last):
  File "/Users/davidmontague/Library/Application Support/JetBrains/PyCharm2023.1/scratches/scratch_8.py", line 8, in <module>
    class A(BaseModel):
  File "/Users/davidmontague/Programming/pydantic/pydantic/pydantic/main.py", line 124, in __new__
    _model_construction.complete_model_class(
  File "/Users/davidmontague/Programming/pydantic/pydantic/pydantic/_internal/_model_construction.py", line 185, in complete_model_class
    cls.__pydantic_validator__ = SchemaValidator(inner_schema, core_config)
pydantic_core._pydantic_core.SchemaError: Error building "typed-dict" validator:
  SchemaError: Field "x":
  SchemaError: Error building "nullable" validator:
  SchemaError: Error building "union" validator:
  SchemaError: Error building "function" validator:
  SchemaError: Error building "dict" validator:
  SchemaError: Error building "nullable" validator:
  SchemaError: Error building "union" validator:
  SchemaError: Error building "function" validator:
  SchemaError: Error building "dict" validator:
  SchemaError: Error building "is-instance" validator:
  SchemaError: 'cls' must be valid as the first argument to 'isinstance'
```
Some notes on this, taken from a comment I just made on [a discussion](https://github.com/pydantic/pydantic/discussions/5779#discussioncomment-5921877).

* Once the type alias syntax introduced in this PEP https://peps.python.org/pep-0695/#generic-type-alias is introduced, I think that will be the obvious way to handle it, though that will require python 3.12+.
* Also, it looks like there is active work toward backporting the TypeAliasType into typing_extensions, and once it is added, I think that will be the obvious way to support it in earlier python versions. See https://github.com/python/typing_extensions/issues/159 and https://github.com/python/typing_extensions/pull/160 for reference.
* For what it's worth, I think the pydantic implementation may end up looking similar to this: https://github.com/pydantic/pydantic/commit/ef5dd26c36e90a331fdb6a0a920c3ce619094366. I later reworked this to try to automatically infer the alias name. Ultimately that was a lot more complicated and likely would have perf and other implications, but fortunately `TypeAliasType` is similar _enough_ to the type `RecursiveType` type I introduced in that commit that I think the same implementation/tests/etc. should work.

(@adriangb seems you are already on top of this, and I'm happy to defer to your plan here if you had a different implementation in mind, just adding a comment here before I forget.)

Resolving patch:
diff --git a/pydantic/_internal/_core_utils.py b/pydantic/_internal/_core_utils.py
--- a/pydantic/_internal/_core_utils.py
+++ b/pydantic/_internal/_core_utils.py
@@ -4,7 +4,7 @@
 from typing import Any, Callable, Iterable, TypeVar, Union, cast
 
 from pydantic_core import CoreSchema, core_schema
-from typing_extensions import TypeGuard, get_args
+from typing_extensions import TypeAliasType, TypeGuard, get_args
 
 from . import _repr
 
@@ -81,8 +81,11 @@ def get_type_ref(type_: type[Any], args_override: tuple[type[Any], ...] | None =
         args = generic_metadata['args'] or args
 
     module_name = getattr(origin, '__module__', '<No __module__>')
-    qualname = getattr(origin, '__qualname__', f'<No __qualname__: {origin}>')
-    type_ref = f'{module_name}.{qualname}:{id(origin)}'
+    if isinstance(origin, TypeAliasType):
+        type_ref = f'{module_name}.{origin.__name__}'
+    else:
+        qualname = getattr(origin, '__qualname__', f'<No __qualname__: {origin}>')
+        type_ref = f'{module_name}.{qualname}:{id(origin)}'
 
     arg_refs: list[str] = []
     for arg in args:
diff --git a/pydantic/_internal/_generate_schema.py b/pydantic/_internal/_generate_schema.py
--- a/pydantic/_internal/_generate_schema.py
+++ b/pydantic/_internal/_generate_schema.py
@@ -14,10 +14,10 @@
 from inspect import Parameter, _ParameterKind, signature
 from itertools import chain
 from types import FunctionType, LambdaType, MethodType
-from typing import TYPE_CHECKING, Any, Callable, ForwardRef, Iterable, Mapping, TypeVar, Union
+from typing import TYPE_CHECKING, Any, Callable, ForwardRef, Iterable, Mapping, TypeVar, Union, cast
 
 from pydantic_core import CoreSchema, core_schema
-from typing_extensions import Annotated, Final, Literal, TypedDict, get_args, get_origin, is_typeddict
+from typing_extensions import Annotated, Final, Literal, TypeAliasType, TypedDict, get_args, get_origin, is_typeddict
 
 from ..annotated import GetCoreSchemaHandler, GetJsonSchemaHandler
 from ..errors import PydanticSchemaGenerationError, PydanticUndefinedAnnotation, PydanticUserError
@@ -437,6 +437,10 @@ def _generate_schema(self, obj: Any) -> core_schema.CoreSchema:  # noqa: C901
             return self._dataclass_schema(obj, None)
 
         origin = get_origin(obj)
+
+        if isinstance(obj, TypeAliasType) or isinstance(origin, TypeAliasType):
+            return self._type_alias_type_schema(obj)
+
         if origin is None:
             return self._arbitrary_type_schema(obj, obj)
 
@@ -653,6 +657,37 @@ def _union_schema(self, union_type: Any) -> core_schema.CoreSchema:
             s = core_schema.nullable_schema(s)
         return s
 
+    def _type_alias_type_schema(
+        self,
+        obj: Any,  # TypeAliasType
+    ) -> CoreSchema:
+        origin = get_origin(obj)
+        if origin is not None and _typing_extra.origin_is_type_alias_type(origin):  # type: ignore
+            origin = cast(Any, origin)
+            ref, schema = self._get_or_cache_recursive_ref(origin)
+            if schema is not None:
+                return schema
+            namespace = (self.types_namespace or {}).copy()
+            new_namespace = {**_typing_extra.get_cls_types_namespace(origin), **namespace}
+            annotation = origin.__value__
+        else:
+            ref, schema = self._get_or_cache_recursive_ref(obj)
+            if schema is not None:
+                return schema
+            namespace = (self.types_namespace or {}).copy()
+            new_namespace = {**_typing_extra.get_cls_types_namespace(obj), **namespace}
+            annotation = obj.__value__
+        self.types_namespace = new_namespace
+        typevars_map = get_standard_typevars_map(obj)
+        annotation = replace_types(annotation, typevars_map)
+        schema = self.generate_schema(annotation)
+        assert schema['type'] != 'definitions'
+        schema['ref'] = ref  # type: ignore
+        self.types_namespace = namespace or None
+        self.recursion_cache[obj] = schema  # type: ignore
+        self.definitions[ref] = schema
+        return schema
+
     def _literal_schema(self, literal_type: Any) -> core_schema.LiteralSchema:
         """
         Generate schema for a Literal.
@@ -1051,10 +1086,7 @@ def _annotated_schema(self, annotated_type: Any) -> core_schema.CoreSchema:
         """
         Generate schema for an Annotated type, e.g. `Annotated[int, Field(...)]` or `Annotated[int, Gt(0)]`.
         """
-        first_arg, *other_args = get_args(annotated_type)
-
-        source_type, annotations = self._prepare_annotations(first_arg, other_args)
-
+        source_type, *annotations = get_args(annotated_type)
         schema = self._apply_annotations(lambda x: x, source_type, annotations)
         # put the default validator last so that TypeAdapter.get_default_value() works
         # even if there are function validators involved
@@ -1357,9 +1389,7 @@ def apply_single_annotation(
             schema['schema'] = inner
         return schema
 
-    schema = _known_annotated_metadata.apply_known_metadata(metadata, schema)
-
-    return schema
+    return _known_annotated_metadata.apply_known_metadata(metadata, schema.copy())
 
 
 def wrap_default(field_info: FieldInfo, schema: core_schema.CoreSchema) -> core_schema.CoreSchema:
diff --git a/pydantic/_internal/_typing_extra.py b/pydantic/_internal/_typing_extra.py
--- a/pydantic/_internal/_typing_extra.py
+++ b/pydantic/_internal/_typing_extra.py
@@ -12,7 +12,7 @@
 from types import GetSetDescriptorType
 from typing import TYPE_CHECKING, Any, ForwardRef
 
-from typing_extensions import Annotated, Final, Literal, TypeGuard, get_args, get_origin
+from typing_extensions import Annotated, Final, Literal, TypeAliasType, TypeGuard, get_args, get_origin
 
 if TYPE_CHECKING:
     from ._dataclasses import StandardDataclass
@@ -443,3 +443,7 @@ def is_dataclass(_cls: type[Any]) -> TypeGuard[type[StandardDataclass]]:
     # The dataclasses.is_dataclass function doesn't seem to provide TypeGuard functionality,
     # so I created this convenience function
     return dataclasses.is_dataclass(_cls)
+
+
+def origin_is_type_alias_type(origin: Any) -> TypeGuard[TypeAliasType]:
+    return isinstance(origin, TypeAliasType)


Test patch:
diff --git a/tests/test_type_alias_type.py b/tests/test_type_alias_type.py
new file mode 100644
--- /dev/null
+++ b/tests/test_type_alias_type.py
@@ -0,0 +1,303 @@
+import datetime
+from typing import Dict, List, Tuple, TypeVar, Union
+
+import pytest
+from annotated_types import MaxLen
+from typing_extensions import Annotated, TypeAliasType
+
+from pydantic import ValidationError
+from pydantic.type_adapter import TypeAdapter
+
+T = TypeVar('T')
+
+JsonType = TypeAliasType('JsonType', Union[List['JsonType'], Dict[str, 'JsonType'], str, int, float, bool, None])
+RecursiveGenericAlias = TypeAliasType(
+    'RecursiveGenericAlias', List[Union['RecursiveGenericAlias[T]', T]], type_params=(T,)
+)
+MyList = TypeAliasType('MyList', List[T], type_params=(T,))
+# try mixing with implicit type aliases
+ShortMyList = Annotated[MyList[T], MaxLen(1)]
+ShortRecursiveGenericAlias = Annotated[RecursiveGenericAlias[T], MaxLen(1)]
+
+
+def test_type_alias() -> None:
+    t = TypeAdapter(MyList[int])
+
+    assert t.validate_python(['1', '2']) == [1, 2]
+
+    with pytest.raises(ValidationError) as exc_info:
+        t.validate_python(['a'])
+    assert exc_info.value.errors(include_url=False) == [
+        {
+            'type': 'int_parsing',
+            'loc': (0,),
+            'msg': 'Input should be a valid integer, unable to parse string as an integer',
+            'input': 'a',
+        }
+    ]
+
+    assert t.json_schema() == {'type': 'array', 'items': {'type': 'integer'}}
+
+
+def test_recursive_type_alias() -> None:
+    t = TypeAdapter(JsonType)
+
+    assert t.validate_python({'a': [True, [{'b': None}]]}) == {'a': [True, [{'b': None}]]}
+
+    with pytest.raises(ValidationError) as exc_info:
+        t.validate_python({'a': datetime.date(year=1992, month=12, day=11)})
+    assert exc_info.value.errors(include_url=False) == [
+        {
+            'type': 'list_type',
+            'loc': ('list[nullable[union[list[...],dict[str,...],str,int,float,bool]]]',),
+            'msg': 'Input should be a valid list',
+            'input': {'a': datetime.date(1992, 12, 11)},
+        },
+        {
+            'type': 'list_type',
+            'loc': ('dict[str,...]', 'a', 'list[nullable[union[list[...],dict[str,...],str,int,float,bool]]]'),
+            'msg': 'Input should be a valid list',
+            'input': datetime.date(1992, 12, 11),
+        },
+        {
+            'type': 'dict_type',
+            'loc': ('dict[str,...]', 'a', 'dict[str,...]'),
+            'msg': 'Input should be a valid dictionary',
+            'input': datetime.date(1992, 12, 11),
+        },
+        {
+            'type': 'string_type',
+            'loc': ('dict[str,...]', 'a', 'str'),
+            'msg': 'Input should be a valid string',
+            'input': datetime.date(1992, 12, 11),
+        },
+        {
+            'type': 'int_type',
+            'loc': ('dict[str,...]', 'a', 'int'),
+            'msg': 'Input should be a valid integer',
+            'input': datetime.date(1992, 12, 11),
+        },
+        {
+            'type': 'float_type',
+            'loc': ('dict[str,...]', 'a', 'float'),
+            'msg': 'Input should be a valid number',
+            'input': datetime.date(1992, 12, 11),
+        },
+        {
+            'type': 'bool_type',
+            'loc': ('dict[str,...]', 'a', 'bool'),
+            'msg': 'Input should be a valid boolean',
+            'input': datetime.date(1992, 12, 11),
+        },
+        {
+            'type': 'string_type',
+            'loc': ('str',),
+            'msg': 'Input should be a valid string',
+            'input': {'a': datetime.date(1992, 12, 11)},
+        },
+        {
+            'type': 'int_type',
+            'loc': ('int',),
+            'msg': 'Input should be a valid integer',
+            'input': {'a': datetime.date(1992, 12, 11)},
+        },
+        {
+            'type': 'float_type',
+            'loc': ('float',),
+            'msg': 'Input should be a valid number',
+            'input': {'a': datetime.date(1992, 12, 11)},
+        },
+        {
+            'type': 'bool_type',
+            'loc': ('bool',),
+            'msg': 'Input should be a valid boolean',
+            'input': {'a': datetime.date(1992, 12, 11)},
+        },
+    ]
+
+    assert t.json_schema() == {
+        'allOf': [{'$ref': '#/$defs/JsonType'}],
+        '$defs': {
+            'JsonType': {
+                'anyOf': [
+                    {'type': 'array', 'items': {'$ref': '#/$defs/JsonType'}},
+                    {'type': 'object', 'additionalProperties': {'$ref': '#/$defs/JsonType'}},
+                    {'type': 'string'},
+                    {'type': 'integer'},
+                    {'type': 'number'},
+                    {'type': 'boolean'},
+                    {'type': 'null'},
+                ]
+            }
+        },
+    }
+
+
+def test_type_alias_annotated() -> None:
+    t = TypeAdapter(ShortMyList[int])
+
+    assert t.validate_python(['1']) == [1]
+
+    with pytest.raises(ValidationError) as exc_info:
+        t.validate_python([1, 2])
+    assert exc_info.value.errors(include_url=False) == [
+        {
+            'type': 'too_long',
+            'loc': (),
+            'msg': 'List should have at most 1 item after validation, not 2',
+            'input': [1, 2],
+            'ctx': {'field_type': 'List', 'max_length': 1, 'actual_length': 2},
+        }
+    ]
+
+    assert t.json_schema() == {'type': 'array', 'items': {'type': 'integer'}, 'maxItems': 1}
+
+
+@pytest.mark.xfail(reason='Not working yet, we cannot apply constraints to named/ref types')
+def test_type_alias_annotated_defs() -> None:
+    # force use of refs by referencing the schema in multiple places
+    t = TypeAdapter(Tuple[ShortMyList[int], ShortMyList[int]])
+
+    assert t.validate_python((['1'], ['2'])) == ([1], [2])
+
+    with pytest.raises(ValidationError) as exc_info:
+        t.validate_python(([1, 2], [1, 2]))
+    assert exc_info.value.errors(include_url=False) == [
+        {
+            'type': 'too_long',
+            'loc': (0,),
+            'msg': 'List should have at most 1 item after validation, not 2',
+            'input': [1, 2],
+            'ctx': {'field_type': 'List', 'max_length': 1, 'actual_length': 2},
+        },
+        {
+            'type': 'too_long',
+            'loc': (1,),
+            'msg': 'List should have at most 1 item after validation, not 2',
+            'input': [1, 2],
+            'ctx': {'field_type': 'List', 'max_length': 1, 'actual_length': 2},
+        },
+    ]
+
+    assert t.json_schema() == {
+        'type': 'array',
+        'minItems': 2,
+        'prefixItems': [
+            {'$ref': '#/$defs/MyList_MaxLen_max_length_1_'},
+            {'$ref': '#/$defs/MyList_MaxLen_max_length_1_'},
+        ],
+        'maxItems': 2,
+        '$defs': {'MyList_MaxLen_max_length_1_': {'type': 'array', 'items': {'type': 'integer'}, 'maxItems': 1}},
+    }
+
+
+def test_recursive_generic_type_alias() -> None:
+    t = TypeAdapter(RecursiveGenericAlias[int])
+
+    assert t.validate_python([[['1']]]) == [[[1]]]
+
+    with pytest.raises(ValidationError) as exc_info:
+        t.validate_python([[['a']]])
+    assert exc_info.value.errors(include_url=False) == [
+        {
+            'type': 'list_type',
+            'loc': (0, 'list[union[...,int]]', 0, 'list[union[...,int]]', 0, 'list[union[...,int]]'),
+            'msg': 'Input should be a valid list',
+            'input': 'a',
+        },
+        {
+            'type': 'int_parsing',
+            'loc': (0, 'list[union[...,int]]', 0, 'list[union[...,int]]', 0, 'int'),
+            'msg': 'Input should be a valid integer, unable to parse string as an integer',
+            'input': 'a',
+        },
+        {
+            'type': 'int_type',
+            'loc': (0, 'list[union[...,int]]', 0, 'int'),
+            'msg': 'Input should be a valid integer',
+            'input': ['a'],
+        },
+        {'type': 'int_type', 'loc': (0, 'int'), 'msg': 'Input should be a valid integer', 'input': [['a']]},
+    ]
+
+    assert t.json_schema() == {
+        'allOf': [{'$ref': '#/$defs/RecursiveGenericAlias'}],
+        '$defs': {
+            'RecursiveGenericAlias': {
+                'type': 'array',
+                'items': {'anyOf': [{'$ref': '#/$defs/RecursiveGenericAlias'}, {'type': 'integer'}]},
+            }
+        },
+    }
+
+
+@pytest.mark.xfail(reason='Not working yet, we cannot apply constraints to named/ref types')
+def test_recursive_generic_type_alias_annotated() -> None:
+    t = TypeAdapter(ShortRecursiveGenericAlias[int])
+
+    assert t.validate_python([[]]) == [[]]
+
+    with pytest.raises(ValidationError) as exc_info:
+        t.validate_python([[], []])
+    assert exc_info.value.errors(include_url=False) == [
+        {
+            'type': 'too_long',
+            'loc': (),
+            'msg': 'List should have at most 1 item after validation, not 2',
+            'input': [[], []],
+            'ctx': {'field_type': 'List', 'max_length': 1, 'actual_length': 2},
+        }
+    ]
+
+    assert t.json_schema() == {
+        'type': 'array',
+        'items': {'anyOf': [{'$ref': '#/$defs/RecursiveGenericAlias'}, {'type': 'integer'}]},
+        'maxItems': 1,
+        '$defs': {
+            'RecursiveGenericAlias': {
+                'type': 'array',
+                'items': {'anyOf': [{'$ref': '#/$defs/RecursiveGenericAlias'}, {'type': 'integer'}]},
+            }
+        },
+    }
+
+
+@pytest.mark.xfail(reason='Not working yet, we cannot apply constraints to named/ref types')
+def test_recursive_generic_type_alias_annotated_defs() -> None:
+    # force use of refs by referencing the schema in multiple places
+    t = TypeAdapter(Tuple[ShortRecursiveGenericAlias[int], ShortRecursiveGenericAlias[int]])
+
+    assert t.validate_python(([[]], [[]])) == ([[]], [[]])
+
+    with pytest.raises(ValidationError) as exc_info:
+        t.validate_python(([[], []], [[]]))
+    assert exc_info.value.errors(include_url=False) == [
+        {
+            'type': 'too_long',
+            'loc': (0,),
+            'msg': 'List should have at most 1 item after validation, not 2',
+            'input': [[], []],
+            'ctx': {'field_type': 'List', 'max_length': 1, 'actual_length': 2},
+        }
+    ]
+
+    assert t.json_schema() == {
+        'type': 'array',
+        'minItems': 2,
+        'prefixItems': [
+            {'$ref': '#/$defs/RecursiveGenericAlias_MaxLen_max_length_1_'},
+            {'$ref': '#/$defs/RecursiveGenericAlias_MaxLen_max_length_1_'},
+        ],
+        'maxItems': 2,
+        '$defs': {
+            'RecursiveGenericAlias': {
+                'type': 'array',
+                'items': {'anyOf': [{'$ref': '#/$defs/RecursiveGenericAlias'}, {'type': 'integer'}]},
+            },
+            'RecursiveGenericAlias_MaxLen_max_length_1_': {
+                'type': 'array',
+                'items': {'anyOf': [{'$ref': '#/$defs/RecursiveGenericAlias'}, {'type': 'integer'}]},
+                'maxItems': 1,
+            },
+        },
+    }