Repository: pydantic/pydantic

Issue:
(2.4.0/2.4.1) LookupError during schema gen (simple repro)
### Initial Checks

- [X] I confirm that I'm using Pydantic V2

### Description

Was hoping the 2.4.1 patch would fix, but there's a new schema error on these two new versions. (same code works 2.3.0)

Edit: managed to make a simple reproduction.


```
Traceback (most recent call last):
  File "repo/ptest.py", line 33, in <module>
    Foo.model_json_schema()
  File "repo/venv/lib/python3.11/site-packages/pydantic/main.py", line 385, in model_json_schema
    return model_json_schema(
           ^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 2153, in model_json_schema
    return schema_generator_instance.generate(cls.__pydantic_core_schema__, mode=mode)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 410, in generate
    json_schema: JsonSchemaValue = self.generate_inner(schema)
                                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 549, in generate_inner
    json_schema = current_handler(schema)
                  ^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/_internal/_schema_generation_shared.py", line 36, in __call__
    return self.handler(__core_schema)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 506, in handler_func
    json_schema = generate_for_schema_type(schema_or_field)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 1730, in definitions_schema
    self.generate_inner(definition)
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 549, in generate_inner
    json_schema = current_handler(schema)
                  ^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/_internal/_schema_generation_shared.py", line 36, in __call__
    return self.handler(__core_schema)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 523, in new_handler_func
    json_schema = js_modify_function(schema_or_field, current_handler)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/main.py", line 601, in __get_pydantic_json_schema__
    return __handler(__core_schema)
           ^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/_internal/_schema_generation_shared.py", line 36, in __call__
    return self.handler(__core_schema)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 523, in new_handler_func
    json_schema = js_modify_function(schema_or_field, current_handler)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py", line 213, in modify_model_json_schema
    json_schema = handler(schema_or_field)
                  ^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/_internal/_schema_generation_shared.py", line 36, in __call__
    return self.handler(__core_schema)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 506, in handler_func
    json_schema = generate_for_schema_type(schema_or_field)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 1318, in model_schema
    json_schema = self.generate_inner(schema['schema'])
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 549, in generate_inner
    json_schema = current_handler(schema)
                  ^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/_internal/_schema_generation_shared.py", line 36, in __call__
    return self.handler(__core_schema)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 506, in handler_func
    json_schema = generate_for_schema_type(schema_or_field)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 1410, in model_fields_schema
    json_schema = self._named_required_fields_schema(named_required_fields)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 1221, in _named_required_fields_schema
    field_json_schema = self.generate_inner(field).copy()
                        ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 549, in generate_inner
    json_schema = current_handler(schema)
                  ^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/_internal/_schema_generation_shared.py", line 36, in __call__
    return self.handler(__core_schema)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 541, in new_handler_func
    json_schema = js_modify_function(schema_or_field, current_handler)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/_internal/_generate_schema.py", line 955, in json_schema_update_func
    json_schema = {**handler(schema), **json_schema_updates}
                     ^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/_internal/_schema_generation_shared.py", line 36, in __call__
    return self.handler(__core_schema)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 506, in handler_func
    json_schema = generate_for_schema_type(schema_or_field)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 1289, in model_field_schema
    return self.generate_inner(schema['schema'])
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 549, in generate_inner
    json_schema = current_handler(schema)
                  ^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/_internal/_schema_generation_shared.py", line 36, in __call__
    return self.handler(__core_schema)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 506, in handler_func
    json_schema = generate_for_schema_type(schema_or_field)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 1016, in nullable_schema
    inner_json_schema = self.generate_inner(schema['schema'])
                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 549, in generate_inner
    json_schema = current_handler(schema)
                  ^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/_internal/_schema_generation_shared.py", line 36, in __call__
    return self.handler(__core_schema)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/json_schema.py", line 526, in new_handler_func
    original_schema = current_handler.resolve_ref_schema(json_schema)
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "repo/venv/lib/python3.11/site-packages/pydantic/_internal/_schema_generation_shared.py", line 57, in resolve_ref_schema
    raise LookupError(
LookupError: Could not find a ref for #/$defs/__main____Bar-Input__1. Maybe you tried to call resolve_ref_schema from within a recursive model?
```

### Example Code

```Python
from pydantic import BaseModel


class Foo(BaseModel):
    maybe_bar: "Bar | None"


class Bar(BaseModel):
    foo: Foo


Foo.model_json_schema()
```

### Python, Pydantic & OS Version

```Text
pydantic version: 2.4.1
        pydantic-core version: 2.10.1
          pydantic-core build: profile=release pgo=true
                 install path: /repo/venv/lib/python3.11/site-packages/pydantic
               python version: 3.11.5 (main, Aug 25 2023, 13:19:50) [GCC 11.4.0]
                     platform: Linux-6.2.0-33-generic-x86_64-with-glibc2.35
             related packages: typing_extensions-4.8.0 fastapi-0.103.1 mypy-1.5.1 email-validator-2.0.0
```



Discussion:
I've got a slightly related issue with recursive type in [geojson-pydantic](https://github.com/developmentseed/geojson-pydantic/issues/146). Happy to open a new issue if it's not related at all

```python
from pydantic import BaseModel
from typing_extensions import Annotated
from typing import List, Literal, Union, Tuple
from pydantic import Field


class Point(BaseModel):
    type: Literal["Point"]
    coordinates: Union[Tuple[float, float], Tuple[float, float, float]]


class GeometryCollection(BaseModel):
    type: Literal["GeometryCollection"]
    geometries: List["Geometry"]


Geometry = Annotated[
    Union[
        Point,
        GeometryCollection,
    ],
    Field(discriminator="type"),
]

GeometryCollection.model_rebuild()

---------------------------------------------------------------------------
SchemaError                               Traceback (most recent call last)
<ipython-input-2-2c593a18e1c0> in <cell line: 25>()
     23 ]
     24 
---> 25 GeometryCollection.model_rebuild()
     26 

~/Dev/venv/py39/lib/python3.9/site-packages/pydantic/main.py in model_rebuild(cls, force, raise_errors, _parent_namespace_depth, _types_namespace)
    468             # manually override defer_build so complete_model_class doesn't skip building the model again
    469             config = {**cls.model_config, 'defer_build': False}
--> 470             return _model_construction.complete_model_class(
    471                 cls,
    472                 cls.__name__,

~/Dev/venv/py39/lib/python3.9/site-packages/pydantic/_internal/_model_construction.py in complete_model_class(cls, cls_name, config_wrapper, raise_errors, types_namespace)
    496 
    497     # debug(schema)
--> 498     cls.__pydantic_core_schema__ = schema = validate_core_schema(schema)
    499     cls.__pydantic_validator__ = create_schema_validator(schema, core_config, config_wrapper.plugin_settings)
    500     cls.__pydantic_serializer__ = SchemaSerializer(schema, core_config)

~/Dev/venv/py39/lib/python3.9/site-packages/pydantic/_internal/_core_utils.py in validate_core_schema(schema)
    629     if 'PYDANTIC_SKIP_VALIDATING_CORE_SCHEMAS' in os.environ:
    630         return schema
--> 631     return _validate_core_schema(schema)

SchemaError: Invalid Schema:
definitions.definitions.1.model.schema.model-fields.fields.geometries.schema.list.items_schema.tagged-union.choices.GeometryCollection
  Recursion error - cyclic reference detected [type=recursion_loop, input_value={'type': 'model', 'cls': ...yCollection:4353815568'}, input_type=dict]
    For further information visit https://errors.pydantic.dev/2.4/v/recursion_loop
```
seeing it here too, and a local git blame puts the change at #7624


and a slight variant on the original reproducible example gives a `RecursionError` rather than a `LookupError`

```python
from typing import Optional

from pydantic import BaseModel


class Foo(BaseModel):
    bar: "Bar"


class Bar(BaseModel):
    foo: Optional[Foo]


Foo.model_rebuild()
print(Bar.model_json_schema())
```


```pytb
  File "/Users/talley/mambaforge/envs/useq-schema/lib/python3.11/site-packages/pydantic/json_schema.py", line 215, in <dictcomp>
    self.remap_defs_ref(DefsRef(key)): self.remap_json_schema(value)
                                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/talley/mambaforge/envs/useq-schema/lib/python3.11/site-packages/pydantic/json_schema.py", line 219, in remap_json_schema
    schema[key] = self.remap_json_schema(value)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/talley/mambaforge/envs/useq-schema/lib/python3.11/site-packages/pydantic/json_schema.py", line 219, in remap_json_schema
    schema[key] = self.remap_json_schema(value)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/talley/mambaforge/envs/useq-schema/lib/python3.11/site-packages/pydantic/json_schema.py", line 219, in remap_json_schema
    schema[key] = self.remap_json_schema(value)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/talley/mambaforge/envs/useq-schema/lib/python3.11/site-packages/pydantic/json_schema.py", line 208, in remap_json_schema
    return [self.remap_json_schema(item) for item in schema]
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/talley/mambaforge/envs/useq-schema/lib/python3.11/site-packages/pydantic/json_schema.py", line 208, in <listcomp>
    return [self.remap_json_schema(item) for item in schema]
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/talley/mambaforge/envs/useq-schema/lib/python3.11/site-packages/pydantic/json_schema.py", line 219, in remap_json_schema
    schema[key] = self.remap_json_schema(value)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/talley/mambaforge/envs/useq-schema/lib/python3.11/site-packages/pydantic/json_schema.py", line 206, in remap_json_schema
    return self.remap_json_ref(JsonRef(schema))
                               ^^^^^^^^^^^^^^^
RecursionError: maximum recursion depth exceeded while calling a Python object

```
I just want to thank everyone for being so quick to report these schema building issues introduced in 2.4 with clean reproductions of the issue. Every report is helping us improve our test suite, and we are working urgently to address these problems.

Resolving patch:
diff --git a/pydantic/_internal/_generate_schema.py b/pydantic/_internal/_generate_schema.py
--- a/pydantic/_internal/_generate_schema.py
+++ b/pydantic/_internal/_generate_schema.py
@@ -557,6 +557,19 @@ def _model_schema(self, cls: type[BaseModel]) -> core_schema.CoreSchema:
                 self.defs.definitions[model_ref] = self._post_process_generated_schema(schema)
                 return core_schema.definition_reference_schema(model_ref)
 
+    def _unpack_refs_defs(self, schema: CoreSchema) -> CoreSchema:
+        """Unpack all 'definitions' schemas into `GenerateSchema.defs.definitions`
+        and return the inner schema.
+        """
+
+        def get_ref(s: CoreSchema) -> str:
+            return s['ref']  # type: ignore
+
+        if schema['type'] == 'definitions':
+            self.defs.definitions.update({get_ref(s): s for s in schema['definitions']})
+            schema = schema['schema']
+        return schema
+
     def _generate_schema_from_property(self, obj: Any, source: Any) -> core_schema.CoreSchema | None:
         """Try to generate schema from either the `__get_pydantic_core_schema__` function or
         `__pydantic_core_schema__` property.
@@ -593,6 +606,8 @@ def _generate_schema_from_property(self, obj: Any, source: Any) -> core_schema.C
                     source, CallbackGetCoreSchemaHandler(self._generate_schema, self, ref_mode=ref_mode)
                 )
 
+        schema = self._unpack_refs_defs(schema)
+
         ref = get_ref(schema)
         if ref:
             self.defs.definitions[ref] = self._post_process_generated_schema(schema)


Test patch:
diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.py
--- a/tests/test_edge_cases.py
+++ b/tests/test_edge_cases.py
@@ -2663,7 +2663,7 @@ class Outer(BaseModel):
         b: Annotated[Union[Root1, Root2], Field(discriminator='kind')]
 
     validated = Outer.model_validate({'a': {'kind': '1', 'two': None}, 'b': {'kind': '2', 'one': None}})
-    assert validated == Outer(a=Root1(Model1(two=None)), b=Root2(Model2(one=None)))
+    assert validated == Outer(a=Root1(root=Model1(two=None)), b=Root2(root=Model2(one=None)))
 
     assert Outer.model_json_schema() == {
         '$defs': {
@@ -2686,15 +2686,7 @@ class Outer(BaseModel):
                 'type': 'object',
             },
             'Root1': {'allOf': [{'$ref': '#/$defs/Model1'}], 'title': 'Root1'},
-            'Root2': {
-                'properties': {
-                    'kind': {'const': '2', 'default': '2', 'title': 'Kind'},
-                    'one': {'anyOf': [{'$ref': '#/$defs/Model1'}, {'type': 'null'}]},
-                },
-                'required': ['one'],
-                'title': 'Model2',
-                'type': 'object',
-            },
+            'Root2': {'allOf': [{'$ref': '#/$defs/Model2'}], 'title': 'Root2'},
         },
         'properties': {
             'a': {
diff --git a/tests/test_json_schema.py b/tests/test_json_schema.py
--- a/tests/test_json_schema.py
+++ b/tests/test_json_schema.py
@@ -5627,3 +5627,32 @@ class SerializationModel(Model):
     # Ensure the submodels' JSON schemas match the expected mode even when the opposite value is specified:
     assert ValidationModel.model_json_schema(mode='serialization') == Model.model_json_schema(mode='validation')
     assert SerializationModel.model_json_schema(mode='validation') == Model.model_json_schema(mode='serialization')
+
+
+def test_recursive_non_generic_model() -> None:
+    class Foo(BaseModel):
+        maybe_bar: Union[None, 'Bar']
+
+    class Bar(BaseModel):
+        foo: Foo
+
+    # insert_assert(Bar(foo=Foo(maybe_bar=None)).model_dump())
+    assert Bar.model_validate({'foo': {'maybe_bar': None}}).model_dump() == {'foo': {'maybe_bar': None}}
+    # insert_assert(Bar.model_json_schema())
+    assert Bar.model_json_schema() == {
+        '$defs': {
+            'Bar': {
+                'properties': {'foo': {'$ref': '#/$defs/Foo'}},
+                'required': ['foo'],
+                'title': 'Bar',
+                'type': 'object',
+            },
+            'Foo': {
+                'properties': {'maybe_bar': {'anyOf': [{'$ref': '#/$defs/Bar'}, {'type': 'null'}]}},
+                'required': ['maybe_bar'],
+                'title': 'Foo',
+                'type': 'object',
+            },
+        },
+        'allOf': [{'$ref': '#/$defs/Bar'}],
+    }