Repository: litestar-org/litestar

Issue:
Bug: pydantic PrivateAttr fields are inspected for type hints
### Description

If a private field is annotated with unresolvable ForwardRef, openapi schema build fails (see example)

### URL to code causing the issue

_No response_

### MCVE

```python
from typing import TYPE_CHECKING

from litestar import Litestar, post
from litestar._openapi.plugin import OpenAPIPlugin
from pydantic import BaseModel

if TYPE_CHECKING:
    from typing import Any


class Model(BaseModel):
    class Config:
        underscore_attrs_are_private = True

    _value: "Any"


@post("/")
def hello_world(data: Model) -> dict[str, str]:
    """Keeping the tradition alive with hello world."""
    return {"hello": "world"}


def main():
    app = Litestar(route_handlers=[hello_world], )

    app.plugins.get(OpenAPIPlugin).provide_openapi()


if __name__ == '__main__':
    main()
```


### Steps to reproduce

```bash
run the example
```


### Screenshots

```bash
no screenshots
```


### Logs

```bash
.../litestar_schema/app.py:21: LitestarWarning: Use of a synchronous callable <function hello_world at 0x102eafeb0> without setting sync_to_thread is discouraged since synchronous callables can block the main thread if they perform blocking operations. If the callable is guaranteed to be non-blocking, you can set sync_to_thread=False to skip this warning, or set the environmentvariable LITESTAR_WARN_IMPLICIT_SYNC_TO_THREAD=0 to disable warnings of this type entirely.
  def hello_world(data: Model) -> dict[str, str]:
Traceback (most recent call last):
  File ".../litestar_schema/app.py", line 33, in <module>
    main()
  File ".../litestar_schema/app.py", line 29, in main
    app.plugins.get(OpenAPIPlugin).provide_openapi()
  File "/Users/mike0sv/mambaforge/envs/evidently/lib/python3.10/site-packages/litestar/_openapi/plugin.py", line 46, in provide_openapi
    self._openapi_schema = self._build_openapi_schema()
  File "/Users/mike0sv/mambaforge/envs/evidently/lib/python3.10/site-packages/litestar/_openapi/plugin.py", line 37, in _build_openapi_schema
    openapi.paths = {
  File "/Users/mike0sv/mambaforge/envs/evidently/lib/python3.10/site-packages/litestar/_openapi/plugin.py", line 38, in <dictcomp>
    route.path_format or "/": create_path_item_for_route(context, route)
  File "/Users/mike0sv/mambaforge/envs/evidently/lib/python3.10/site-packages/litestar/_openapi/path_item.py", line 137, in create_path_item_for_route
    return path_item_factory.create_path_item()
  File "/Users/mike0sv/mambaforge/envs/evidently/lib/python3.10/site-packages/litestar/_openapi/path_item.py", line 42, in create_path_item
    operation = self.create_operation_for_handler_method(route_handler, HttpMethod(http_method))
  File "/Users/mike0sv/mambaforge/envs/evidently/lib/python3.10/site-packages/litestar/_openapi/path_item.py", line 66, in create_operation_for_handler_method
    request_body = create_request_body(
  File "/Users/mike0sv/mambaforge/envs/evidently/lib/python3.10/site-packages/litestar/_openapi/request_body.py", line 49, in create_request_body
    schema = schema_creator.for_field_definition(data_field)
  File "/Users/mike0sv/mambaforge/envs/evidently/lib/python3.10/site-packages/litestar/_openapi/schema_generation/schema.py", line 327, in for_field_definition
    result = self.for_plugin(field_definition, plugin_for_annotation)
  File "/Users/mike0sv/mambaforge/envs/evidently/lib/python3.10/site-packages/litestar/_openapi/schema_generation/schema.py", line 487, in for_plugin
    schema = plugin.to_openapi_schema(field_definition=field_definition, schema_creator=self)
  File "/Users/mike0sv/mambaforge/envs/evidently/lib/python3.10/site-packages/litestar/contrib/pydantic/pydantic_schema_plugin.py", line 234, in to_openapi_schema
    return self.for_pydantic_model(field_definition=field_definition, schema_creator=schema_creator)
  File "/Users/mike0sv/mambaforge/envs/evidently/lib/python3.10/site-packages/litestar/contrib/pydantic/pydantic_schema_plugin.py", line 250, in for_pydantic_model
    unwrapped_annotation, annotation_hints = pydantic_get_unwrapped_annotation_and_type_hints(annotation)
  File "/Users/mike0sv/mambaforge/envs/evidently/lib/python3.10/site-packages/litestar/contrib/pydantic/utils.py", line 162, in pydantic_get_unwrapped_annotation_and_type_hints
    return annotation, get_type_hints(annotation, include_extras=True)
  File "/Users/mike0sv/mambaforge/envs/evidently/lib/python3.10/site-packages/typing_extensions.py", line 1104, in get_type_hints
    hint = typing.get_type_hints(
  File "/Users/mike0sv/mambaforge/envs/evidently/lib/python3.10/typing.py", line 1836, in get_type_hints
    value = _eval_type(value, base_globals, base_locals)
  File "/Users/mike0sv/mambaforge/envs/evidently/lib/python3.10/typing.py", line 327, in _eval_type
    return t._evaluate(globalns, localns, recursive_guard)
  File "/Users/mike0sv/mambaforge/envs/evidently/lib/python3.10/typing.py", line 695, in _evaluate
    eval(self.__forward_code__, globalns, localns),
  File "<string>", line 1, in <module>
NameError: name 'Any' is not defined. Did you mean: 'any'?
```


### Litestar Version

litestar==2.5.1


### Platform

- [ ] Linux
- [X] Mac
- [ ] Windows
- [ ] Other (Please specify in the description above)

<!-- POLAR PLEDGE BADGE START -->
---
> [!NOTE]  
> While we are open for sponsoring on [GitHub Sponsors](https://github.com/sponsors/litestar-org/) and 
> [OpenCollective](https://opencollective.com/litestar), we also utilize [Polar.sh](https://polar.sh/) to engage in pledge-based sponsorship.
>
> Check out all issues funded or available for funding [on our Polar.sh dashboard](https://polar.sh/litestar-org)
> * If you would like to see an issue prioritized, make a pledge towards it!
> * We receive the pledge once the issue is completed & verified
> * This, along with engagement in the community, helps us know which features are a priority to our users.

<a href="https://polar.sh/litestar-org/litestar/issues/3150">
<picture>
  <source media="(prefers-color-scheme: dark)" srcset="https://polar.sh/api/github/litestar-org/litestar/issues/3150/pledge.svg?darkmode=1">
  <img alt="Fund with Polar" src="https://polar.sh/api/github/litestar-org/litestar/issues/3150/pledge.svg">
</picture>
</a>
<!-- POLAR PLEDGE BADGE END -->



Discussion:
~~Okay so I think the issue here is that we don't respect the `Config.underscore_attrs_are_private`. IMO we simply shouldn't touch the underscore fields at all if this is set. @litestar-org/maintainers?~~

We actually do. This is a different issue.
Issue here is that while we do exclude those fields from the schema, we still try to extract their type annotations. 

Resolving patch:
diff --git a/litestar/contrib/pydantic/pydantic_schema_plugin.py b/litestar/contrib/pydantic/pydantic_schema_plugin.py
--- a/litestar/contrib/pydantic/pydantic_schema_plugin.py
+++ b/litestar/contrib/pydantic/pydantic_schema_plugin.py
@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Optional
 
 from typing_extensions import Annotated
 
@@ -11,14 +11,15 @@
     is_pydantic_constrained_field,
     is_pydantic_model_class,
     is_pydantic_undefined,
-    pydantic_get_unwrapped_annotation_and_type_hints,
+    pydantic_get_type_hints_with_generics_resolved,
+    pydantic_unwrap_and_get_origin,
 )
 from litestar.exceptions import MissingDependencyException
 from litestar.openapi.spec import Example, OpenAPIFormat, OpenAPIType, Schema
 from litestar.plugins import OpenAPISchemaPlugin
 from litestar.types import Empty
 from litestar.typing import FieldDefinition
-from litestar.utils import is_class_and_subclass
+from litestar.utils import is_class_and_subclass, is_generic
 
 try:
     # check if we have pydantic v2 installed, and try to import both versions
@@ -249,17 +250,22 @@ def for_pydantic_model(cls, field_definition: FieldDefinition, schema_creator: S
         """
 
         annotation = field_definition.annotation
-        unwrapped_annotation, annotation_hints = pydantic_get_unwrapped_annotation_and_type_hints(annotation)
+        if is_generic(annotation):
+            is_generic_model = True
+            model = pydantic_unwrap_and_get_origin(annotation) or annotation
+        else:
+            is_generic_model = False
+            model = annotation
 
-        if is_pydantic_2_model(annotation):
-            model_config = annotation.model_config
-            model_field_info = unwrapped_annotation.model_fields
+        if is_pydantic_2_model(model):
+            model_config = model.model_config
+            model_field_info = model.model_fields
             title = model_config.get("title")
             example = model_config.get("example")
             is_v2_model = True
         else:
             model_config = annotation.__config__
-            model_field_info = unwrapped_annotation.__fields__
+            model_field_info = model.__fields__
             title = getattr(model_config, "title", None)
             example = getattr(model_config, "example", None)
             is_v2_model = False
@@ -268,15 +274,34 @@ def for_pydantic_model(cls, field_definition: FieldDefinition, schema_creator: S
             k: getattr(f, "field_info", f) for k, f in model_field_info.items()
         }
 
+        if is_v2_model:
+            # extract the annotations from the FieldInfo. This allows us to skip fields
+            # which have been marked as private
+            model_annotations = {k: field_info.annotation for k, field_info in model_fields.items()}  # type: ignore[union-attr]
+
+        else:
+            # pydantic v1 requires some workarounds here
+            model_annotations = {
+                k: f.outer_type_ if f.required else Optional[f.outer_type_] for k, f in model.__fields__.items()
+            }
+
+        if is_generic_model:
+            # if the model is generic, resolve the type variables. We pass in the
+            # already extracted annotations, to keep the logic of respecting private
+            # fields consistent with the above
+            model_annotations = pydantic_get_type_hints_with_generics_resolved(
+                annotation, model_annotations=model_annotations, include_extras=True
+            )
+
         property_fields = {
-            f.alias if f.alias and schema_creator.prefer_alias else k: FieldDefinition.from_kwarg(
-                annotation=Annotated[annotation_hints[k], f, f.metadata]  # type: ignore[union-attr]
+            field_info.alias if field_info.alias and schema_creator.prefer_alias else k: FieldDefinition.from_kwarg(
+                annotation=Annotated[model_annotations[k], field_info, field_info.metadata]  # type: ignore[union-attr]
                 if is_v2_model
-                else Annotated[annotation_hints[k], f],  # pyright: ignore
-                name=f.alias if f.alias and schema_creator.prefer_alias else k,
-                default=Empty if schema_creator.is_undefined(f.default) else f.default,
+                else Annotated[model_annotations[k], field_info],  # pyright: ignore
+                name=field_info.alias if field_info.alias and schema_creator.prefer_alias else k,
+                default=Empty if schema_creator.is_undefined(field_info.default) else field_info.default,
             )
-            for k, f in model_fields.items()
+            for k, field_info in model_fields.items()
         }
 
         computed_field_definitions = create_field_definitions_for_computed_fields(
diff --git a/litestar/contrib/pydantic/utils.py b/litestar/contrib/pydantic/utils.py
--- a/litestar/contrib/pydantic/utils.py
+++ b/litestar/contrib/pydantic/utils.py
@@ -8,7 +8,7 @@
 from litestar.params import KwargDefinition
 from litestar.types import Empty
 from litestar.typing import FieldDefinition
-from litestar.utils import is_class_and_subclass
+from litestar.utils import deprecated, is_class_and_subclass
 from litestar.utils.predicates import is_generic
 from litestar.utils.typing import (
     _substitute_typevars,
@@ -129,24 +129,32 @@ def pydantic_get_type_hints_with_generics_resolved(
     globalns: dict[str, Any] | None = None,
     localns: dict[str, Any] | None = None,
     include_extras: bool = False,
+    model_annotations: dict[str, Any] | None = None,
 ) -> dict[str, Any]:
     if pydantic_v2 is Empty or (pydantic_v1 is not Empty and is_class_and_subclass(annotation, pydantic_v1.BaseModel)):
         return get_type_hints_with_generics_resolved(annotation)
 
     origin = pydantic_unwrap_and_get_origin(annotation)
     if origin is None:
-        type_hints = get_type_hints(annotation, globalns=globalns, localns=localns, include_extras=include_extras)
+        if model_annotations is None:  # pragma: no cover
+            model_annotations = get_type_hints(
+                annotation, globalns=globalns, localns=localns, include_extras=include_extras
+            )
         typevar_map = {p: p for p in annotation.__pydantic_generic_metadata__["parameters"]}
     else:
-        type_hints = get_type_hints(origin, globalns=globalns, localns=localns, include_extras=include_extras)
+        if model_annotations is None:
+            model_annotations = get_type_hints(
+                origin, globalns=globalns, localns=localns, include_extras=include_extras
+            )
         args = annotation.__pydantic_generic_metadata__["args"]
         parameters = origin.__pydantic_generic_metadata__["parameters"]
         typevar_map = dict(zip(parameters, args))
 
-    return {n: _substitute_typevars(type_, typevar_map) for n, type_ in type_hints.items()}
+    return {n: _substitute_typevars(type_, typevar_map) for n, type_ in model_annotations.items()}
 
 
-def pydantic_get_unwrapped_annotation_and_type_hints(annotation: Any) -> tuple[Any, dict[str, Any]]:
+@deprecated(version="2.6.2")
+def pydantic_get_unwrapped_annotation_and_type_hints(annotation: Any) -> tuple[Any, dict[str, Any]]:  # pragma:  pver
     """Get the unwrapped annotation and the type hints after resolving generics.
 
     Args:


Test patch:
diff --git a/tests/unit/test_contrib/test_pydantic/models.py b/tests/unit/test_contrib/test_pydantic/models.py
--- a/tests/unit/test_contrib/test_pydantic/models.py
+++ b/tests/unit/test_contrib/test_pydantic/models.py
@@ -1,4 +1,4 @@
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Union
 
 from pydantic import BaseModel
 from pydantic.dataclasses import dataclass as pydantic_dataclass
@@ -15,6 +15,7 @@ class PydanticDataclassPerson:
     id: str
     optional: Optional[str]
     complex: Dict[str, List[Dict[str, str]]]
+    union: Union[int, List[str]]
     pets: Optional[List[DataclassPet]] = None
 
 
@@ -24,6 +25,7 @@ class PydanticPerson(BaseModel):
     id: str
     optional: Optional[str]
     complex: Dict[str, List[Dict[str, str]]]
+    union: Union[int, List[str]]
     pets: Optional[List[DataclassPet]] = None
 
 
@@ -33,6 +35,7 @@ class PydanticV1Person(BaseModelV1):
     id: str
     optional: Optional[str]
     complex: Dict[str, List[Dict[str, str]]]
+    union: Union[int, List[str]]
     pets: Optional[List[DataclassPet]] = None
 
 
@@ -43,4 +46,5 @@ class PydanticV1DataclassPerson:
     id: str
     optional: Optional[str]
     complex: Dict[str, List[Dict[str, str]]]
+    union: Union[int, List[str]]
     pets: Optional[List[DataclassPet]] = None
diff --git a/tests/unit/test_contrib/test_pydantic/test_integration.py b/tests/unit/test_contrib/test_pydantic/test_integration.py
--- a/tests/unit/test_contrib/test_pydantic/test_integration.py
+++ b/tests/unit/test_contrib/test_pydantic/test_integration.py
@@ -98,7 +98,7 @@ def my_route_handler(param: int, data: PydanticPerson) -> None:
         response = client.post("/123", json={"first_name": "moishe"})
         extra = response.json().get("extra")
         assert extra is not None
-        assert len(extra) == 4
+        assert len(extra) == 5
 
 
 def test_default_error_handling_v1() -> None:
@@ -110,7 +110,7 @@ def my_route_handler(param: int, data: PydanticV1Person) -> None:
         response = client.post("/123", json={"first_name": "moishe"})
         extra = response.json().get("extra")
         assert extra is not None
-        assert len(extra) == 3
+        assert len(extra) == 4
 
 
 def test_signature_model_invalid_input(
@@ -172,3 +172,35 @@ def test(
                     "key": "other_child.val.1",
                 },
             ]
+
+
+class V1ModelWithPrivateFields(pydantic_v1.BaseModel):
+    class Config:
+        underscore_fields_are_private = True
+
+    _field: str = pydantic_v1.PrivateAttr()
+    # include an invalid annotation here to ensure we never touch those fields
+    _underscore_field: "foo"  # type: ignore[name-defined] # noqa: F821
+    bar: str
+
+
+class V2ModelWithPrivateFields(pydantic_v2.BaseModel):
+    class Config:
+        underscore_fields_are_private = True
+
+    _field: str = pydantic_v2.PrivateAttr()
+    # include an invalid annotation here to ensure we never touch those fields
+    _underscore_field: "foo"  # type: ignore[name-defined] # noqa: F821
+    bar: str
+
+
+@pytest.mark.parametrize("model_type", [V1ModelWithPrivateFields, V2ModelWithPrivateFields])
+def test_private_fields(model_type: Type[Union[pydantic_v1.BaseModel, pydantic_v2.BaseModel]]) -> None:
+    @post("/")
+    async def handler(data: V2ModelWithPrivateFields) -> V2ModelWithPrivateFields:
+        return data
+
+    with create_test_client([handler]) as client:
+        res = client.post("/", json={"bar": "value"})
+        assert res.status_code == 201
+        assert res.json() == {"bar": "value"}
diff --git a/tests/unit/test_contrib/test_pydantic/test_openapi.py b/tests/unit/test_contrib/test_pydantic/test_openapi.py
--- a/tests/unit/test_contrib/test_pydantic/test_openapi.py
+++ b/tests/unit/test_contrib/test_pydantic/test_openapi.py
@@ -346,6 +346,7 @@ def handler(data: cls) -> cls:
                         "items": {"type": "object", "additionalProperties": {"type": "string"}},
                     },
                 },
+                "union": {"oneOf": [{"type": "integer"}, {"items": {"type": "string"}, "type": "array"}]},
                 "pets": {
                     "oneOf": [
                         {"type": "null"},
@@ -357,7 +358,7 @@ def handler(data: cls) -> cls:
                 },
             },
             "type": "object",
-            "required": ["complex", "first_name", "id", "last_name"],
+            "required": ["complex", "first_name", "id", "last_name", "union"],
             "title": f"{cls.__name__}",
         }
 
@@ -561,7 +562,7 @@ def test_create_schema_for_pydantic_model_with_annotated_model_attribute(
         f"""
 {'from __future__ import annotations' if with_future_annotations else ''}
 from typing_extensions import Annotated
-{'from pydantic import BaseModel' if pydantic_version == 'v1' else 'from pydantic.v1 import BaseModel'}
+{'from pydantic import BaseModel' if pydantic_version == 'v2' else 'from pydantic.v1 import BaseModel'}
 
 class Foo(BaseModel):
     foo: Annotated[int, "Foo description"]
diff --git a/tests/unit/test_contrib/test_pydantic/test_schema_plugin.py b/tests/unit/test_contrib/test_pydantic/test_schema_plugin.py
--- a/tests/unit/test_contrib/test_pydantic/test_schema_plugin.py
+++ b/tests/unit/test_contrib/test_pydantic/test_schema_plugin.py
@@ -80,3 +80,30 @@ class Model(pydantic_v2.BaseModel):
     assert schema.properties
     assert schema.properties["string"] == Schema(min_length=1, type=OpenAPIType.STRING)
     assert schema.properties["bytes_"] == Schema(min_length=1, type=OpenAPIType.STRING)
+
+
+class V1ModelWithPrivateFields(pydantic_v1.BaseModel):
+    class Config:
+        underscore_fields_are_private = True
+
+    _field: str = pydantic_v1.PrivateAttr()
+    # include an invalid annotation here to ensure we never touch those fields
+    _underscore_field: "foo"  # type: ignore[name-defined]  # noqa: F821
+
+
+class V2ModelWithPrivateFields(pydantic_v2.BaseModel):
+    class Config:
+        underscore_fields_are_private = True
+
+    _field: str = pydantic_v2.PrivateAttr()
+    # include an invalid annotation here to ensure we never touch those fields
+    _underscore_field: "foo"  # type: ignore[name-defined] # noqa: F821
+
+
+@pytest.mark.parametrize("model_class", [V1ModelWithPrivateFields, V2ModelWithPrivateFields])
+def test_exclude_private_fields(model_class: Type[Union[pydantic_v1.BaseModel, pydantic_v2.BaseModel]]) -> None:
+    # https://github.com/litestar-org/litestar/issues/3150
+    schema = PydanticSchemaPlugin.for_pydantic_model(
+        FieldDefinition.from_annotation(model_class), schema_creator=SchemaCreator(plugins=[PydanticSchemaPlugin()])
+    )
+    assert not schema.properties