Repository: litestar-org/litestar

Issue:
Bug: SQA MappedAsDataclass breaks DTO for request body?
### Description

Started encountering undefined name errors with the SQA DTO when using the `MappedAsDataclass` mixin, but only for routes where the DTO annotated type is the request data.

Only having the type in the return annotation works fine.

(Excerpt from my reproduction repo) With an ORM such as this

```python
from sqlalchemy.orm import DeclarativeBase, MappedAsDataclass

class Base(MappedAsDataclass, DeclarativeBase):
    ...

class Post(Base):
    ...
```

Routes without a DTO base request body work fine:

```python
@get( "{id:int}", return_dto=PostDTOs.one)
async def fetch_one(id: int, *, session: Session) -> Post:
    ...
```

But routes like a create or update do not:

```python
@post("/create" dto=PostDTOs.partial, return_dto=PostDTOs.one)
async def create(*, data: Post, session: Session) -> Post:
    ...

@patch("/update", dto=PostDTOs.partial, return_dto=PostDTOs.one)
async def update(id: int, *, data: Post, session: Session) -> Post:
    ...
```
```
  File "/Users/.../.pyenv/versions/3.11.1/lib/python3.11/typing.py", line 854, in _evaluate
    eval(self.__forward_code__, globalns, localns),
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "<string>", line 1, in <module>
NameError: name 'User' is not defined
```

I've created a minimalish repo for testing that reproduces this but thought I'd check here before opening an issue.

https://github.com/nilsso/litestar-test-repro/blob/main/litestar_test/orm/base.py#L7-L12

### URL to code causing the issue

https://github.com/nilsso/litestar-test-repro

### MCVE

_No response_

### Steps to reproduce

- Clone the reproduction repo
- `poetry install`
- `poetry shell`
- `litestar --app litestar_test.app:app run --reload --debug`
- Navigate to <http://localhost:8000/schema/elements#/operations/PostUpdateUpdate>
- Try updating a post
  - ID: `1`
  - Body: `{ "title": "foo" }`

### Screenshots

_No response_

### Logs

```
ERROR - 2023-08-11 13:29:26,190 - litestar - config - exception raised on http connection to route /post/update

Traceback (most recent call last):
  File "/Users/.../Library/Caches/pypoetry/virtualenvs/litestar-test-repro-ZjCOeTJE-py3.11/lib/python3.11/site-packages/litestar/middleware/exceptions/middleware.py", line 174, in __call__
    await self.app(scope, receive, send)
  File "/Users/.../Library/Caches/pypoetry/virtualenvs/litestar-test-repro-ZjCOeTJE-py3.11/lib/python3.11/site-packages/litestar/routes/http.py", line 79, in handle
    response = await self._get_response_for_request(
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/.../Library/Caches/pypoetry/virtualenvs/litestar-test-repro-ZjCOeTJE-py3.11/lib/python3.11/site-packages/litestar/routes/http.py", line 131, in _get_response_for_request
    response = await self._call_handler_function(
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/.../Library/Caches/pypoetry/virtualenvs/litestar-test-repro-ZjCOeTJE-py3.11/lib/python3.11/site-packages/litestar/routes/http.py", line 160, in _call_handler_function
    response_data, cleanup_group = await self._get_response_data(
                                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/.../Library/Caches/pypoetry/virtualenvs/litestar-test-repro-ZjCOeTJE-py3.11/lib/python3.11/site-packages/litestar/routes/http.py", line 194, in _get_response_data
    parsed_kwargs = route_handler.signature_model.parse_values_from_connection_kwargs(
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/.../Library/Caches/pypoetry/virtualenvs/litestar-test-repro-ZjCOeTJE-py3.11/lib/python3.11/site-packages/litestar/_signature/model.py", line 190, in parse_values_from_connection_kwargs
    return convert(kwargs, cls, strict=False, dec_hook=deserializer, str_keys=True).to_dict()
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/.../Library/Caches/pypoetry/virtualenvs/litestar-test-repro-ZjCOeTJE-py3.11/lib/python3.11/site-packages/msgspec/_utils.py", line 209, in get_dataclass_info
    hints = get_class_annotations(obj)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/.../Library/Caches/pypoetry/virtualenvs/litestar-test-repro-ZjCOeTJE-py3.11/lib/python3.11/site-packages/msgspec/_utils.py", line 131, in get_class_annotations
    value = typing._eval_type(value, cls_locals, cls_globals)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/.../.pyenv/versions/3.11.1/lib/python3.11/typing.py", line 359, in _eval_type
    return t._evaluate(globalns, localns, recursive_guard)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/.../.pyenv/versions/3.11.1/lib/python3.11/typing.py", line 854, in _evaluate
    eval(self.__forward_code__, globalns, localns),
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "<string>", line 1, in <module>
NameError: name 'User' is not defined
Traceback (most recent call last):
  File "/Users/.../Library/Caches/pypoetry/virtualenvs/litestar-test-repro-ZjCOeTJE-py3.11/lib/python3.11/site-packages/litestar/middleware/exceptions/middleware.py", line 174, in __call__
    await self.app(scope, receive, send)
  File "/Users/.../Library/Caches/pypoetry/virtualenvs/litestar-test-repro-ZjCOeTJE-py3.11/lib/python3.11/site-packages/litestar/routes/http.py", line 79, in handle
    response = await self._get_response_for_request(
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/.../Library/Caches/pypoetry/virtualenvs/litestar-test-repro-ZjCOeTJE-py3.11/lib/python3.11/site-packages/litestar/routes/http.py", line 131, in _get_response_for_request
    response = await self._call_handler_function(
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/.../Library/Caches/pypoetry/virtualenvs/litestar-test-repro-ZjCOeTJE-py3.11/lib/python3.11/site-packages/litestar/routes/http.py", line 160, in _call_handler_function
    response_data, cleanuINFO:     127.0.0.1:62502 - "PATCH /post/update?id=1 HTTP/1.1" 500 Internal Server Error
p_group = await self._get_response_data(
                                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/.../Library/Caches/pypoetry/virtualenvs/litestar-test-repro-ZjCOeTJE-py3.11/lib/python3.11/site-packages/litestar/routes/http.py", line 194, in _get_response_data
    parsed_kwargs = route_handler.signature_model.parse_values_from_connection_kwargs(
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/.../Library/Caches/pypoetry/virtualenvs/litestar-test-repro-ZjCOeTJE-py3.11/lib/python3.11/site-packages/litestar/_signature/model.py", line 190, in parse_values_from_connection_kwargs
    return convert(kwargs, cls, strict=False, dec_hook=deserializer, str_keys=True).to_dict()
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/.../Library/Caches/pypoetry/virtualenvs/litestar-test-repro-ZjCOeTJE-py3.11/lib/python3.11/site-packages/msgspec/_utils.py", line 209, in get_dataclass_info
    hints = get_class_annotations(obj)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/.../Library/Caches/pypoetry/virtualenvs/litestar-test-repro-ZjCOeTJE-py3.11/lib/python3.11/site-packages/msgspec/_utils.py", line 131, in get_class_annotations
    value = typing._eval_type(value, cls_locals, cls_globals)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/.../.pyenv/versions/3.11.1/lib/python3.11/typing.py", line 359, in _eval_type
    return t._evaluate(globalns, localns, recursive_guard)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/.../.pyenv/versions/3.11.1/lib/python3.11/typing.py", line 854, in _evaluate
    eval(self.__forward_code__, globalns, localns),
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "<string>", line 1, in <module>
NameError: name 'User' is not defined
```

### Litestar Version

2.0.0rc1 (directly from main branch)

### Platform

- [X] Linux
- [X] Mac
- [ ] Windows
- [ ] Other (Please specify in the description above)

<!-- POLAR PLEDGE BADGE START -->
---

## Funding
* If you would like to see an issue prioritized, make a pledge towards it!
* We receive the pledge once the issue is completed & verified

<a href="https://polar.sh/litestar-org/litestar/issues/2149">
<picture>
  <source media="(prefers-color-scheme: dark)" srcset="https://polar.sh/api/github/litestar-org/litestar/issues/2149/pledge.svg?darkmode=1">
  <img alt="Fund with Polar" src="https://polar.sh/api/github/litestar-org/litestar/issues/2149/pledge.svg">
</picture>
</a>
<!-- POLAR PLEDGE BADGE END -->



Discussion:
Can you provide a stripped down MCVE for this?
The repository you posted is rather involved and does a lot of things that could potentially affect this behaviour. Ideally this would be a single SQLAlchemy model, DTO and route handler which reproduce the behaviour you're encountering.
> Can you provide a stripped down MCVE for this? The repository you posted is rather involved and does a lot of things that could potentially affect this behaviour. Ideally this would be a single SQLAlchemy model, DTO and route handler which reproduce the behaviour you're encountering.

Sorry yeah I'll try to come up with a more minimal example. I'm having a tough time coming up with the sort of partial update route that I'd want, so I'll try to replicate with just a create route.
Okay I think I've got it as stripped down as possible, in two branches. The problem seems to be related to msgspec, forward reference/import related, and only with regards to submodels. So the most minimal I can get it is a `User` model with a `Post` posts relationship.

- [Minimal without imports *works fine*](https://github.com/nilsso/litestar-test-repro/tree/min_no_imports)
- [Minimal *with* imports *breaks*](https://github.com/nilsso/litestar-test-repro/tree/min_with_imports)

To replicate:

1. Clone and checkout branch "min_with_imports"/"min_no_imports".
1. `poetry install`, `poetry shell`
1. Run `litestar --app litestar_test.app:app run --reload --debug`.
   [An SQLite database "test.sqlite" will be initialized with a User and a Post.](https://github.com/nilsso/litestar-test-repro/blob/min_no_imports/litestar_test/app.py#L45-L52)
1. Navigate to `http://localhost:8000/schema/elements#/operations/PostCreateCreate`
1. Send API request attempting to create a new Post with user ID 1 (body as follows)
```json
{
  "user_id": 1,
  "title": "foo"
}
```
In the "without imports" branch works fine. Can go to [fetch many](http://localhost:8000/schema/elements#/operations/PostFetchMany) to check.

In the "with imports" branch breaks with the following error
```
  File "/Users/.../Library/Caches/pypoetry/virtualenvs/litestar-test-repro-ZjCOeTJE-py3.11/lib/python3.11/site-packages/litestar/_signature/model.py", line 190, in parse_values_from_connection_kwargs
    return convert(kwargs, cls, strict=False, dec_hook=deserializer, str_keys=True).to_dict()
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/.../Library/Caches/pypoetry/virtualenvs/litestar-test-repro-ZjCOeTJE-py3.11/lib/python3.11/site-packages/msgspec/_utils.py", line 209, in get_dataclass_info
    hints = get_class_annotations(obj)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/.../Library/Caches/pypoetry/virtualenvs/litestar-test-repro-ZjCOeTJE-py3.11/lib/python3.11/site-packages/msgspec/_utils.py", line 131, in get_class_annotations
    value = typing._eval_type(value, cls_locals, cls_globals)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/.../.pyenv/versions/3.11.1/lib/python3.11/typing.py", line 359, in _eval_type
    return t._evaluate(globalns, localns, recursive_guard)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/.../.pyenv/versions/3.11.1/lib/python3.11/typing.py", line 854, in _evaluate
    eval(self.__forward_code__, globalns, localns),
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "<string>", line 1, in <module>
NameError: name 'User' is not defined
```
Can also test the [fetch many](http://localhost:8000/schema/elements#/operations/PostFetchMany) route while in the "with imports" branch to see that it works. Failure seems to be with the create route.
https://docs.litestar.dev/2/usage/routing/handlers.html#signature-namespace

You'll need to add `User` to the signature namespace so the forward reference can be resolved.

Doesn't seem to resolve the issue. I get the same error. It seems to be unique to dataclasses (via `MappedAsDataclass`).

Probably has to do with generated `__init__` from `MappedAsDataclass`, even with `init=False`?
> Doesn't seem to resolve the issue. I get the same error. It seems to be unique to dataclasses (via `MappedAsDataclass`).
> 
> Probably has to do with generated `__init__` from `MappedAsDataclass`, even with `init=False`?

Is this an issue on our end of sqla? @nilsso 
The error raises when we use `msgspec.convert()` to convert the connection data into the signature model instance. 

I haven't confirmed all details of this - particularly the parts pertaining to msgspec internals, but here's my best guess.

https://github.com/litestar-org/litestar/blob/737a45a60e416ff1d96ce1e75b0b97b8f0247d5c/litestar/_signature/model.py#L173-L191

In this function, `kwargs` is the data that has originated from the client which we are parsing/validating per the signature of the handler. Inspecting, we can see:

```py
kwargs.get('data') = Post(title='foo', user_id=1, id=None, user=None)
```

This is as we expect - because the DTO operates before the signature model and has already transformed the raw data into the SQLAlchemy `Post` instance.

Next, I inspected the type of the `data` keyword arg in the signature models fields:

```py
cls.__annotations__.get('data') = <class 'litestar_test.orm.post.Post'>
```
On the signature model we have annotated the type of `data` to `Post` and when `Post` is a dataclass (as in when inheriting from `MappedAsDataclass`) this sends msgspec on a different path to when `Post` is a plain sqlalchemy model, and that path includes inspecting the types of the dataclass. One can only access names that are available in the module scope of the object to be inspected, and given that the `User` name is imported in the `post.py` module in a type checking block:

```py
if TYPE_CHECKING:
    from .user import User
```

...that error is thrown from `msgspec._utils.get_class_annotations()`.

In previous incarnations of the signature model, if a handler's `data` kwarg was managed by a DTO, we'd type it as `Any` on the signature model, b/c there is little to gain by having it typed otherwise. The data is already validated and parsed into an instance, so we really just want the signature model to give us back exactly what we gave it.
> In previous incarnations of the signature model, if a handler's `data` kwarg was managed by a DTO, we'd type it as `Any` on the signature model, b/c there is little to gain by having it typed otherwise. The data is already validated and parsed into an instance, so we really just want the signature model to give us back exactly what we gave it.

This is what I'm talking about here: https://github.com/litestar-org/litestar/commit/b9814c2564b25569b59d962e816126347ac4d286#diff-3bd2004653c6a335364eb4f668cab0283ea1c9bffdfc52ae1537f2dbb9e248a3L53-L60

Before https://github.com/litestar-org/litestar/pull/2088 we would type the data parameter on the signature as `Any` if it was managed by a DTO because the DTO is doing the validation, we don't need the signature model to double up on this.. For whatever reason, it was removed.
The exact specifics of this issue is that we leverage sqlalchemy's mapper registry to inject names into the namespace when we resolve type annotations to build the DTO transfer models (happens in advanced-alchemy now days: https://github.com/jolt-org/advanced-alchemy/blob/5da93a2d22ee70777be11a5e27d205d6699cbfae/advanced_alchemy/extensions/litestar/dto.py#L227).

This allows related models to be imported inside `if TYPE_CHECKING:` without error, however, msgspec doesn't have that same luxury.

I.e., in this module from the reproduction repo:

```py
from __future__ import annotations

from typing import TYPE_CHECKING

from sqlalchemy import ForeignKey
from sqlalchemy.orm import Mapped, mapped_column, relationship

from .base import Base

if TYPE_CHECKING:
    from .user import User


class Post(Base):
    __tablename__ = "post"

    id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True, default=None)
    title: Mapped[str]
    user_id: Mapped[int] = mapped_column(ForeignKey("user.id"), default=None,)
    user: Mapped[User] = relationship(back_populates="posts")
```
...even though the `User` model isn't imported at runtime, we have it in the namespace for resolving type annotations at runtime via the mapper registry.

However, when msgspec tries to inspect the annotations as it validates the `Post` that we pass to `convert()` as part of the signature model instantiation, it cannot find `User` in the module scope b/c it doesn't exist, and we get the failure as above.

The thing is, we already know that the `Post` instance is valid, because we've already validated it with the DTO, and if this wasn't a dataclass, msgspec would just do an `isinstance()` test on it and wave it through. But, because it is a dataclass and msgspec does rich validation on them, it tries to parse the types, and we get the error.

So, if we type `data` as `Any` on the signature model when it is covered by a DTO, then this issue is fixed, but more than that, we should type `data` as `Any` on the signature model because we don't need to validate it twice.

Resolving patch:
diff --git a/litestar/_signature/model.py b/litestar/_signature/model.py
--- a/litestar/_signature/model.py
+++ b/litestar/_signature/model.py
@@ -252,6 +252,7 @@ def create(
                 field_definition=field_definition,
                 type_decoders=[*(type_decoders or []), *DEFAULT_TYPE_DECODERS],
                 meta_data=meta_data,
+                data_dto=data_dto,
             )
 
             default = field_definition.default if field_definition.has_default else NODEFAULT
@@ -277,7 +278,12 @@ def _create_annotation(
         field_definition: FieldDefinition,
         type_decoders: TypeDecodersSequence,
         meta_data: Meta | None = None,
+        data_dto: type[AbstractDTO] | None = None,
     ) -> Any:
+        # DTOs have already validated their data, so we can just use Any here
+        if field_definition.name == "data" and data_dto:
+            return Any
+
         annotation = _normalize_annotation(field_definition=field_definition)
 
         if annotation is Any:


Test patch:
diff --git a/tests/unit/test_signature/test_parsing.py b/tests/unit/test_signature/test_parsing.py
--- a/tests/unit/test_signature/test_parsing.py
+++ b/tests/unit/test_signature/test_parsing.py
@@ -1,12 +1,15 @@
+from dataclasses import dataclass
 from types import ModuleType
 from typing import Any, Callable, Iterable, List, Optional, Sequence, Union
 from unittest.mock import MagicMock
 
+import msgspec
 import pytest
 from typing_extensions import Annotated
 
 from litestar import get
 from litestar._signature import SignatureModel
+from litestar.dto import DataclassDTO
 from litestar.params import Body, Parameter
 from litestar.status_codes import HTTP_200_OK, HTTP_204_NO_CONTENT
 from litestar.testing import TestClient, create_test_client
@@ -181,3 +184,30 @@ def handler(param: annotation) -> None:  # pyright: ignore
         response = client.get("/?param=foo&param=bar&param=123")
         assert response.status_code == 500
         assert "TypeError: Type unions may not contain more than one array-like" in response.text
+
+
+def test_dto_data_typed_as_any() -> None:
+    """DTOs already validate the payload, we don't need the signature model to do it too.
+
+    https://github.com/litestar-org/litestar/issues/2149
+    """
+
+    @dataclass
+    class Test:
+        a: str
+
+    dto = DataclassDTO[Test]
+
+    def fn(data: Test) -> None:
+        pass
+
+    model = SignatureModel.create(
+        dependency_name_set=set(),
+        fn=fn,
+        data_dto=dto,
+        parsed_signature=ParsedSignature.from_fn(fn, signature_namespace={"Test": Test}),
+        type_decoders=[],
+    )
+    (field,) = msgspec.structs.fields(model)
+    assert field.name == "data"
+    assert field.type is Any