Repository: SeldonIO/MLServer

Issue:
fastapi 0.70 failing tests
When using `fastapi` latest version, some tests fail e.g. `test_infer`, if we use an older version of it (e.g. `0.68.2`) tests pass.

We need to take a deeper look just in case it uncovers a more subtle issue with asyncio (judging by the error we get).

here is the error for reference

```
est_infer[sum-model-None] _____________________________________

rest_client = <starlette.testclient.TestClient object at 0x7fa456254bd0>
inference_request = InferenceRequest(id=None, parameters=None, inputs=[RequestInput(name='input-0', shape=[1, 3], datatype='INT32', parameters=None, data=TensorData(__root__=[1, 2, 3]))], outputs=None)
model_name = 'sum-model', model_version = None

    @pytest.mark.parametrize(
        "model_name,model_version", [("sum-model", "v1.2.3"), ("sum-model", None)]
    )
    def test_infer(rest_client, inference_request, model_name, model_version):
        endpoint = f"/v2/models/{model_name}/infer"
        if model_version is not None:
            endpoint = f"/v2/models/{model_name}/versions/{model_version}/infer"
>       response = rest_client.post(endpoint, json=inference_request.dict())

tests/rest/test_endpoints.py:57: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../miniconda3/envs/test_nest_io/lib/python3.7/site-packages/requests/sessions.py:590: in post
    return self.request('POST', url, data=data, json=json, **kwargs)
../../miniconda3/envs/test_nest_io/lib/python3.7/site-packages/starlette/testclient.py:484: in request
    json=json,
../../miniconda3/envs/test_nest_io/lib/python3.7/site-packages/requests/sessions.py:542: in request
    resp = self.send(prep, **send_kwargs)
../../miniconda3/envs/test_nest_io/lib/python3.7/site-packages/requests/sessions.py:655: in send
    r = adapter.send(request, **kwargs)
../../miniconda3/envs/test_nest_io/lib/python3.7/site-packages/starlette/testclient.py:266: in send
    raise exc
../../miniconda3/envs/test_nest_io/lib/python3.7/site-packages/starlette/testclient.py:263: in send
    portal.call(self.app, scope, receive, send)
../../miniconda3/envs/test_nest_io/lib/python3.7/site-packages/anyio/from_thread.py:230: in call
    return cast(T_Retval, self.start_task_soon(func, *args).result())
../../miniconda3/envs/test_nest_io/lib/python3.7/concurrent/futures/_base.py:435: in result
    return self.__get_result()
../../miniconda3/envs/test_nest_io/lib/python3.7/concurrent/futures/_base.py:384: in __get_result
    raise self._exception
../../miniconda3/envs/test_nest_io/lib/python3.7/site-packages/anyio/from_thread.py:177: in _call_func
    retval = await retval
../../miniconda3/envs/test_nest_io/lib/python3.7/site-packages/fastapi/applications.py:208: in __call__
    await super().__call__(scope, receive, send)
../../miniconda3/envs/test_nest_io/lib/python3.7/site-packages/starlette/applications.py:112: in __call__
    await self.middleware_stack(scope, receive, send)
../../miniconda3/envs/test_nest_io/lib/python3.7/site-packages/starlette/middleware/errors.py:181: in __call__
    raise exc
../../miniconda3/envs/test_nest_io/lib/python3.7/site-packages/starlette/middleware/errors.py:159: in __call__
    await self.app(scope, receive, _send)
../../miniconda3/envs/test_nest_io/lib/python3.7/site-packages/starlette/exceptions.py:82: in __call__
    raise exc
../../miniconda3/envs/test_nest_io/lib/python3.7/site-packages/starlette/exceptions.py:71: in __call__
    await self.app(scope, receive, sender)
../../miniconda3/envs/test_nest_io/lib/python3.7/site-packages/starlette/routing.py:656: in __call__
    await route.handle(scope, receive, send)
../../miniconda3/envs/test_nest_io/lib/python3.7/site-packages/starlette/routing.py:259: in handle
    await self.app(scope, receive, send)
../../miniconda3/envs/test_nest_io/lib/python3.7/site-packages/starlette/routing.py:61: in app
    response = await func(request)
mlserver/rest/app.py:24: in custom_route_handler
    return await original_route_handler(request)
../../miniconda3/envs/test_nest_io/lib/python3.7/site-packages/fastapi/routing.py:227: in app
    dependant=dependant, values=values, is_coroutine=is_coroutine
../../miniconda3/envs/test_nest_io/lib/python3.7/site-packages/fastapi/routing.py:159: in run_endpoint_function
    return await dependant.call(**values)
mlserver/rest/endpoints.py:52: in infer
    return await self._data_plane.infer(payload, model_name, model_version)
mlserver/handlers/dataplane.py:60: in infer
    prediction = await model.predict(payload)
mlserver/batching/hooks.py:43: in _inner
    return await batcher.predict(payload)
mlserver/batching/adaptive.py:36: in predict
    return await self._wait_response(internal_id)
mlserver/batching/adaptive.py:55: in _wait_response
    response = await async_response
mlserver/batching/adaptive.py:85: in _batcher
    async for batched in self._batch_requests():
mlserver/batching/adaptive.py:112: in _batch_requests
    read_op, timeout=timeout
../../miniconda3/envs/test_nest_io/lib/python3.7/asyncio/tasks.py:442: in wait_for
    return fut.result()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <Queue at 0x7fa456575490 maxsize=10 tasks=1>

    async def get(self):
        """Remove and return an item from the queue.
    
        If queue is empty, wait until an item is available.
        """
        while self.empty():
            getter = self._loop.create_future()
            self._getters.append(getter)
            try:
>               await getter
E               RuntimeError: Task <Task pending coro=<Queue.get() running at /home/sa/miniconda3/envs/test_nest_io/lib/python3.7/asyncio/queues.py:159> cb=[_release_waiter(<Future pendi...4566d8550>()]>)() at /home/sa/miniconda3/envs/test_nest_io/lib/python3.7/asyncio/tasks.py:392]> got Future <Future pending> attached to a different loop

../../miniconda3/envs/test_nest_io/lib/python3.7/asyncio/queues.py:159: RuntimeError
```



Discussion:
as part of this ticket we should revert any changes in the dev environment that has been added to pin fastapi to an older version.
(`requirements/dev.txt`) 

Resolving patch:
diff --git a/mlserver/batching/adaptive.py b/mlserver/batching/adaptive.py
--- a/mlserver/batching/adaptive.py
+++ b/mlserver/batching/adaptive.py
@@ -1,9 +1,13 @@
+# Required to deal with annotations including `Queue`
+# https://mypy.readthedocs.io/en/latest/common_issues.html#issues-with-code-at-runtime
+from __future__ import annotations
+
 import time
 import asyncio
 
 from asyncio import Future, Queue, wait_for, Task
 from functools import partial
-from typing import AsyncIterator, Awaitable, Dict, Tuple
+from typing import AsyncIterator, Awaitable, Dict, Optional, Tuple
 
 from ..model import MLModel
 from ..types import (
@@ -24,9 +28,7 @@ def __init__(self, model: MLModel):
 
         # Save predict function before it gets decorated
         self._predict_fn = model.predict
-        self._requests: Queue[Tuple[str, InferenceRequest]] = Queue(
-            maxsize=self._max_batch_size
-        )
+        self.__requests: Optional[Queue[Tuple[str, InferenceRequest]]] = None
         self._async_responses: Dict[str, Future[InferenceResponse]] = {}
         self._batching_task = None
 
@@ -35,11 +37,21 @@ async def predict(self, req: InferenceRequest) -> InferenceResponse:
         self._start_batcher_if_needed()
         return await self._wait_response(internal_id)
 
+    @property
+    def _requests(self) -> Queue[Tuple[str, InferenceRequest]]:
+        # NOTE: We need to create Queue within the async request path (and not
+        # during __init__!!) to ensure that it shares the same AsyncIO loop.
+        if self.__requests is None:
+            self.__requests = Queue(maxsize=self._max_batch_size)
+
+        return self.__requests
+
     async def _queue_request(
         self,
         req: InferenceRequest,
     ) -> Tuple[str, Awaitable[InferenceResponse]]:
         internal_id = generate_uuid()
+
         await self._requests.put((internal_id, req))
 
         loop = asyncio.get_running_loop()
diff --git a/mlserver/rest/errors.py b/mlserver/rest/errors.py
--- a/mlserver/rest/errors.py
+++ b/mlserver/rest/errors.py
@@ -10,7 +10,7 @@ class APIErrorResponse(BaseModel):
     error: Optional[str] = None
 
 
-def handle_mlserver_error(request: Request, exc: MLServerError) -> Response:
+async def handle_mlserver_error(request: Request, exc: MLServerError) -> Response:
     err_res = APIErrorResponse(error=str(exc))
     return Response(status_code=status.HTTP_400_BAD_REQUEST, content=err_res.dict())
 
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -35,9 +35,7 @@ def _load_description() -> str:
     packages=find_packages(exclude=["tests", "tests.*"]),
     install_requires=[
         "click",
-        # We pin version of fastapi
-        # check https://github.com/SeldonIO/MLServer/issues/340
-        "fastapi==0.68.2",
+        "fastapi",
         "grpcio",
         "importlib-metadata;python_version<'3.8'",
         "numpy",


Test patch:
diff --git a/tests/fixtures.py b/tests/fixtures.py
--- a/tests/fixtures.py
+++ b/tests/fixtures.py
@@ -6,7 +6,7 @@
 
 class SumModel(MLModel):
     @custom_handler(rest_path="/my-custom-endpoint")
-    def my_payload(self, payload: list) -> int:
+    async def my_payload(self, payload: list) -> int:
         return sum(payload)
 
     async def predict(self, payload: InferenceRequest) -> InferenceResponse: