Repository: bentoml/BentoML

Issue:
API server SLOs
- [ ]  `max-latency` & `timeout`
    - [x] api server timeout
    - [x]  provide both max-latency and timeout in BentoServer config
    - [x]  default `max-latency`:  `10s`
    - [ ]  default `timeout = 1.5 * max-latency`
    - [ ]  assert that `max-latency` < `timeout`
    - [ ]  provide `bentoml serve` CLI arg for `--max-latency`
    - [ ]  target: 90% < max-latency
- [ ]  `--max-request-size` ?
    - [ ]  provide BentoServer config
    - [ ]  default to `10MB`


Discussion:
Should we implement max latency similar to the `deadline` feature in gRPC or have a 10 max latency PER runner?
Happy to test the implementation down the road & provide feedback. I have a 100% reproducible situation where I run into timeouts even though the code runs fine (I see my result in the terminal)
hey @parano / @bojiang , is the timeout config already implemented? I don't see it in the `bentoml serve` (1.0.16) yet... is it possible to pass this config somehow differently to the container?

found it, thx:
https://docs.bentoml.org/en/latest/guides/configuration.html
`docker run -e BENTOML_CONFIG_OPTIONS='runners.timeout=3600' -it --rm -p 3000:3000 your_service serve --production`
Yes, but the `timeout` config on `app` doesn't work currently. We will work on improving this. Thank you.


Resolving patch:
diff --git a/src/bentoml/_internal/server/http/timeout.py b/src/bentoml/_internal/server/http/timeout.py
new file mode 100644
--- /dev/null
+++ b/src/bentoml/_internal/server/http/timeout.py
@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import anyio
+from starlette.responses import PlainTextResponse
+
+if TYPE_CHECKING:
+    from ... import external_typing as ext
+
+
+class TimeoutMiddleware:
+    def __init__(self, app: ext.ASGIApp, timeout: int) -> None:
+        self.app = app
+        self.timeout = timeout
+
+    async def __call__(
+        self, scope: ext.ASGIScope, receive: ext.ASGIReceive, send: ext.ASGISend
+    ) -> None:
+        if scope["type"] not in ("http", "websocket"):
+            return await self.app(scope, receive, send)
+        async with anyio.create_task_group():
+            try:
+                with anyio.fail_after(self.timeout):
+                    await self.app(scope, receive, send)
+            except TimeoutError:
+                resp = PlainTextResponse(
+                    f"Not able to process the request in {self.timeout} seconds",
+                    status_code=504,
+                )
+                await resp(scope, receive, send)
diff --git a/src/bentoml/_internal/server/http_app.py b/src/bentoml/_internal/server/http_app.py
--- a/src/bentoml/_internal/server/http_app.py
+++ b/src/bentoml/_internal/server/http_app.py
@@ -253,6 +253,11 @@ def client_request_hook(span: Span, _: dict[str, t.Any]) -> None:
                     )
                 )
 
+        from .http.timeout import TimeoutMiddleware
+
+        api_server_timeout = BentoMLContainer.api_server_config.timeout.get()
+        middlewares.append(Middleware(TimeoutMiddleware, timeout=api_server_timeout))
+
         return middlewares
 
     @property
@@ -378,6 +383,9 @@ async def api_func(request: Request) -> Response:
                     )
                 else:
                     response = JSONResponse("", status_code=status)
+            except asyncio.CancelledError:
+                # Special handling for Python 3.7 compatibility
+                raise
             except Exception:  # pylint: disable=broad-except
                 # For all unexpected error, return 500 by default. For example,
                 # if users' model raises an error of division by zero.


Test patch:
diff --git a/tests/e2e/bento_server_http/configs/timeout.yml b/tests/e2e/bento_server_http/configs/timeout.yml
new file mode 100644
--- /dev/null
+++ b/tests/e2e/bento_server_http/configs/timeout.yml
@@ -0,0 +1,2 @@
+api_server:
+  timeout: 1
diff --git a/tests/e2e/bento_server_http/service.py b/tests/e2e/bento_server_http/service.py
--- a/tests/e2e/bento_server_http/service.py
+++ b/tests/e2e/bento_server_http/service.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import typing as t
+import asyncio
 from typing import TYPE_CHECKING
 
 import numpy as np
@@ -52,6 +53,12 @@ def echo_data_metric(data: str) -> str:
     return data
 
 
+@svc.api(input=bentoml.io.Text(), output=bentoml.io.Text())
+async def echo_delay(data: str) -> str:
+    await asyncio.sleep(5)
+    return data
+
+
 @svc.api(input=bentoml.io.Text(), output=bentoml.io.Text())
 def ensure_metrics_are_registered(data: str) -> str:  # pylint: disable=unused-argument
     counters = [
diff --git a/tests/e2e/bento_server_http/tests/test_serve.py b/tests/e2e/bento_server_http/tests/test_serve.py
--- a/tests/e2e/bento_server_http/tests/test_serve.py
+++ b/tests/e2e/bento_server_http/tests/test_serve.py
@@ -1,7 +1,11 @@
+import os
 import sys
 import time
 
+import pytest
+
 import bentoml
+from bentoml.exceptions import BentoMLException
 
 
 def test_http_server(bentoml_home: str):
@@ -88,3 +92,17 @@ def test_serve_from_svc():
         # the process, it should be negative.
         # on all other platforms, this should be 0.
         assert server.process.poll() <= 0
+
+
+def test_serve_with_timeout(bentoml_home: str):
+    server = bentoml.HTTPServer("service.py:svc", port=12346)
+    config_file = os.path.abspath("configs/timeout.yml")
+    env = os.environ.copy()
+    env.update(BENTOML_CONFIG=config_file)
+
+    with server.start(env=env) as client:
+        with pytest.raises(
+            BentoMLException,
+            match="504: b'Not able to process the request in 1 seconds'",
+        ):
+            client.echo_delay()