Repository: bentoml/BentoML

Issue:
Add configuration option for runner timeout
Long model inference request could timeout the current runner request. Runner timeout is currently hardcoded at 5 minutes. We should make the runner timeout value configurable. SageMaker deployment through bentoctl can generate the configuration with an updated timeout value during build.


Discussion:

Resolving patch:
diff --git a/bentoml/_internal/configuration/containers.py b/bentoml/_internal/configuration/containers.py
--- a/bentoml/_internal/configuration/containers.py
+++ b/bentoml/_internal/configuration/containers.py
@@ -97,6 +97,7 @@ def _is_ip_address(addr: str) -> bool:
             Optional("response_content_type"): Or(bool, None),
         },
     },
+    Optional("timeout"): And(int, _larger_than_zero),
 }
 
 SCHEMA = Schema(
@@ -216,10 +217,10 @@ def __init__(
 
             global_runner_cfg = {
                 k: self.config["runners"][k]
-                for k in ("batching", "resources", "logging")
+                for k in ("batching", "resources", "logging", "timeout")
             }
             for key in self.config["runners"]:
-                if key not in ["batching", "resources", "logging"]:
+                if key not in ["batching", "resources", "logging", "timeout"]:
                     runner_cfg = self.config["runners"][key]
 
                     # key is a runner name
diff --git a/bentoml/_internal/runner/runner_handle/remote.py b/bentoml/_internal/runner/runner_handle/remote.py
--- a/bentoml/_internal/runner/runner_handle/remote.py
+++ b/bentoml/_internal/runner/runner_handle/remote.py
@@ -41,6 +41,15 @@ def __init__(self, runner: Runner):  # pylint: disable=super-init-not-called
     def _remote_runner_server_map(self) -> dict[str, str]:
         return BentoMLContainer.remote_runner_mapping.get()
 
+    @property
+    def runner_timeout(self) -> int:
+        "return the configured timeout for this runner."
+        runner_cfg = BentoMLContainer.runners_config.get()
+        if self._runner.name in runner_cfg:
+            return runner_cfg[self._runner.name]["timeout"]
+        else:
+            return runner_cfg["timeout"]
+
     def _close_conn(self) -> None:
         if self._conn:
             self._conn.close()
@@ -81,7 +90,6 @@ def _get_conn(self) -> BaseConnector:
     @property
     def _client(
         self,
-        timeout_sec: float | None = None,
     ) -> ClientSession:
         import aiohttp
 
@@ -100,11 +108,7 @@ def strip_query_params(url: yarl.URL) -> str:
                 return str(url.with_query(None))
 
             jar = aiohttp.DummyCookieJar()
-            if timeout_sec is not None:
-                timeout = aiohttp.ClientTimeout(total=timeout_sec)
-            else:
-                DEFAULT_TIMEOUT = aiohttp.ClientTimeout(total=5 * 60)
-                timeout = DEFAULT_TIMEOUT
+            timeout = aiohttp.ClientTimeout(total=self.runner_timeout)
             self._client_cache = aiohttp.ClientSession(
                 trace_configs=[
                     create_trace_config(


Test patch:
diff --git a/tests/unit/_internal/test_configuration.py b/tests/unit/_internal/test_configuration.py
--- a/tests/unit/_internal/test_configuration.py
+++ b/tests/unit/_internal/test_configuration.py
@@ -90,3 +90,21 @@ def test_runner_gpu_configuration():
     bentoml_cfg = get_bentomlconfiguration_from_str(GPU_INDEX_WITH_STRING)
     # this behaviour can be confusing
     assert bentoml_cfg["runners"]["resources"] == {"nvidia.com/gpu": "[1, 2, 4]"}
+
+
+RUNNER_TIMEOUTS = """\
+runners:
+    timeout: 50
+    test_runner_1:
+        timeout: 100
+    test_runner_2:
+        resources: system
+"""
+
+
+def test_runner_timeouts():
+    bentoml_cfg = get_bentomlconfiguration_from_str(RUNNER_TIMEOUTS)
+    runner_cfg = bentoml_cfg["runners"]
+    assert runner_cfg["timeout"] == 50
+    assert runner_cfg["test_runner_1"]["timeout"] == 100
+    assert runner_cfg["test_runner_2"]["timeout"] == 50