Repository: bentoml/BentoML
Pull request #5222 (2025-02-19 04:21:30)

feat(metrics): extend histogram buckets to support LLM latencies
Extends histogram bucket ranges to support LLM and GenAI workloads with latencies up to 180s.

Changes:
- Update DEFAULT_BUCKET to extend range from 10s to 180s
- Update tests to verify new ranges
- Update documentation

Link to Devin run: https://app.devin.ai/sessions/4adfe03711b94552b7fca1144f65d7aa
Requested by: user

devin-ai-integration[bot]: ### 🤖 Devin AI Engineer

I'll be helping with this pull request! Here's what you should know:

✅ I will automatically:
- Address comments on this PR. Add "(aside)" to your comment to have me ignore it.
- Look at CI failures and help fix them

Note: I can only respond to comments from users who have write access to this repository.

⚙️ Control Options:
- [ ] Disable automatic comment and CI monitoring

github-actions[bot]: ![Code Coverage](https://img.shields.io/badge/coverage-54.65%25-critical?style=flat)

Package | Statements
------- | ----------
bentoml | 54.65% (9818 / 17965)
**Summary** | 54.65% (9818 / 17965)

<!-- Sticky Pull Request Comment -->

Patch:
diff --git a/docs/source/build-with-bentoml/observability/metrics.rst b/docs/source/build-with-bentoml/observability/metrics.rst
index 817e84e1..1c484b98 100644
--- a/docs/source/build-with-bentoml/observability/metrics.rst
+++ b/docs/source/build-with-bentoml/observability/metrics.rst
@@ -132,7 +132,7 @@ You can customize the `duration bucket size <https://prometheus.io/docs/practice
     - ``duration.factor`` must be greater than 1 to ensure each subsequent bucket is larger than the previous one.
     - The buckets for the ``adaptive_batch_size`` Histogram are calculated based on the ``max_batch_size`` defined. The bucket sizes start at 1 and increase exponentially up to the ``max_batch_size`` with a factor of 2.
 
-By default, BentoML uses the `duration buckets <https://github.com/prometheus/client_python/blob/f17a8361ad3ed5bc47f193ac03b00911120a8d81/prometheus_client/metrics.py#L544>`_ provided by Prometheus.
+By default, BentoML provides optimized histogram buckets ranging from 5ms to 180s, carefully distributed to monitor both fast API calls and long-running LLM/GenAI inference requests. The buckets are strategically placed to cover key latency ranges: fast API calls (5ms-50ms), regular API calls (100ms-1s), long API calls (2.5s-10s), and LLM inference (30s-180s).
 
 Create custom metrics
 ---------------------
@@ -176,9 +176,11 @@ To define custom metrics, use the metric classes from the ``prometheus_client``
               documentation="Time taken for inference",
               labelnames=["endpoint"],
               buckets=(
-                0.005, 0.01, 0.025, 0.05, 0.075,
-                0.1, 0.25, 0.5, 0.75, 1.0,
-                2.5, 5.0, 7.5, 10.0, float("inf"),
+                0.005, 0.01, 0.025, 0.05,         # Fast API calls (5ms - 50ms)
+                0.1, 0.25, 0.5, 1.0,              # Regular API calls (100ms - 1s)
+                2.5, 5.0, 10.0,                   # Long API calls (2.5s - 10s)
+                30.0, 60.0, 120.0, 180.0,         # LLM models (30s - 180s)
+                float("inf"),
               ),
           )
 
diff --git a/src/_bentoml_impl/loader.py b/src/_bentoml_impl/loader.py
index a5d2a775..c47fc063 100644
--- a/src/_bentoml_impl/loader.py
+++ b/src/_bentoml_impl/loader.py
@@ -207,6 +207,10 @@ def import_service(
         return svc
 
     except (ImportError, AttributeError, KeyError, AssertionError) as e:
+        if original_model_store is not None:
+            from bentoml._internal.configuration.containers import BentoMLContainer
+
+            BentoMLContainer.model_store.set(original_model_store)
         sys_path = sys.path.copy()
         message = f'Failed to import service "{service_identifier}": {e}, sys.path: {sys_path}, cwd: {pathlib.Path.cwd()}'
         if (
@@ -221,11 +225,6 @@ def import_service(
         if extra_python_path is not None:
             sys.path.remove(extra_python_path)
 
-        if original_model_store is not None:
-            from bentoml._internal.configuration.containers import BentoMLContainer
-
-            BentoMLContainer.model_store.set(original_model_store)
-
         if original_path is not None:
             os.chdir(original_path)
 
diff --git a/src/bentoml/_internal/utils/metrics.py b/src/bentoml/_internal/utils/metrics.py
index 51f10bc8..3b521ace 100644
--- a/src/bentoml/_internal/utils/metrics.py
+++ b/src/bentoml/_internal/utils/metrics.py
@@ -6,17 +6,18 @@ DEFAULT_BUCKET = (
     0.005,
     0.01,
     0.025,
-    0.05,
-    0.075,
+    0.05,  # Fast API calls (5ms - 50ms)
     0.1,
     0.25,
     0.5,
-    0.75,
-    1.0,
+    1.0,  # Regular API calls (100ms - 1s)
     2.5,
     5.0,
-    7.5,
-    10.0,
+    10.0,  # Long API calls (2.5s - 10s)
+    30.0,
+    60.0,
+    120.0,
+    180.0,  # LLM models (30s - 180s)
     INF,
 )
 


Test patch:
diff --git a/tests/unit/_internal/utils/test_metrics.py b/tests/unit/_internal/utils/test_metrics.py
index a1adbef5..c54001f9 100644
--- a/tests/unit/_internal/utils/test_metrics.py
+++ b/tests/unit/_internal/utils/test_metrics.py
@@ -15,9 +15,29 @@ def _assert_equal(actual: tuple[float, ...], expected: tuple[float, ...]):
 
 
 def test_exponential_buckets():
+    # Test with extended range for LLM workloads
     _assert_equal(
-        exponential_buckets(0.005, 2.0, 10.0),
-        (0.005, 0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.0, INF),
+        exponential_buckets(0.005, 2.0, 180.0),
+        (
+            0.005,
+            0.01,
+            0.02,
+            0.04,
+            0.08,
+            0.16,
+            0.32,
+            0.64,
+            1.28,
+            2.56,
+            5.12,
+            10.24,
+            20.48,
+            40.96,
+            81.92,
+            163.84,
+            180.0,
+            INF,
+        ),
     )
     _assert_equal(
         exponential_buckets(1, 1.5, 100.0),
@@ -187,6 +207,26 @@ def test_linear_buckets():
     )
 
 
+def test_default_bucket():
+    # Verify DEFAULT_BUCKET covers appropriate ranges for both fast APIs and LLM workloads
+    from bentoml._internal.utils.metrics import DEFAULT_BUCKET
+
+    # Test bucket count
+    assert len(DEFAULT_BUCKET) == 16  # Optimized number of buckets
+
+    # Test key latency boundaries
+    assert DEFAULT_BUCKET[0] == 0.005  # Fast API calls start (5ms)
+    assert DEFAULT_BUCKET[4] == 0.1  # Regular API calls start (100ms)
+    assert DEFAULT_BUCKET[8] == 2.5  # Long API calls start (2.5s)
+    assert DEFAULT_BUCKET[11] == 30.0  # LLM models start (30s)
+    assert DEFAULT_BUCKET[-2] == 180.0  # Maximum LLM latency (180s)
+    assert DEFAULT_BUCKET[-1] == float("inf")
+
+    # Test monotonic increase
+    for i in range(len(DEFAULT_BUCKET) - 1):
+        assert DEFAULT_BUCKET[i] < DEFAULT_BUCKET[i + 1]
+
+
 def test_metric_name():
     assert metric_name("runner_name", "metric_name") == "runner_name_metric_name"
     assert metric_name("runner.name", "metric_name") == "runner::name_metric_name"