Repository: njhill/vllm
Pull request #6: [Misc][DP] Fix AsyncLLM metrics for multi-API server deployments
Author: login: njhill
Created: 2025-05-13T18:56:06Z | Merged: 2025-05-16T19:51:07Z



Diff:
diff --git a/vllm/entrypoints/cli/serve.py b/vllm/entrypoints/cli/serve.py
index 5c2f94e08ed0..4fe1d3a780ce 100644
--- a/vllm/entrypoints/cli/serve.py
+++ b/vllm/entrypoints/cli/serve.py
@@ -26,6 +26,7 @@
 from vllm.v1.engine.core import EngineCoreProc
 from vllm.v1.engine.core_client import CoreEngineProcManager
 from vllm.v1.executor.abstract import Executor
+from vllm.v1.metrics.prometheus import setup_multiprocess_prometheus
 from vllm.v1.utils import (CoreEngine, get_engine_client_zmq_addr,
                            wait_for_engine_startup)
 
@@ -162,7 +163,9 @@ def run_multi_api_server(args: argparse.Namespace):
 
     assert not args.headless
     num_api_servers = args.api_server_count
-    # assert num_api_servers > 1
+    assert num_api_servers > 1
+
+    setup_multiprocess_prometheus()
 
     listen_address, sock = setup_server(args)
 
diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py
index 97936048dc20..7261bdfa03ce 100644
--- a/vllm/entrypoints/openai/api_server.py
+++ b/vllm/entrypoints/openai/api_server.py
@@ -24,6 +24,8 @@
 from fastapi.exceptions import RequestValidationError
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse, Response, StreamingResponse
+from prometheus_client import make_asgi_app
+from prometheus_fastapi_instrumentator import Instrumentator
 from starlette.concurrency import iterate_in_threadpool
 from starlette.datastructures import State
 from starlette.routing import Mount
@@ -97,6 +99,7 @@
 from vllm.usage.usage_lib import UsageContext
 from vllm.utils import (Device, FlexibleArgumentParser, get_open_zmq_ipc_path,
                         is_valid_ipv6_address, set_ulimit)
+from vllm.v1.metrics.prometheus import get_prometheus_registry
 from vllm.version import __version__ as VLLM_VERSION
 
 TIMEOUT_KEEP_ALIVE = 5  # seconds
@@ -323,22 +326,9 @@ async def validate_json_request(raw_request: Request):
 
 
 def mount_metrics(app: FastAPI):
-    # Lazy import for prometheus multiprocessing.
-    # We need to set PROMETHEUS_MULTIPROC_DIR environment variable
-    # before prometheus_client is imported.
-    # See https://prometheus.github.io/client_python/multiprocess/
-    from prometheus_client import (REGISTRY, CollectorRegistry, make_asgi_app,
-                                   multiprocess)
-    from prometheus_fastapi_instrumentator import Instrumentator
-
-    registry = REGISTRY
-
-    prometheus_multiproc_dir_path = os.getenv("PROMETHEUS_MULTIPROC_DIR", None)
-    if prometheus_multiproc_dir_path is not None:
-        logger.debug("vLLM to use %s as PROMETHEUS_MULTIPROC_DIR",
-                     prometheus_multiproc_dir_path)
-        registry = CollectorRegistry()
-        multiprocess.MultiProcessCollector(registry)
+    """Mount prometheus metrics to a FastAPI app."""
+
+    registry = get_prometheus_registry()
 
     Instrumentator(
         excluded_handlers=[
diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py
index ab6200e6f18a..7026793befb0 100644
--- a/vllm/v1/engine/async_llm.py
+++ b/vllm/v1/engine/async_llm.py
@@ -34,6 +34,7 @@
 from vllm.v1.executor.abstract import Executor
 from vllm.v1.metrics.loggers import (StatLoggerBase, StatLoggerFactory,
                                      setup_default_loggers)
+from vllm.v1.metrics.prometheus import shutdown_prometheus
 from vllm.v1.metrics.stats import IterationStats, SchedulerStats
 
 logger = init_logger(__name__)
@@ -198,6 +199,8 @@ def __del__(self):
     def shutdown(self):
         """Shutdown, cleaning up the background proc and IPC."""
 
+        shutdown_prometheus()
+
         if engine_core := getattr(self, "engine_core", None):
             engine_core.shutdown()
 
diff --git a/vllm/v1/metrics/loggers.py b/vllm/v1/metrics/loggers.py
index 3bc6e291342b..48e03f4e99d4 100644
--- a/vllm/v1/metrics/loggers.py
+++ b/vllm/v1/metrics/loggers.py
@@ -12,13 +12,12 @@
 from vllm.logger import init_logger
 from vllm.v1.core.kv_cache_utils import PrefixCachingMetrics
 from vllm.v1.engine import FinishReason
+from vllm.v1.metrics.prometheus import unregister_vllm_metrics
 from vllm.v1.metrics.stats import IterationStats, SchedulerStats
 from vllm.v1.spec_decode.metrics import SpecDecodingLogging, SpecDecodingProm
 
 logger = init_logger(__name__)
 
-_LOCAL_LOGGING_INTERVAL_SEC = 5.0
-
 StatLoggerFactory = Callable[[VllmConfig, int], "StatLoggerBase"]
 
 
@@ -143,7 +142,8 @@ def log_engine_initialized(self):
 class PrometheusStatLogger(StatLoggerBase):
 
     def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
-        self._unregister_vllm_metrics()
+
+        unregister_vllm_metrics()
         self.vllm_config = vllm_config
         self.engine_index = engine_index
         # Use this flag to hide metrics that were deprecated in
@@ -168,11 +168,13 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
         self.gauge_scheduler_running = prometheus_client.Gauge(
             name="vllm:num_requests_running",
             documentation="Number of requests in model execution batches.",
+            multiprocess_mode="mostrecent",
             labelnames=labelnames).labels(*labelvalues)
 
         self.gauge_scheduler_waiting = prometheus_client.Gauge(
             name="vllm:num_requests_waiting",
             documentation="Number of requests waiting to be processed.",
+            multiprocess_mode="mostrecent",
             labelnames=labelnames).labels(*labelvalues)
 
         #
@@ -181,6 +183,7 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
         self.gauge_gpu_cache_usage = prometheus_client.Gauge(
             name="vllm:gpu_cache_usage_perc",
             documentation="GPU KV-cache usage. 1 means 100 percent usage.",
+            multiprocess_mode="mostrecent",
             labelnames=labelnames).labels(*labelvalues)
 
         self.counter_gpu_prefix_cache_queries = prometheus_client.Counter(
@@ -241,6 +244,9 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
                 buckets=build_1_2_5_buckets(max_model_len),
                 labelnames=labelnames).labels(*labelvalues)
 
+        # TODO: This metric might be incorrect in case of using multiple
+        # api_server counts which uses prometheus mp.
+        # See: https://github.com/vllm-project/vllm/pull/18053
         self.histogram_iteration_tokens = \
             prometheus_client.Histogram(
                 name="vllm:iteration_tokens_total",
@@ -339,6 +345,9 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
         #
         # LoRA metrics
         #
+
+        # TODO: This metric might be incorrect in case of using multiple
+        # api_server counts which uses prometheus mp.
         self.gauge_lora_info: Optional[prometheus_client.Gauge] = None
         if vllm_config.lora_config is not None:
             self.labelname_max_lora = "max_lora"
@@ -349,13 +358,16 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
                 prometheus_client.Gauge(
                     name="vllm:lora_requests_info",
                     documentation="Running stats on lora requests.",
+                    multiprocess_mode="sum",
                     labelnames=[
                         self.labelname_max_lora,
                         self.labelname_waiting_lora_adapters,
                         self.labelname_running_lora_adapters,
-                    ])
+                    ],
+                )
 
     def log_metrics_info(self, type: str, config_obj: SupportsMetricsInfo):
+
         metrics_info = config_obj.metrics_info()
         metrics_info["engine"] = self.engine_index
 
@@ -371,7 +383,9 @@ def log_metrics_info(self, type: str, config_obj: SupportsMetricsInfo):
         info_gauge = prometheus_client.Gauge(
             name=name,
             documentation=documentation,
-            labelnames=metrics_info.keys()).labels(**metrics_info)
+            multiprocess_mode="mostrecent",
+            labelnames=metrics_info.keys(),
+        ).labels(**metrics_info)
         info_gauge.set(1)
 
     def record(self, scheduler_stats: Optional[SchedulerStats],
@@ -445,13 +459,6 @@ def record(self, scheduler_stats: Optional[SchedulerStats],
             self.gauge_lora_info.labels(**lora_info_labels)\
                                 .set_to_current_time()
 
-    @staticmethod
-    def _unregister_vllm_metrics():
-        # Unregister any existing vLLM collectors (for CI/CD
-        for collector in list(prometheus_client.REGISTRY._collector_to_names):
-            if hasattr(collector, "_name") and "vllm" in collector._name:
-                prometheus_client.REGISTRY.unregister(collector)
-
     def log_engine_initialized(self):
         self.log_metrics_info("cache_config", self.vllm_config.cache_config)
 
diff --git a/vllm/v1/metrics/prometheus.py b/vllm/v1/metrics/prometheus.py
new file mode 100644
index 000000000000..c958d7cd31f0
--- /dev/null
+++ b/vllm/v1/metrics/prometheus.py
@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import tempfile
+from typing import Optional
+
+from prometheus_client import REGISTRY, CollectorRegistry, multiprocess
+
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+# Global temporary directory for prometheus multiprocessing
+_prometheus_multiproc_dir: Optional[tempfile.TemporaryDirectory] = None
+
+
+def setup_multiprocess_prometheus():
+    """Set up prometheus multiprocessing directory if not already configured.
+    
+    """
+    global _prometheus_multiproc_dir
+
+    if "PROMETHEUS_MULTIPROC_DIR" not in os.environ:
+        # Make TemporaryDirectory for prometheus multiprocessing
+        # Note: global TemporaryDirectory will be automatically
+        # cleaned up upon exit.
+        _prometheus_multiproc_dir = tempfile.TemporaryDirectory()
+        os.environ["PROMETHEUS_MULTIPROC_DIR"] = _prometheus_multiproc_dir.name
+        logger.debug("Created PROMETHEUS_MULTIPROC_DIR at %s",
+                     _prometheus_multiproc_dir.name)
+    else:
+        logger.warning("Found PROMETHEUS_MULTIPROC_DIR was set by user. "
+                       "This directory must be wiped between vLLM runs or "
+                       "you will find inaccurate metrics. Unset the variable "
+                       "and vLLM will properly handle cleanup.")
+
+
+def get_prometheus_registry():
+    """Get the appropriate prometheus registry based on multiprocessing 
+    configuration.
+    
+    Returns:
+        Registry: A prometheus registry
+    """
+    if os.getenv("PROMETHEUS_MULTIPROC_DIR") is not None:
+        logger.debug("Using multiprocess registry for prometheus metrics")
+        registry = CollectorRegistry()
+        multiprocess.MultiProcessCollector(registry)
+        return registry
+
+    return REGISTRY
+
+
+def unregister_vllm_metrics():
+    """Unregister any existing vLLM collectors from the prometheus registry.
+    
+    This is useful for testing and CI/CD where metrics may be registered
+    multiple times across test runs.
+    """
+    registry = get_prometheus_registry()
+    # Unregister any existing vLLM collectors
+    for collector in list(registry._collector_to_names):
+        if hasattr(collector, "_name") and "vllm" in collector._name:
+            registry.unregister(collector)
+
+
+def shutdown_prometheus():
+    """Shutdown prometheus metrics."""
+    try:
+        pid = os.getpid()
+        multiprocess.mark_process_dead(pid)
+        logger.debug("Marked Prometheus metrics for process %d as dead", pid)
+    except Exception as e:
+        logger.error("Error during metrics cleanup: %s", str(e))


Reviews:
user: login: njhill
body: Thanks for this @kouroshHakha.

I have a few general comments:

When there is a single api-server, we log metrics that come back from each engine separately with the engine index as a label. In the multi-api-server PR I changed the logic on the engine side to only send it's `SchedulerStats` back to *one* of the client api-servers. So hopefully for the metrics corresponding to these, not much more should be needed apart from making sure the gauges use "mostrecent" mode.

However, other metrics are computed during the loop in `async_llm.py` based on the requests that were processed in that iteration (in `IterationStats`). With multiple API servers, the processing of these requests for a given engine will in general be distributed amongst the api-servers since the outputs are sent back to each based on which originally sent the request.

These we may have to look at more closely and on a case-by-case basis since for example some are histograms where we assume the count corresponds to the number of iterations that have run on the corresponding engine, and we'll now be recording multiple of these (could be between 1 and num api servers).

From your PR description:
> 2. Set "sum" as default mode for gauges with special handling for lora_info metric

I don't see this anywhere in the changes?
state: COMMENTED
author_association: OWNER
submitted_at: 2025-05-14T01:57:34Z
user: login: kouroshHakha
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-14T17:24:54Z
user: login: kouroshHakha
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-14T19:14:48Z
user: login: kouroshHakha
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-14T19:16:44Z
user: login: kouroshHakha
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-14T19:17:57Z
user: login: njhill
state: COMMENTED
author_association: OWNER
submitted_at: 2025-05-14T23:26:56Z
user: login: njhill
body: Thanks @kouroshHakha

as well as the inline comments it would be good to make the change suggested by @markmc to move all the prometheus-touching logic into the prometheus package, calling utility methods from there as needed.
state: COMMENTED
author_association: OWNER
submitted_at: 2025-05-15T18:02:40Z
user: login: kouroshHakha
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-15T18:15:40Z
user: login: kouroshHakha
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-15T18:23:47Z
user: login: kouroshHakha
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-15T18:36:06Z
user: login: kouroshHakha
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-15T18:38:37Z
user: login: njhill
state: COMMENTED
author_association: OWNER
submitted_at: 2025-05-15T18:50:30Z
user: login: njhill
state: COMMENTED
author_association: OWNER
submitted_at: 2025-05-15T18:51:59Z
user: login: kouroshHakha
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-15T22:29:11Z
user: login: njhill
body: Thanks @kouroshHakha 
state: COMMENTED
author_association: OWNER
submitted_at: 2025-05-15T23:49:16Z
user: login: kouroshHakha
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-16T00:55:51Z
user: login: kouroshHakha
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-16T00:56:58Z
user: login: markmc
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-16T06:19:13Z
user: login: markmc
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-16T06:22:19Z
user: login: markmc
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-16T06:25:29Z
user: login: njhill
state: COMMENTED
author_association: OWNER
submitted_at: 2025-05-16T15:32:23Z
user: login: kouroshHakha
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-16T16:34:07Z
user: login: kouroshHakha
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-16T16:35:11Z
user: login: markmc
body: Super minor comments
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-16T17:09:42Z
user: login: njhill
body: Couple more small comments since @markmc also had some :)

Could you also add a similar warning comment to the `iteration_tokens_total` metric?
state: COMMENTED
author_association: OWNER
submitted_at: 2025-05-16T17:17:54Z
user: login: kouroshHakha
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-16T17:37:49Z
user: login: kouroshHakha
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-16T17:37:56Z
user: login: kouroshHakha
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-16T17:40:22Z
user: login: kouroshHakha
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-16T17:41:15Z
user: login: kouroshHakha
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-16T17:46:36Z
user: login: markmc
state: COMMENTED
author_association: NONE
submitted_at: 2025-05-16T17:50:00Z

Review comments:
diff_hunk: @@ -346,32 +347,63 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
             self.labelname_running_lora_adapters = "running_lora_adapters"
             self.max_lora = vllm_config.lora_config.max_loras
             self.gauge_lora_info = \
-                prometheus_client.Gauge(
+                self._create_gauge(
                     name="vllm:lora_requests_info",
                     documentation="Running stats on lora requests.",
                     labelnames=[
                         self.labelname_max_lora,
                         self.labelname_waiting_lora_adapters,
                         self.labelname_running_lora_adapters,
-                    ])
+                    ],
+                    multiprocess_mode="livemostrecent"
+                )
+
+    def _create_gauge(self,
+                      name: str,
+                      documentation: Optional[str],
+                      labelnames: list[str],
+                      multiprocess_mode: str = "all"):
+        return prometheus_client.Gauge(name=name,
+                                       documentation=documentation,
+                                       labelnames=labelnames,
+                                       multiprocess_mode=multiprocess_mode)
+
+    def _create_counter(self, name: str, documentation: Optional[str],
+                        labelnames: list[str]):
+        return prometheus_client.Counter(name=name,
+                                         documentation=documentation,
+                                         labelnames=labelnames)
path: vllm/v1/metrics/loggers.py
user: login: njhill
body: What's the purpose of adding this indirection if all of the args are just passed to the corresponding constructors?
created_at: 2025-05-14T01:34:47Z
author_association: OWNER
diff_hunk: @@ -198,6 +198,19 @@ def __del__(self):
     def shutdown(self):
         """Shutdown, cleaning up the background proc and IPC."""
 
+        # Upon shutdown of this process, we should mark the process as dead
+        # See https://prometheus.github.io/client_python/multiprocess/
+        try:
+            import os
+
+            from prometheus_client import multiprocess
+
+            multiprocess.mark_process_dead(os.getpid())
+            logger.debug("Marked Prometheus metrics for process %d as dead",
+                         os.getpid())
+        except Exception as e:
+            logger.error("Error during metrics cleanup: %s", str(e))
+
path: vllm/v1/engine/async_llm.py
user: login: njhill
body: Does it matter if we run this even when prometheus logging is disabled?
created_at: 2025-05-14T01:36:20Z
author_association: OWNER
diff_hunk: @@ -162,7 +166,20 @@ def run_multi_api_server(args: argparse.Namespace):
 
     assert not args.headless
     num_api_servers = args.api_server_count
-    # assert num_api_servers > 1
+
+    assert num_api_servers > 1
+    if "PROMETHEUS_MULTIPROC_DIR" not in os.environ:
+        # Make TemporaryDirectory for prometheus multiprocessing
+        # Note: global TemporaryDirectory will be automatically
+        #   cleaned up upon exit.
+        global prometheus_multiproc_dir
+        prometheus_multiproc_dir = tempfile.TemporaryDirectory()
+        os.environ["PROMETHEUS_MULTIPROC_DIR"] = prometheus_multiproc_dir.name
+    else:
+        logger.warning("Found PROMETHEUS_MULTIPROC_DIR was set by user. "
+                       "This directory must be wiped between vLLM runs or "
+                       "you will find inaccurate metrics. Unset the variable "
+                       "and vLLM will properly handle cleanup.")
path: vllm/entrypoints/cli/serve.py
user: login: njhill
body: We should probably only do this is prometheus logging is disabled .. at least we probably shouldn't if `log_stats` is `False`.
created_at: 2025-05-14T01:40:57Z
author_association: OWNER
diff_hunk: @@ -165,31 +165,32 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
         #
         # Scheduler state
         #
-        self.gauge_scheduler_running = prometheus_client.Gauge(
+        self.gauge_scheduler_running = self._create_gauge(
             name="vllm:num_requests_running",
             documentation="Number of requests in model execution batches.",
-            labelnames=labelnames).labels(*labelvalues)
+            labelnames=labelnames,
+            multiprocess_mode="all").labels(*labelvalues)
path: vllm/v1/metrics/loggers.py
user: login: njhill
body: ```suggestion
            labelnames=labelnames).labels(*labelvalues)
```
created_at: 2025-05-14T01:42:11Z
author_association: OWNER
diff_hunk: @@ -346,32 +347,63 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
             self.labelname_running_lora_adapters = "running_lora_adapters"
             self.max_lora = vllm_config.lora_config.max_loras
             self.gauge_lora_info = \
-                prometheus_client.Gauge(
+                self._create_gauge(
                     name="vllm:lora_requests_info",
                     documentation="Running stats on lora requests.",
                     labelnames=[
                         self.labelname_max_lora,
                         self.labelname_waiting_lora_adapters,
                         self.labelname_running_lora_adapters,
-                    ])
+                    ],
+                    multiprocess_mode="livemostrecent"
+                )
+
+    def _create_gauge(self,
+                      name: str,
+                      documentation: Optional[str],
+                      labelnames: list[str],
+                      multiprocess_mode: str = "all"):
+        return prometheus_client.Gauge(name=name,
+                                       documentation=documentation,
+                                       labelnames=labelnames,
+                                       multiprocess_mode=multiprocess_mode)
+
+    def _create_counter(self, name: str, documentation: Optional[str],
+                        labelnames: list[str]):
+        return prometheus_client.Counter(name=name,
+                                         documentation=documentation,
+                                         labelnames=labelnames)
path: vllm/v1/metrics/loggers.py
user: login: kouroshHakha
body: Mostly modularity. We can extend these, for example in this PR https://github.com/vllm-project/vllm/pull/17925 we want to wrap these primitives with their Ray equivalent.  
created_at: 2025-05-14T17:24:54Z
author_association: NONE
diff_hunk: @@ -198,6 +198,19 @@ def __del__(self):
     def shutdown(self):
         """Shutdown, cleaning up the background proc and IPC."""
 
+        # Upon shutdown of this process, we should mark the process as dead
+        # See https://prometheus.github.io/client_python/multiprocess/
+        try:
+            import os
+
+            from prometheus_client import multiprocess
+
+            multiprocess.mark_process_dead(os.getpid())
+            logger.debug("Marked Prometheus metrics for process %d as dead",
+                         os.getpid())
+        except Exception as e:
+            logger.error("Error during metrics cleanup: %s", str(e))
+
path: vllm/v1/engine/async_llm.py
user: login: kouroshHakha
body: so this part of the shutdown logic runs through and is a no-op when `log-stats=False`. So I if we want to be too pedantic about something like `prometheus_client` not being installed on the host we can only gate the logic on `log_stats`. But I think it's better to keep it like this. The only edge case I can think of is if `prometheus_client` package is not installed in which case the try-except block will just emit a `logger.error` that won't be raised unless the user wants. So it's fine by default.
created_at: 2025-05-14T19:14:48Z
author_association: NONE
diff_hunk: @@ -198,6 +198,19 @@ def __del__(self):
     def shutdown(self):
         """Shutdown, cleaning up the background proc and IPC."""
 
+        # Upon shutdown of this process, we should mark the process as dead
+        # See https://prometheus.github.io/client_python/multiprocess/
+        try:
+            import os
+
+            from prometheus_client import multiprocess
+
+            multiprocess.mark_process_dead(os.getpid())
+            logger.debug("Marked Prometheus metrics for process %d as dead",
+                         os.getpid())
+        except Exception as e:
+            logger.error("Error during metrics cleanup: %s", str(e))
+
path: vllm/v1/engine/async_llm.py
user: login: kouroshHakha
body: I added more comments to clarify the choice.
created_at: 2025-05-14T19:16:44Z
author_association: NONE
diff_hunk: @@ -162,7 +166,20 @@ def run_multi_api_server(args: argparse.Namespace):
 
     assert not args.headless
     num_api_servers = args.api_server_count
-    # assert num_api_servers > 1
+
+    assert num_api_servers > 1
+    if "PROMETHEUS_MULTIPROC_DIR" not in os.environ:
+        # Make TemporaryDirectory for prometheus multiprocessing
+        # Note: global TemporaryDirectory will be automatically
+        #   cleaned up upon exit.
+        global prometheus_multiproc_dir
+        prometheus_multiproc_dir = tempfile.TemporaryDirectory()
+        os.environ["PROMETHEUS_MULTIPROC_DIR"] = prometheus_multiproc_dir.name
+    else:
+        logger.warning("Found PROMETHEUS_MULTIPROC_DIR was set by user. "
+                       "This directory must be wiped between vLLM runs or "
+                       "you will find inaccurate metrics. Unset the variable "
+                       "and vLLM will properly handle cleanup.")
path: vllm/entrypoints/cli/serve.py
user: login: kouroshHakha
body: Setting the env var even if log_stats is false is fine? it doesn't hurt? Why do you think we should gate it on log_stats?  or even to be more precise when `Prometheus` logger is used? 
created_at: 2025-05-14T19:17:57Z
author_association: NONE
diff_hunk: @@ -346,32 +347,63 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
             self.labelname_running_lora_adapters = "running_lora_adapters"
             self.max_lora = vllm_config.lora_config.max_loras
             self.gauge_lora_info = \
-                prometheus_client.Gauge(
+                self._create_gauge(
                     name="vllm:lora_requests_info",
                     documentation="Running stats on lora requests.",
                     labelnames=[
                         self.labelname_max_lora,
                         self.labelname_waiting_lora_adapters,
                         self.labelname_running_lora_adapters,
-                    ])
+                    ],
+                    multiprocess_mode="livemostrecent"
+                )
+
+    def _create_gauge(self,
+                      name: str,
+                      documentation: Optional[str],
+                      labelnames: list[str],
+                      multiprocess_mode: str = "all"):
+        return prometheus_client.Gauge(name=name,
+                                       documentation=documentation,
+                                       labelnames=labelnames,
+                                       multiprocess_mode=multiprocess_mode)
+
+    def _create_counter(self, name: str, documentation: Optional[str],
+                        labelnames: list[str]):
+        return prometheus_client.Counter(name=name,
+                                         documentation=documentation,
+                                         labelnames=labelnames)
path: vllm/v1/metrics/loggers.py
user: login: njhill
body: It might make more sense to make this particular change in that other PR then since it's not directly related to this one.  cc @markmc
created_at: 2025-05-14T23:26:56Z
author_association: OWNER
diff_hunk: @@ -168,20 +171,23 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
         self.gauge_scheduler_running = prometheus_client.Gauge(
             name="vllm:num_requests_running",
             documentation="Number of requests in model execution batches.",
-            labelnames=labelnames).labels(*labelvalues)
+            labelnames=labelnames,
+            multiprocess_mode="mostrecent").labels(*labelvalues)
path: vllm/v1/metrics/loggers.py
user: login: njhill
body: presumably `multiprocess_mode` is ignored when not in multiprocess mode?
created_at: 2025-05-15T17:48:57Z
author_association: OWNER
diff_hunk: @@ -353,13 +359,19 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
                         self.labelname_max_lora,
                         self.labelname_waiting_lora_adapters,
                         self.labelname_running_lora_adapters,
-                    ])
+                    ],
+                    multiprocess_mode="livemostrecent"
+                )
 
     def log_metrics_info(self, type: str, config_obj: SupportsMetricsInfo):
+        # Lazy import prometheus_client
+        import prometheus_client
+
         metrics_info = config_obj.metrics_info()
         metrics_info["engine"] = self.engine_index
 
         name, documentation = None, None
+        multiprocess_mode = "mostrecent"
path: vllm/v1/metrics/loggers.py
user: login: njhill
body: What's the reason for this variable?
created_at: 2025-05-15T17:52:06Z
author_association: OWNER
diff_hunk: @@ -456,14 +472,15 @@ def log_engine_initialized(self):
         self.log_metrics_info("cache_config", self.vllm_config.cache_config)
 
 
-def build_buckets(mantissa_lst: list[int], max_value: int) -> list[int]:
+def build_buckets(mantissa_lst: list[int],
+                  max_value: int) -> list[Union[int, float]]:
path: vllm/v1/metrics/loggers.py
user: login: njhill
body: What's the reason for these changes? I'm not sure what's wrong with `list[int]` and regardless I think it's unrelated to the PR purpose?
created_at: 2025-05-15T17:53:01Z
author_association: OWNER
diff_hunk: @@ -353,13 +359,19 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
                         self.labelname_max_lora,
                         self.labelname_waiting_lora_adapters,
                         self.labelname_running_lora_adapters,
-                    ])
+                    ],
+                    multiprocess_mode="livemostrecent"
path: vllm/v1/metrics/loggers.py
user: login: njhill
body: Is this lora_info gauge another potentially problematic one? Since it's updated via IteratonStats and not SchedulerStats ... So I'm not sure "livemostrecent" is the right thing to use here.

I haven't looked closely at what this metric / how it is computed, maybe "sum" would fit better. I have a feeling though even that might not be correct, because it may be counting e.g. the number of unique lora adapters across the running requests and so it's not really possible to just combine the separate counts when the requests from a given engine are partitioned.

If there's not an easy answer we could include this in the list that we document as not being correct when mutli-api-servers are in play.
created_at: 2025-05-15T18:00:34Z
author_association: OWNER
diff_hunk: @@ -168,20 +171,23 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
         self.gauge_scheduler_running = prometheus_client.Gauge(
             name="vllm:num_requests_running",
             documentation="Number of requests in model execution batches.",
-            labelnames=labelnames).labels(*labelvalues)
+            labelnames=labelnames,
+            multiprocess_mode="mostrecent").labels(*labelvalues)
path: vllm/v1/metrics/loggers.py
user: login: kouroshHakha
body: yes correct. 
created_at: 2025-05-15T18:15:40Z
author_association: NONE
diff_hunk: @@ -456,14 +472,15 @@ def log_engine_initialized(self):
         self.log_metrics_info("cache_config", self.vllm_config.cache_config)
 
 
-def build_buckets(mantissa_lst: list[int], max_value: int) -> list[int]:
+def build_buckets(mantissa_lst: list[int],
+                  max_value: int) -> list[Union[int, float]]:
path: vllm/v1/metrics/loggers.py
user: login: kouroshHakha
body: reverting. linter artifact when I had the indirection
created_at: 2025-05-15T18:23:46Z
author_association: NONE
diff_hunk: @@ -353,13 +359,19 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
                         self.labelname_max_lora,
                         self.labelname_waiting_lora_adapters,
                         self.labelname_running_lora_adapters,
-                    ])
+                    ],
+                    multiprocess_mode="livemostrecent"
path: vllm/v1/metrics/loggers.py
user: login: kouroshHakha
body: In v0 world, this was livemostrecent. I assumed it was intentional, so I kept it. 
https://github.com/vllm-project/vllm/blob/main/vllm/engine/metrics.py#L80

It is potentially one of those problematic ones. But supporting lora metrics even falls lower in priority than the other one. So we may as well just record it in the docs. I'd keep as is since it's coming from historical context anyways. 
created_at: 2025-05-15T18:36:06Z
author_association: NONE
diff_hunk: @@ -353,13 +359,19 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
                         self.labelname_max_lora,
                         self.labelname_waiting_lora_adapters,
                         self.labelname_running_lora_adapters,
-                    ])
+                    ],
+                    multiprocess_mode="livemostrecent"
+                )
 
     def log_metrics_info(self, type: str, config_obj: SupportsMetricsInfo):
+        # Lazy import prometheus_client
+        import prometheus_client
+
         metrics_info = config_obj.metrics_info()
         metrics_info["engine"] = self.engine_index
 
         name, documentation = None, None
+        multiprocess_mode = "mostrecent"
path: vllm/v1/metrics/loggers.py
user: login: kouroshHakha
body: waiting for a green light from you to refactor this function entirely. It's a bit wierd that it's doing some conditioning but the condition is always true when I search across the project globally.
created_at: 2025-05-15T18:38:37Z
author_association: NONE
diff_hunk: @@ -353,13 +359,19 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
                         self.labelname_max_lora,
                         self.labelname_waiting_lora_adapters,
                         self.labelname_running_lora_adapters,
-                    ])
+                    ],
+                    multiprocess_mode="livemostrecent"
path: vllm/v1/metrics/loggers.py
user: login: njhill
body: In V0 I don't think were were ever logging the same metrics in multiple places, so the aggregation mode was probably irrelevant. I do think "sum" is probably slightly less bad here. But probably we should just disable this metric when there are multiple API servers since I think the values will just be wrong (and can document of course).
created_at: 2025-05-15T18:50:30Z
author_association: OWNER
diff_hunk: @@ -353,13 +359,19 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
                         self.labelname_max_lora,
                         self.labelname_waiting_lora_adapters,
                         self.labelname_running_lora_adapters,
-                    ])
+                    ],
+                    multiprocess_mode="livemostrecent"
+                )
 
     def log_metrics_info(self, type: str, config_obj: SupportsMetricsInfo):
+        # Lazy import prometheus_client
+        import prometheus_client
+
         metrics_info = config_obj.metrics_info()
         metrics_info["engine"] = self.engine_index
 
         name, documentation = None, None
+        multiprocess_mode = "mostrecent"
path: vllm/v1/metrics/loggers.py
user: login: njhill
body: I would say just leave it at least for this PR? Could always look into it some more and open a separate PR to refactor...
created_at: 2025-05-15T18:51:59Z
author_association: OWNER
diff_hunk: @@ -353,13 +359,19 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
                         self.labelname_max_lora,
                         self.labelname_waiting_lora_adapters,
                         self.labelname_running_lora_adapters,
-                    ])
+                    ],
+                    multiprocess_mode="livemostrecent"
path: vllm/v1/metrics/loggers.py
user: login: kouroshHakha
body: I made it sum. We can simply document for now.
created_at: 2025-05-15T22:29:11Z
author_association: NONE
diff_hunk: @@ -0,0 +1,105 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import re
+import tempfile
+from typing import Optional
+
+from prometheus_client import (REGISTRY, CollectorRegistry, make_asgi_app,
+                               multiprocess)
+from prometheus_fastapi_instrumentator import Instrumentator
+from starlette.routing import Mount
+
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+# Global temporary directory for prometheus multiprocessing
+_prometheus_multiproc_dir: Optional[tempfile.TemporaryDirectory] = None
+
+
+def setup_multiprocess_prometheus():
+    """Set up prometheus multiprocessing directory if not already configured.
+    
+    """
+    global _prometheus_multiproc_dir
+
+    if "PROMETHEUS_MULTIPROC_DIR" not in os.environ:
+        # Make TemporaryDirectory for prometheus multiprocessing
+        # Note: global TemporaryDirectory will be automatically
+        # cleaned up upon exit.
+        _prometheus_multiproc_dir = tempfile.TemporaryDirectory()
+        os.environ["PROMETHEUS_MULTIPROC_DIR"] = _prometheus_multiproc_dir.name
+        logger.debug("Created PROMETHEUS_MULTIPROC_DIR at %s",
+                     _prometheus_multiproc_dir.name)
+    else:
+        logger.warning("Found PROMETHEUS_MULTIPROC_DIR was set by user. "
+                       "This directory must be wiped between vLLM runs or "
+                       "you will find inaccurate metrics. Unset the variable "
+                       "and vLLM will properly handle cleanup.")
+
+
+def unregister_vllm_metrics():
+    """Unregister any existing vLLM collectors from the prometheus registry.
+    
+    This is useful for testing and CI/CD where metrics may be registered
+    multiple times across test runs.
+    """
+    # Unregister any existing vLLM collectors
+    for collector in list(REGISTRY._collector_to_names):
+        if hasattr(collector, "_name") and "vllm" in collector._name:
+            REGISTRY.unregister(collector)
+
+
+def get_prometheus_registry():
+    """Get the appropriate prometheus registry based on multiprocessing 
+    configuration.
+    
+    Returns:
+        Registry: A prometheus registry
+    """
+    if os.getenv("PROMETHEUS_MULTIPROC_DIR") is not None:
+        logger.debug("Using multiprocess registry for prometheus metrics")
+        registry = CollectorRegistry()
+        multiprocess.MultiProcessCollector(registry)
+        return registry
+    else:
+        return REGISTRY
+
+
+def mount_metrics(app):
path: vllm/v1/metrics/prometheus.py
user: login: njhill
body: ```suggestion
def mount_metrics(app: FastAPI):
```
created_at: 2025-05-15T23:46:19Z
author_association: OWNER
diff_hunk: @@ -0,0 +1,105 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import re
+import tempfile
+from typing import Optional
+
+from prometheus_client import (REGISTRY, CollectorRegistry, make_asgi_app,
+                               multiprocess)
+from prometheus_fastapi_instrumentator import Instrumentator
+from starlette.routing import Mount
+
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+# Global temporary directory for prometheus multiprocessing
+_prometheus_multiproc_dir: Optional[tempfile.TemporaryDirectory] = None
+
+
+def setup_multiprocess_prometheus():
+    """Set up prometheus multiprocessing directory if not already configured.
+    
+    """
+    global _prometheus_multiproc_dir
+
+    if "PROMETHEUS_MULTIPROC_DIR" not in os.environ:
+        # Make TemporaryDirectory for prometheus multiprocessing
+        # Note: global TemporaryDirectory will be automatically
+        # cleaned up upon exit.
+        _prometheus_multiproc_dir = tempfile.TemporaryDirectory()
+        os.environ["PROMETHEUS_MULTIPROC_DIR"] = _prometheus_multiproc_dir.name
+        logger.debug("Created PROMETHEUS_MULTIPROC_DIR at %s",
+                     _prometheus_multiproc_dir.name)
+    else:
+        logger.warning("Found PROMETHEUS_MULTIPROC_DIR was set by user. "
+                       "This directory must be wiped between vLLM runs or "
+                       "you will find inaccurate metrics. Unset the variable "
+                       "and vLLM will properly handle cleanup.")
+
+
+def unregister_vllm_metrics():
+    """Unregister any existing vLLM collectors from the prometheus registry.
+    
+    This is useful for testing and CI/CD where metrics may be registered
+    multiple times across test runs.
+    """
+    # Unregister any existing vLLM collectors
+    for collector in list(REGISTRY._collector_to_names):
+        if hasattr(collector, "_name") and "vllm" in collector._name:
+            REGISTRY.unregister(collector)
+
+
+def get_prometheus_registry():
+    """Get the appropriate prometheus registry based on multiprocessing 
+    configuration.
+    
+    Returns:
+        Registry: A prometheus registry
+    """
+    if os.getenv("PROMETHEUS_MULTIPROC_DIR") is not None:
+        logger.debug("Using multiprocess registry for prometheus metrics")
+        registry = CollectorRegistry()
+        multiprocess.MultiProcessCollector(registry)
+        return registry
+    else:
+        return REGISTRY
+
+
+def mount_metrics(app):
path: vllm/v1/metrics/prometheus.py
user: login: njhill
body: I think it might be better to leave this method in api_server.py, and just call `get_prometheus_registry()` from here
created_at: 2025-05-15T23:47:14Z
author_association: OWNER
diff_hunk: @@ -0,0 +1,105 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import re
+import tempfile
+from typing import Optional
+
+from prometheus_client import (REGISTRY, CollectorRegistry, make_asgi_app,
+                               multiprocess)
+from prometheus_fastapi_instrumentator import Instrumentator
+from starlette.routing import Mount
+
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+# Global temporary directory for prometheus multiprocessing
+_prometheus_multiproc_dir: Optional[tempfile.TemporaryDirectory] = None
+
+
+def setup_multiprocess_prometheus():
+    """Set up prometheus multiprocessing directory if not already configured.
+    
+    """
+    global _prometheus_multiproc_dir
+
+    if "PROMETHEUS_MULTIPROC_DIR" not in os.environ:
+        # Make TemporaryDirectory for prometheus multiprocessing
+        # Note: global TemporaryDirectory will be automatically
+        # cleaned up upon exit.
+        _prometheus_multiproc_dir = tempfile.TemporaryDirectory()
+        os.environ["PROMETHEUS_MULTIPROC_DIR"] = _prometheus_multiproc_dir.name
+        logger.debug("Created PROMETHEUS_MULTIPROC_DIR at %s",
+                     _prometheus_multiproc_dir.name)
+    else:
+        logger.warning("Found PROMETHEUS_MULTIPROC_DIR was set by user. "
+                       "This directory must be wiped between vLLM runs or "
+                       "you will find inaccurate metrics. Unset the variable "
+                       "and vLLM will properly handle cleanup.")
+
+
+def unregister_vllm_metrics():
+    """Unregister any existing vLLM collectors from the prometheus registry.
+    
+    This is useful for testing and CI/CD where metrics may be registered
+    multiple times across test runs.
+    """
+    # Unregister any existing vLLM collectors
+    for collector in list(REGISTRY._collector_to_names):
+        if hasattr(collector, "_name") and "vllm" in collector._name:
+            REGISTRY.unregister(collector)
+
+
+def get_prometheus_registry():
+    """Get the appropriate prometheus registry based on multiprocessing 
+    configuration.
+    
+    Returns:
+        Registry: A prometheus registry
+    """
+    if os.getenv("PROMETHEUS_MULTIPROC_DIR") is not None:
+        logger.debug("Using multiprocess registry for prometheus metrics")
+        registry = CollectorRegistry()
+        multiprocess.MultiProcessCollector(registry)
+        return registry
+    else:
+        return REGISTRY
+
+
+def mount_metrics(app):
+    """Mount prometheus metrics to a FastAPI app.
+    
+    Args:
+        app: FastAPI application
+    """
+
+    registry = get_prometheus_registry()
+
+    Instrumentator(
+        excluded_handlers=[
+            "/metrics",
+            "/health",
+            "/load",
+            "/ping",
+            "/version",
+            "/server_info",
+        ],
+        registry=registry,
+    ).add().instrument(app).expose(app)
+
+    # Add prometheus asgi middleware to route /metrics requests
+    metrics_route = Mount("/metrics", make_asgi_app(registry=registry))
+
+    # Workaround for 307 Redirect for /metrics
+    metrics_route.path_regex = re.compile("^/metrics(?P<path>.*)$")
+    app.routes.append(metrics_route)
+
+
+def mark_process_dead(pid):
path: vllm/v1/metrics/prometheus.py
user: login: njhill
body: ```suggestion
def mark_process_dead(pid: int):
```
created_at: 2025-05-15T23:47:22Z
author_association: OWNER
diff_hunk: @@ -0,0 +1,105 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import re
+import tempfile
+from typing import Optional
+
+from prometheus_client import (REGISTRY, CollectorRegistry, make_asgi_app,
+                               multiprocess)
+from prometheus_fastapi_instrumentator import Instrumentator
+from starlette.routing import Mount
+
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+# Global temporary directory for prometheus multiprocessing
+_prometheus_multiproc_dir: Optional[tempfile.TemporaryDirectory] = None
+
+
+def setup_multiprocess_prometheus():
+    """Set up prometheus multiprocessing directory if not already configured.
+    
+    """
+    global _prometheus_multiproc_dir
+
+    if "PROMETHEUS_MULTIPROC_DIR" not in os.environ:
+        # Make TemporaryDirectory for prometheus multiprocessing
+        # Note: global TemporaryDirectory will be automatically
+        # cleaned up upon exit.
+        _prometheus_multiproc_dir = tempfile.TemporaryDirectory()
+        os.environ["PROMETHEUS_MULTIPROC_DIR"] = _prometheus_multiproc_dir.name
+        logger.debug("Created PROMETHEUS_MULTIPROC_DIR at %s",
+                     _prometheus_multiproc_dir.name)
+    else:
+        logger.warning("Found PROMETHEUS_MULTIPROC_DIR was set by user. "
+                       "This directory must be wiped between vLLM runs or "
+                       "you will find inaccurate metrics. Unset the variable "
+                       "and vLLM will properly handle cleanup.")
+
+
+def unregister_vllm_metrics():
+    """Unregister any existing vLLM collectors from the prometheus registry.
+    
+    This is useful for testing and CI/CD where metrics may be registered
+    multiple times across test runs.
+    """
+    # Unregister any existing vLLM collectors
+    for collector in list(REGISTRY._collector_to_names):
+        if hasattr(collector, "_name") and "vllm" in collector._name:
+            REGISTRY.unregister(collector)
+
+
+def get_prometheus_registry():
+    """Get the appropriate prometheus registry based on multiprocessing 
+    configuration.
+    
+    Returns:
+        Registry: A prometheus registry
+    """
+    if os.getenv("PROMETHEUS_MULTIPROC_DIR") is not None:
+        logger.debug("Using multiprocess registry for prometheus metrics")
+        registry = CollectorRegistry()
+        multiprocess.MultiProcessCollector(registry)
+        return registry
+    else:
path: vllm/v1/metrics/prometheus.py
user: login: njhill
body: nit: redundant else
created_at: 2025-05-15T23:48:21Z
author_association: OWNER
diff_hunk: @@ -198,6 +200,13 @@ def __del__(self):
     def shutdown(self):
         """Shutdown, cleaning up the background proc and IPC."""
 
+        try:
+            mark_process_dead(os.getpid())
+            logger.debug("Marked Prometheus metrics for process %d as dead",
+                         os.getpid())
+        except Exception as e:
+            logger.error("Error during metrics cleanup: %s", str(e))
path: vllm/v1/engine/async_llm.py
user: login: njhill
body: put the try/except inside the method too?
created_at: 2025-05-15T23:48:50Z
author_association: OWNER
diff_hunk: @@ -198,6 +200,13 @@ def __del__(self):
     def shutdown(self):
         """Shutdown, cleaning up the background proc and IPC."""
 
+        try:
+            mark_process_dead(os.getpid())
+            logger.debug("Marked Prometheus metrics for process %d as dead",
+                         os.getpid())
+        except Exception as e:
+            logger.error("Error during metrics cleanup: %s", str(e))
path: vllm/v1/engine/async_llm.py
user: login: kouroshHakha
body: I think the caller should decide what they want to do. Sg?
created_at: 2025-05-16T00:55:50Z
author_association: NONE
diff_hunk: @@ -0,0 +1,105 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import re
+import tempfile
+from typing import Optional
+
+from prometheus_client import (REGISTRY, CollectorRegistry, make_asgi_app,
+                               multiprocess)
+from prometheus_fastapi_instrumentator import Instrumentator
+from starlette.routing import Mount
+
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+# Global temporary directory for prometheus multiprocessing
+_prometheus_multiproc_dir: Optional[tempfile.TemporaryDirectory] = None
+
+
+def setup_multiprocess_prometheus():
+    """Set up prometheus multiprocessing directory if not already configured.
+    
+    """
+    global _prometheus_multiproc_dir
+
+    if "PROMETHEUS_MULTIPROC_DIR" not in os.environ:
+        # Make TemporaryDirectory for prometheus multiprocessing
+        # Note: global TemporaryDirectory will be automatically
+        # cleaned up upon exit.
+        _prometheus_multiproc_dir = tempfile.TemporaryDirectory()
+        os.environ["PROMETHEUS_MULTIPROC_DIR"] = _prometheus_multiproc_dir.name
+        logger.debug("Created PROMETHEUS_MULTIPROC_DIR at %s",
+                     _prometheus_multiproc_dir.name)
+    else:
+        logger.warning("Found PROMETHEUS_MULTIPROC_DIR was set by user. "
+                       "This directory must be wiped between vLLM runs or "
+                       "you will find inaccurate metrics. Unset the variable "
+                       "and vLLM will properly handle cleanup.")
+
+
+def unregister_vllm_metrics():
+    """Unregister any existing vLLM collectors from the prometheus registry.
+    
+    This is useful for testing and CI/CD where metrics may be registered
+    multiple times across test runs.
+    """
+    # Unregister any existing vLLM collectors
+    for collector in list(REGISTRY._collector_to_names):
+        if hasattr(collector, "_name") and "vllm" in collector._name:
+            REGISTRY.unregister(collector)
+
+
+def get_prometheus_registry():
+    """Get the appropriate prometheus registry based on multiprocessing 
+    configuration.
+    
+    Returns:
+        Registry: A prometheus registry
+    """
+    if os.getenv("PROMETHEUS_MULTIPROC_DIR") is not None:
+        logger.debug("Using multiprocess registry for prometheus metrics")
+        registry = CollectorRegistry()
+        multiprocess.MultiProcessCollector(registry)
+        return registry
+    else:
+        return REGISTRY
+
+
+def mount_metrics(app):
path: vllm/v1/metrics/prometheus.py
user: login: kouroshHakha
body: So this method is very prometheus heavy (meaning it's not just the registry but also other things like mak_asgi_app, etc that are coming from prometheus. This really belongs to prometheus module of vllm. WDYT? 
created_at: 2025-05-16T00:56:57Z
author_association: NONE
diff_hunk: @@ -322,44 +321,6 @@ async def validate_json_request(raw_request: Request):
 router = APIRouter()
 
 
-def mount_metrics(app: FastAPI):
-    # Lazy import for prometheus multiprocessing.
-    # We need to set PROMETHEUS_MULTIPROC_DIR environment variable
-    # before prometheus_client is imported.
-    # See https://prometheus.github.io/client_python/multiprocess/
-    from prometheus_client import (REGISTRY, CollectorRegistry, make_asgi_app,
-                                   multiprocess)
-    from prometheus_fastapi_instrumentator import Instrumentator
-
-    registry = REGISTRY
-
-    prometheus_multiproc_dir_path = os.getenv("PROMETHEUS_MULTIPROC_DIR", None)
-    if prometheus_multiproc_dir_path is not None:
-        logger.debug("vLLM to use %s as PROMETHEUS_MULTIPROC_DIR",
-                     prometheus_multiproc_dir_path)
-        registry = CollectorRegistry()
-        multiprocess.MultiProcessCollector(registry)
path: vllm/entrypoints/openai/api_server.py
user: login: markmc
body: Everything above should be in the prometheus module, returning a `registry`
created_at: 2025-05-16T06:18:26Z
author_association: NONE
diff_hunk: @@ -322,44 +321,6 @@ async def validate_json_request(raw_request: Request):
 router = APIRouter()
 
 
-def mount_metrics(app: FastAPI):
-    # Lazy import for prometheus multiprocessing.
-    # We need to set PROMETHEUS_MULTIPROC_DIR environment variable
-    # before prometheus_client is imported.
-    # See https://prometheus.github.io/client_python/multiprocess/
-    from prometheus_client import (REGISTRY, CollectorRegistry, make_asgi_app,
-                                   multiprocess)
-    from prometheus_fastapi_instrumentator import Instrumentator
-
-    registry = REGISTRY
-
-    prometheus_multiproc_dir_path = os.getenv("PROMETHEUS_MULTIPROC_DIR", None)
-    if prometheus_multiproc_dir_path is not None:
-        logger.debug("vLLM to use %s as PROMETHEUS_MULTIPROC_DIR",
-                     prometheus_multiproc_dir_path)
-        registry = CollectorRegistry()
-        multiprocess.MultiProcessCollector(registry)
-
-    Instrumentator(
-        excluded_handlers=[
-            "/metrics",
-            "/health",
-            "/load",
-            "/ping",
-            "/version",
-            "/server_info",
-        ],
-        registry=registry,
-    ).add().instrument(app).expose(app)
-
-    # Add prometheus asgi middleware to route /metrics requests
-    metrics_route = Mount("/metrics", make_asgi_app(registry=registry))
-
-    # Workaround for 307 Redirect for /metrics
-    metrics_route.path_regex = re.compile("^/metrics(?P<path>.*)$")
-    app.routes.append(metrics_route)
path: vllm/entrypoints/openai/api_server.py
user: login: markmc
body: And all of the rest of it should remain in the API server module, using `registry` returned from the prometheus module
created_at: 2025-05-16T06:19:09Z
author_association: NONE
diff_hunk: @@ -198,6 +200,13 @@ def __del__(self):
     def shutdown(self):
         """Shutdown, cleaning up the background proc and IPC."""
 
+        try:
+            mark_process_dead(os.getpid())
+            logger.debug("Marked Prometheus metrics for process %d as dead",
+                         os.getpid())
+        except Exception as e:
+            logger.error("Error during metrics cleanup: %s", str(e))
path: vllm/v1/engine/async_llm.py
user: login: markmc
body: Agree with Nick. There's nothing the caller can do, and wrapping anything in such a broad try/except implies some knowledge of what the function is doing. I'd move the `os.getpid()` into the prometheus module too and call it `shutdown_prometheus()` or similar
created_at: 2025-05-16T06:22:19Z
author_association: NONE
diff_hunk: @@ -0,0 +1,105 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import re
+import tempfile
+from typing import Optional
+
+from prometheus_client import (REGISTRY, CollectorRegistry, make_asgi_app,
+                               multiprocess)
+from prometheus_fastapi_instrumentator import Instrumentator
+from starlette.routing import Mount
+
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+# Global temporary directory for prometheus multiprocessing
+_prometheus_multiproc_dir: Optional[tempfile.TemporaryDirectory] = None
+
+
+def setup_multiprocess_prometheus():
+    """Set up prometheus multiprocessing directory if not already configured.
+    
+    """
+    global _prometheus_multiproc_dir
+
+    if "PROMETHEUS_MULTIPROC_DIR" not in os.environ:
+        # Make TemporaryDirectory for prometheus multiprocessing
+        # Note: global TemporaryDirectory will be automatically
+        # cleaned up upon exit.
+        _prometheus_multiproc_dir = tempfile.TemporaryDirectory()
+        os.environ["PROMETHEUS_MULTIPROC_DIR"] = _prometheus_multiproc_dir.name
+        logger.debug("Created PROMETHEUS_MULTIPROC_DIR at %s",
+                     _prometheus_multiproc_dir.name)
+    else:
+        logger.warning("Found PROMETHEUS_MULTIPROC_DIR was set by user. "
+                       "This directory must be wiped between vLLM runs or "
+                       "you will find inaccurate metrics. Unset the variable "
+                       "and vLLM will properly handle cleanup.")
+
+
+def unregister_vllm_metrics():
+    """Unregister any existing vLLM collectors from the prometheus registry.
+    
+    This is useful for testing and CI/CD where metrics may be registered
+    multiple times across test runs.
+    """
+    # Unregister any existing vLLM collectors
+    for collector in list(REGISTRY._collector_to_names):
+        if hasattr(collector, "_name") and "vllm" in collector._name:
+            REGISTRY.unregister(collector)
path: vllm/v1/metrics/prometheus.py
user: login: markmc
body: One nice thing about having all this nonsense together in one module, you wonder ...

If we're using `REGISTRY` here, we're assuming multiprocess mode is not enabled? Maybe assert that the env var is not set?
created_at: 2025-05-16T06:25:29Z
author_association: NONE
diff_hunk: @@ -0,0 +1,105 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import re
+import tempfile
+from typing import Optional
+
+from prometheus_client import (REGISTRY, CollectorRegistry, make_asgi_app,
+                               multiprocess)
+from prometheus_fastapi_instrumentator import Instrumentator
+from starlette.routing import Mount
+
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+# Global temporary directory for prometheus multiprocessing
+_prometheus_multiproc_dir: Optional[tempfile.TemporaryDirectory] = None
+
+
+def setup_multiprocess_prometheus():
+    """Set up prometheus multiprocessing directory if not already configured.
+    
+    """
+    global _prometheus_multiproc_dir
+
+    if "PROMETHEUS_MULTIPROC_DIR" not in os.environ:
+        # Make TemporaryDirectory for prometheus multiprocessing
+        # Note: global TemporaryDirectory will be automatically
+        # cleaned up upon exit.
+        _prometheus_multiproc_dir = tempfile.TemporaryDirectory()
+        os.environ["PROMETHEUS_MULTIPROC_DIR"] = _prometheus_multiproc_dir.name
+        logger.debug("Created PROMETHEUS_MULTIPROC_DIR at %s",
+                     _prometheus_multiproc_dir.name)
+    else:
+        logger.warning("Found PROMETHEUS_MULTIPROC_DIR was set by user. "
+                       "This directory must be wiped between vLLM runs or "
+                       "you will find inaccurate metrics. Unset the variable "
+                       "and vLLM will properly handle cleanup.")
+
+
+def unregister_vllm_metrics():
+    """Unregister any existing vLLM collectors from the prometheus registry.
+    
+    This is useful for testing and CI/CD where metrics may be registered
+    multiple times across test runs.
+    """
+    # Unregister any existing vLLM collectors
+    for collector in list(REGISTRY._collector_to_names):
+        if hasattr(collector, "_name") and "vllm" in collector._name:
+            REGISTRY.unregister(collector)
path: vllm/v1/metrics/prometheus.py
user: login: njhill
body: good catch .. logic here should differ here in multproc case I think?
created_at: 2025-05-16T15:32:23Z
author_association: OWNER
diff_hunk: @@ -0,0 +1,105 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import re
+import tempfile
+from typing import Optional
+
+from prometheus_client import (REGISTRY, CollectorRegistry, make_asgi_app,
+                               multiprocess)
+from prometheus_fastapi_instrumentator import Instrumentator
+from starlette.routing import Mount
+
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+# Global temporary directory for prometheus multiprocessing
+_prometheus_multiproc_dir: Optional[tempfile.TemporaryDirectory] = None
+
+
+def setup_multiprocess_prometheus():
+    """Set up prometheus multiprocessing directory if not already configured.
+    
+    """
+    global _prometheus_multiproc_dir
+
+    if "PROMETHEUS_MULTIPROC_DIR" not in os.environ:
+        # Make TemporaryDirectory for prometheus multiprocessing
+        # Note: global TemporaryDirectory will be automatically
+        # cleaned up upon exit.
+        _prometheus_multiproc_dir = tempfile.TemporaryDirectory()
+        os.environ["PROMETHEUS_MULTIPROC_DIR"] = _prometheus_multiproc_dir.name
+        logger.debug("Created PROMETHEUS_MULTIPROC_DIR at %s",
+                     _prometheus_multiproc_dir.name)
+    else:
+        logger.warning("Found PROMETHEUS_MULTIPROC_DIR was set by user. "
+                       "This directory must be wiped between vLLM runs or "
+                       "you will find inaccurate metrics. Unset the variable "
+                       "and vLLM will properly handle cleanup.")
+
+
+def unregister_vllm_metrics():
+    """Unregister any existing vLLM collectors from the prometheus registry.
+    
+    This is useful for testing and CI/CD where metrics may be registered
+    multiple times across test runs.
+    """
+    # Unregister any existing vLLM collectors
+    for collector in list(REGISTRY._collector_to_names):
+        if hasattr(collector, "_name") and "vllm" in collector._name:
+            REGISTRY.unregister(collector)
path: vllm/v1/metrics/prometheus.py
user: login: kouroshHakha
body: good catch :) 
created_at: 2025-05-16T16:34:06Z
author_association: NONE
diff_hunk: @@ -322,44 +321,6 @@ async def validate_json_request(raw_request: Request):
 router = APIRouter()
 
 
-def mount_metrics(app: FastAPI):
-    # Lazy import for prometheus multiprocessing.
-    # We need to set PROMETHEUS_MULTIPROC_DIR environment variable
-    # before prometheus_client is imported.
-    # See https://prometheus.github.io/client_python/multiprocess/
-    from prometheus_client import (REGISTRY, CollectorRegistry, make_asgi_app,
-                                   multiprocess)
-    from prometheus_fastapi_instrumentator import Instrumentator
-
-    registry = REGISTRY
-
-    prometheus_multiproc_dir_path = os.getenv("PROMETHEUS_MULTIPROC_DIR", None)
-    if prometheus_multiproc_dir_path is not None:
-        logger.debug("vLLM to use %s as PROMETHEUS_MULTIPROC_DIR",
-                     prometheus_multiproc_dir_path)
-        registry = CollectorRegistry()
-        multiprocess.MultiProcessCollector(registry)
-
-    Instrumentator(
-        excluded_handlers=[
-            "/metrics",
-            "/health",
-            "/load",
-            "/ping",
-            "/version",
-            "/server_info",
-        ],
-        registry=registry,
-    ).add().instrument(app).expose(app)
-
-    # Add prometheus asgi middleware to route /metrics requests
-    metrics_route = Mount("/metrics", make_asgi_app(registry=registry))
-
-    # Workaround for 307 Redirect for /metrics
-    metrics_route.path_regex = re.compile("^/metrics(?P<path>.*)$")
-    app.routes.append(metrics_route)
path: vllm/entrypoints/openai/api_server.py
user: login: kouroshHakha
body: After doing this, it looks nicer, I have to admit :D 
created_at: 2025-05-16T16:35:10Z
author_association: NONE
diff_hunk: @@ -24,6 +24,8 @@
 from fastapi.exceptions import RequestValidationError
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse, Response, StreamingResponse
+from prometheus_client import make_asgi_app
+from prometheus_fastapi_instrumentator import Instrumentator
path: vllm/entrypoints/openai/api_server.py
user: login: markmc
body: remove?
created_at: 2025-05-16T17:03:46Z
author_association: NONE
diff_hunk: @@ -0,0 +1,78 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import tempfile
+from typing import Optional
+
+from prometheus_client import REGISTRY, CollectorRegistry, multiprocess
+
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+# Global temporary directory for prometheus multiprocessing
+_prometheus_multiproc_dir: Optional[tempfile.TemporaryDirectory] = None
+
+
+def setup_multiprocess_prometheus():
+    """Set up prometheus multiprocessing directory if not already configured.
+    
+    """
+    global _prometheus_multiproc_dir
+
+    if "PROMETHEUS_MULTIPROC_DIR" not in os.environ:
+        # Make TemporaryDirectory for prometheus multiprocessing
+        # Note: global TemporaryDirectory will be automatically
+        # cleaned up upon exit.
+        _prometheus_multiproc_dir = tempfile.TemporaryDirectory()
+        os.environ["PROMETHEUS_MULTIPROC_DIR"] = _prometheus_multiproc_dir.name
+        logger.debug("Created PROMETHEUS_MULTIPROC_DIR at %s",
+                     _prometheus_multiproc_dir.name)
+    else:
+        logger.warning("Found PROMETHEUS_MULTIPROC_DIR was set by user. "
+                       "This directory must be wiped between vLLM runs or "
+                       "you will find inaccurate metrics. Unset the variable "
+                       "and vLLM will properly handle cleanup.")
+
+
+def get_prometheus_registry():
+    """Get the appropriate prometheus registry based on multiprocessing 
+    configuration.
+    
+    Returns:
+        Registry: A prometheus registry
+    """
+    if os.getenv("PROMETHEUS_MULTIPROC_DIR") is not None:
+        logger.debug("Using multiprocess registry for prometheus metrics")
+        registry = CollectorRegistry()
+        multiprocess.MultiProcessCollector(registry)
+        return registry
+
+    return REGISTRY
+
+
+def unregister_vllm_metrics():
+    """Unregister any existing vLLM collectors from the prometheus registry.
+    
+    This is useful for testing and CI/CD where metrics may be registered
+    multiple times across test runs.
+    """
+    registry = get_prometheus_registry()
+    # Unregister any existing vLLM collectors
+    for collector in list(registry._collector_to_names):
+        if hasattr(collector, "_name") and "vllm" in collector._name:
+            registry.unregister(collector)
+
+
+def shutdown_prometheus():
+    """Mark a process as dead in prometheus multiprocessing.
+    
+    Args:
+        pid: Process ID to mark as dead
path: vllm/v1/metrics/prometheus.py
user: login: markmc
body: remove
created_at: 2025-05-16T17:05:13Z
author_association: NONE
diff_hunk: @@ -168,20 +168,23 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
         self.gauge_scheduler_running = prometheus_client.Gauge(
             name="vllm:num_requests_running",
             documentation="Number of requests in model execution batches.",
-            labelnames=labelnames).labels(*labelvalues)
path: vllm/v1/metrics/loggers.py
user: login: markmc
body: If you're re-spinning again, then a very minor stylistic request ...

In the original version this line is all "label stuff"

In the new version, it becomes "label stuff, new line, multiproc stuff, label stuff"

created_at: 2025-05-16T17:07:22Z
author_association: NONE
diff_hunk: @@ -168,20 +168,23 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
         self.gauge_scheduler_running = prometheus_client.Gauge(
             name="vllm:num_requests_running",
             documentation="Number of requests in model execution batches.",
-            labelnames=labelnames).labels(*labelvalues)
+            labelnames=labelnames,
+            multiprocess_mode="mostrecent").labels(*labelvalues)
path: vllm/v1/metrics/loggers.py
user: login: markmc
body: Suggest (for all these `multiprocess_mode` changes)

```
            multiprocess_mode="mostrecent",
            labelnames=labelnames).labels(*labelvalues)
```

created_at: 2025-05-16T17:09:24Z
author_association: NONE
diff_hunk: @@ -353,9 +356,12 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
                         self.labelname_max_lora,
                         self.labelname_waiting_lora_adapters,
                         self.labelname_running_lora_adapters,
-                    ])
+                    ],
+                    multiprocess_mode="sum"
path: vllm/v1/metrics/loggers.py
user: login: njhill
body: Could you add a comment here (maybe above, as part of the "LoRA metrics" comment), explaining that this metric will not be correct when using api-server scaleout, which uses prometheus mp mode.
created_at: 2025-05-16T17:13:57Z
author_association: OWNER
diff_hunk: @@ -24,6 +24,8 @@
 from fastapi.exceptions import RequestValidationError
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse, Response, StreamingResponse
+from prometheus_client import make_asgi_app
+from prometheus_fastapi_instrumentator import Instrumentator
path: vllm/entrypoints/openai/api_server.py
user: login: kouroshHakha
body: can't remove. `mount_metrics` is added back here, so it's needed. That's why I wanted to keep `mount_metrics` entirely in prometheus. 
created_at: 2025-05-16T17:37:49Z
author_association: NONE
diff_hunk: @@ -0,0 +1,78 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import tempfile
+from typing import Optional
+
+from prometheus_client import REGISTRY, CollectorRegistry, multiprocess
+
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+# Global temporary directory for prometheus multiprocessing
+_prometheus_multiproc_dir: Optional[tempfile.TemporaryDirectory] = None
+
+
+def setup_multiprocess_prometheus():
+    """Set up prometheus multiprocessing directory if not already configured.
+    
+    """
+    global _prometheus_multiproc_dir
+
+    if "PROMETHEUS_MULTIPROC_DIR" not in os.environ:
+        # Make TemporaryDirectory for prometheus multiprocessing
+        # Note: global TemporaryDirectory will be automatically
+        # cleaned up upon exit.
+        _prometheus_multiproc_dir = tempfile.TemporaryDirectory()
+        os.environ["PROMETHEUS_MULTIPROC_DIR"] = _prometheus_multiproc_dir.name
+        logger.debug("Created PROMETHEUS_MULTIPROC_DIR at %s",
+                     _prometheus_multiproc_dir.name)
+    else:
+        logger.warning("Found PROMETHEUS_MULTIPROC_DIR was set by user. "
+                       "This directory must be wiped between vLLM runs or "
+                       "you will find inaccurate metrics. Unset the variable "
+                       "and vLLM will properly handle cleanup.")
+
+
+def get_prometheus_registry():
+    """Get the appropriate prometheus registry based on multiprocessing 
+    configuration.
+    
+    Returns:
+        Registry: A prometheus registry
+    """
+    if os.getenv("PROMETHEUS_MULTIPROC_DIR") is not None:
+        logger.debug("Using multiprocess registry for prometheus metrics")
+        registry = CollectorRegistry()
+        multiprocess.MultiProcessCollector(registry)
+        return registry
+
+    return REGISTRY
+
+
+def unregister_vllm_metrics():
+    """Unregister any existing vLLM collectors from the prometheus registry.
+    
+    This is useful for testing and CI/CD where metrics may be registered
+    multiple times across test runs.
+    """
+    registry = get_prometheus_registry()
+    # Unregister any existing vLLM collectors
+    for collector in list(registry._collector_to_names):
+        if hasattr(collector, "_name") and "vllm" in collector._name:
+            registry.unregister(collector)
+
+
+def shutdown_prometheus():
+    """Mark a process as dead in prometheus multiprocessing.
+    
+    Args:
+        pid: Process ID to mark as dead
path: vllm/v1/metrics/prometheus.py
user: login: kouroshHakha
body: done already.
created_at: 2025-05-16T17:37:56Z
author_association: NONE
diff_hunk: @@ -168,20 +168,23 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
         self.gauge_scheduler_running = prometheus_client.Gauge(
             name="vllm:num_requests_running",
             documentation="Number of requests in model execution batches.",
-            labelnames=labelnames).labels(*labelvalues)
path: vllm/v1/metrics/loggers.py
user: login: kouroshHakha
body: sorry did not quite understand the desired style? 
created_at: 2025-05-16T17:40:22Z
author_association: NONE
diff_hunk: @@ -168,20 +168,23 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
         self.gauge_scheduler_running = prometheus_client.Gauge(
             name="vllm:num_requests_running",
             documentation="Number of requests in model execution batches.",
-            labelnames=labelnames).labels(*labelvalues)
path: vllm/v1/metrics/loggers.py
user: login: kouroshHakha
body: oh you mean the order? put multiprocess_mode before label stuff? 
created_at: 2025-05-16T17:41:15Z
author_association: NONE
diff_hunk: @@ -168,20 +168,23 @@ def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
         self.gauge_scheduler_running = prometheus_client.Gauge(
             name="vllm:num_requests_running",
             documentation="Number of requests in model execution batches.",
-            labelnames=labelnames).labels(*labelvalues)
path: vllm/v1/metrics/loggers.py
user: login: kouroshHakha
body: ok done. 
created_at: 2025-05-16T17:46:36Z
author_association: NONE
diff_hunk: @@ -24,6 +24,8 @@
 from fastapi.exceptions import RequestValidationError
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse, Response, StreamingResponse
+from prometheus_client import make_asgi_app
+from prometheus_fastapi_instrumentator import Instrumentator
path: vllm/entrypoints/openai/api_server.py
user: login: markmc
body: Doh, I misread, sorry. That's fine, this stuff is in the "api server" category not the "disgusting prometheus multi proc hackery" category :smiley: 
created_at: 2025-05-16T17:50:00Z
author_association: NONE

PR comments:
user: login: github-actions[bot]
created_at: 2025-05-13T18:56:16Z
body: 👋 Hi! Thank you for contributing to the vLLM project.

💬 Join our developer Slack at https://slack.vllm.ai to discuss your PR in #pr-reviews, coordinate on features in #feat- channels, or join special interest groups in #sig- channels.

Just a reminder: PRs would not trigger full CI run by default. Instead, it would only run `fastcheck` CI which starts running only a small and essential subset of CI tests to quickly catch errors. You can run other CI tests on top of those by going to your `fastcheck` build on Buildkite UI (linked in the PR checks section) and unblock them. If you do not have permission to unblock, ping `simon-mo` or `khluu` to add you in our Buildkite org.

Once the PR is approved and ready to go, your PR reviewer(s) can run CI to test the changes comprehensively before merging.

To run CI, PR reviewers can either: Add `ready` label to the PR or enable auto-merge.

🚀
author_association: NONE
user: login: njhill
created_at: 2025-05-14T02:09:25Z
body: @kouroshHakha in particular I also don't think we should be using the "all" mode which just labels by pid.
author_association: OWNER
user: login: njhill
created_at: 2025-05-14T02:18:25Z
body: We also need to make sure that the `PROMETHEUS_MULTIPROC_DIR` env var is always propagated properly. It was tricky to get this right before because I think it needs to be set prior to importing `prometheus`.

The [docs](https://prometheus.github.io/client_python/multiprocess/) recommend setting it externally but we obviously don't want to have to require that.

> This environment variable should be set from a start-up shell script, and not directly from Python (otherwise it may not propagate to child processes).
author_association: OWNER
user: login: kouroshHakha
created_at: 2025-05-14T17:22:56Z
body: 
> From your PR description: Set "sum" as default mode for gauges with special handling for lora_info metric
I don't see this anywhere in the changes?

I think I changed it to livemostrecent (forgot to update the description)

https://github.com/njhill/vllm/pull/6/files#diff-43531f6ec44e98f78e8f3fd53839a31c4fe4dd1c1b9015a45bf88b6f5dfeaabdR358
author_association: NONE
user: login: kouroshHakha
created_at: 2025-05-15T03:51:12Z
body: @njhill 


Comparing `-asc=1` and `-asc=16` and a counter metric like num_prompt_tokens over time on a fixed workload that has ~2M input tokens. 

![Screenshot 2025-05-14 at 8 47 17 PM](https://github.com/user-attachments/assets/9031e2d5-ad8e-4315-8f7b-f535ef5766e2)
author_association: NONE
user: login: kouroshHakha
created_at: 2025-05-15T04:44:06Z
body: > These we may have to look at more closely and on a case-by-case basis since for example some are histograms where we assume the count corresponds to the number of iterations that have run on the corresponding engine, and we'll now be recording multiple of these (could be between 1 and num api servers).

@njhill I have thought about this for almost the entire day looking at metrics and how they show up on grafana, etc. My conclusion is that we have only one metric that is impacted by hitting multiple iterationState records from different api_servers. That is histogram of `vllm:iteration_tokens_total`. Basically the main insight is that those metrics that are by definition invariant with notion of iteration step remain intact (e.g. time metrics, request level metrics, etc). But `iteration_tokens_total` does not fall into this category.

![Screenshot 2025-05-14 at 9 38 52 PM](https://github.com/user-attachments/assets/75baf054-fc42-4515-a69c-64f317fd27b7)

You can also observe the diff on `vllm:iteration_tokens_total` on the same workload. I can solve this by computing the n_tokens_per_iteration in each engine and attach it to the EngineCoreOutputs of only one of the clients. On the client side we can update the iterationstat.n_total_token_this_iteration only if it's returned by the EngineCoreOutputs. I am not sure if this is easy to do in the scheduler actually 

author_association: NONE
user: login: markmc
created_at: 2025-05-15T16:57:17Z
body: Nice work thinking this through @kouroshHakha 

Let's work through a simple example.

With a single API server and single engine

```
req1: prompt_len=100
req2: prompt_len=50
req3: prompt_len=80

iter1:
  req1: is_prefill=True, new_tokens=1
  req2: is_prefill=True, new_tokens=1
      observe(152)

iter2: 
  req1: is_prefill=False, new_tokens=1
  req2: is_prefill=False, new_tokens=1
  req3: is_prefill=True, new_tokens=1
    observe(83)

iter3:
  req1: is_prefill=False, new_tokens=1
  req2: is_prefill=False, new_tokens=1
  req3: is_prefill=False, new_tokens=1
    observe(3)

iter4:
  req1: is_prefill=False, new_tokens=1
  req2: is_prefill=False, new_tokens=1
  req3: is_prefill=False, new_tokens=1
    observe(3)

result:
  count = 4
  sum = 241
  buckets:
    le=8: 2
    le=128: 3
    le=256: 4
```

With 2 API servers and 2 engines

```
req1: prompt_len=100  ===> routed to API1, Engine1
req2: prompt_len=50  ===> routed to API2, Engine2
req3: prompt_len=80  ===> routed to API1, Engine1

API1, Engine1 iter1:
  req1: is_prefill=True, new_tokens=1
      observe(101)
      
API2, Engine2 iter1:
  req2: is_prefill=True, new_tokens=1
      observe(51)

API1, Engine1 iter2: 
  req1: is_prefill=False, new_tokens=1
  req3: is_prefill=True, new_tokens=1
    observe(82)

API2, Engine2 iter2: 
  req2: is_prefill=False, new_tokens=1
      observe(1)

API1, Engine1 iter3:
  req1: is_prefill=False, new_tokens=1
  req3: is_prefill=False, new_tokens=1
    observe(2)

API2, Engine2 iter3:
  req2: is_prefill=False, new_tokens=1
    observe(1)
    
 API1, Engine1 iter4:
  req1: is_prefill=False, new_tokens=1
  req3: is_prefill=False, new_tokens=1
    observe(2)

API2, Engine2 iter4:
  req2: is_prefill=False, new_tokens=1
    observe(1)

result:
  count = 8
  sum = 241
  buckets:
    le=3: 
    le=8: 5
    le=64: 6
    le=128: 8
```

So the view you get is that the same number of tokens is being generated with more, smaller iterations?

Is that going to be a problem? Surely people are watching trends, or comparing across like-for-like instances, etc. rather than relying on the actual values?

e.g. sure, you'd see a drop if you rolled out a multi-api-server change ... but that might even be reassuring, and make a ton of sense?

wdyt?
author_association: NONE
user: login: markmc
created_at: 2025-05-15T17:04:56Z
body: On the code ... I absolutely detest all this "make sure `PROMETHEUS_MULTIPROC_DIR` env var is set before importing prometheus_client" nonsense!

Firstly, I'm skeptical that the env var needs to be set before *importing* prometheus_client? Yes, we need to set it before creating the first metric, but why before importing? Especially since we're not using `prometheus_client.REGISTRY`?

Maybe I'm missing something there, but I'd like to be really sure we have to lazy import ... that's always going to be super brittle

Secondly, if we can put all the prometheus multiproc nonsense in one place - e.g. `vllm.v1.metrics.prometheus` - then it becomes much more maintainable. In V0, it was sprinkled over a bunch of places. I'd prefer to see no mention of `PROMETHEUS_MULTIPROC_DIR` anywhere except in `vllm.v1.metrics.prometheus`

Does that make sense?
author_association: NONE
user: login: njhill
created_at: 2025-05-15T17:39:47Z
body: Thanks @kouroshHakha @markmc for that careful analysis!

> I can solve this by computing the n_tokens_per_iteration in each engine and attach it to the EngineCoreOutputs of only one of the clients. On the client side we can update the iterationstat.n_total_token_this_iteration only if it's returned by the EngineCoreOutputs. I am not sure if this is easy to do in the scheduler actually

I'm not sure we want to complicate the scheduler or introduce more work there... we should aim to avoid that if possible.

> Is that going to be a problem? Surely people are watching trends, or comparing across like-for-like instances, etc. rather than relying on the actual values?

I'm not sure, it's possible the rate of the "count" of this histogram could be used to track iteration frequency and used in various other derived "per iteration" metrics. Unfortunately the count also won't be a constant multiple of the "original" count since the number of times it's recorded per engine per iteration would vary depending on how the requests are distributed between api servers / engines.

If this is the only problematic one though it seems reasonable to not block the PR and just document this along with the multi-api-server option.

> Firstly, I'm skeptical that the env var needs to be set before _importing_ prometheus_client? Yes, we need to set it before creating the first metric, but why before importing? Especially since we're not using `prometheus_client.REGISTRY`?

I'm not sure exactly, and agree that's horrible, it's just a (possibly incorrect) recollection from when we were wrangling with this some time back with V0. Hopefully it's not really the case

> Secondly, if we can put all the prometheus multiproc nonsense in one place - e.g. `vllm.v1.metrics.prometheus` - then it becomes much more maintainable. In V0, it was sprinkled over a bunch of places. I'd prefer to see no mention of `PROMETHEUS_MULTIPROC_DIR` anywhere except in `vllm.v1.metrics.prometheus`

Good point and I very much agree with this! If we do end up having an import ordering issue it should make that easier to manage too.
author_association: OWNER
user: login: kouroshHakha
created_at: 2025-05-15T18:00:39Z
body: Hey @markmc, 

> So the view you get is that the same number of tokens is being generated with more, smaller iterations?

yep exactly. I liked your toy example. It's really illuminating. Let's consider a scenario that engine is shared between two api servers. I think the conclusion would be the same tho:

```
Two API severs but one engine (all-to-all)

req1: prompt_len=100  ===> routed to API1, Engine1
req2: prompt_len=50  ===> routed to API2, Engine1
req3: prompt_len=80  ===> routed to API1, Engine1


Engine1 iter1:
    req1 (api1): is_prefill=True, new_tokens=1
    req2 (api2): is_prefill=True, new_tokens=1
    API1: observe(101)
    API2: observe(51)

Engine1 iter2:
    req1 (api1): is_prefill=False, new_tokens=1
    req2 (api2): is_prefill=False, new_tokens=1
    req3 (api1): is_prefill=True, new_tokens=1
    API1: observe(82)
    API2: observe(1)

Engine1 iter3:
    req1 (api1): isprefill=False, new_tokens=1
    req2 (api2): is_prefill=False, new_tokens=1
    req3 (api1): is_prefill=False, new_tokens=1
    API1: observe(2)
    API2: observe(1)

Engine1 iter4:
    req1 (api1): is_prefill=False, new_tokens=1
    req2 (api2): is_prefill=False, new_tokens=1
    req3 (api1): is_prefill=False, new_tokens=1
    API1: observe(2)
    API2: observe(1)



result:
  count = 8
  sum = 241
  buckets:
    le=1: 5
    le=8: 0
    le=16: 0
    le=32: 1
    le=64: 2
    le=128: 0


vs. with only one api server

result:
  count = 4
  sum = 241
  buckets:
    le=1: 2
    le=8: 0
    le=16: 0
    le=32: 0
    le=64: 1
    le=128: 1

```

Whether this is a problem or not, goes back to what we want from the system. I think keeping as is, is a reasonable design choice. i.e. As you scale the api server, the total sum remains the same, but it's broken down into smaller steps therefore changing the histogram. If we want to go this route, I think the metric name is a bit inconsistent with what it represents. The iteration_token_total suggest that as long as the number of engines remain the same for each engine I should see the similar distribution of tokens processed per step by the engine. Tho what @njhill suggested also could be a problem. It depends on how much the end user relies on these metrics :) 

But I do agree with you that this is certainly not a big problem. We should certainly not block this PR for this @njhill.

> Secondly, if we can put all the prometheus multiproc nonsense in one place - e.g. vllm.v1.metrics.prometheus - then it becomes much more maintainable. In V0, it was sprinkled over a bunch of places. I'd prefer to see no mention of PROMETHEUS_MULTIPROC_DIR anywhere except in vllm.v1.metrics.prometheus

@njhill @markmc I also think this was actually not necessary. Because once I added the check of `sys.module` during development I noticed I was failing that check but my metrics were still correct. So I don't think having to define the env var before an import is an absolute requirements. It was that based on past comments on V0 code path + the note written on prometheus docs https://prometheus.github.io/client_python/multiprocess/   (`This environment variable should be set from a start-up shell script, and not directly from Python (otherwise it may not propagate to child processes).`)

I will double check this and if the lazy import doesn't end up as a requirement I'd remove it like suggested. This will allow us to self contain Prometheus parts in one place for easier maintenance as well. 






author_association: NONE
user: login: kouroshHakha
created_at: 2025-05-15T22:34:41Z
body: ok @njhill @markmc I separated out prometheus non sense into its own python module under `v1.metrics.prometheus`. Also tested some of the metrics in single vs. multi process scenarios that I found brittle before and can confirm doing eager import of `promptheus_client` the way it is done in this PR is good. 

We can follow up with moving `PrometheusStatLogger` to `v1.metrics.prometheus` later, but I am avoiding this in this PR now to keep the diffs small. 
author_association: NONE
user: login: kouroshHakha
created_at: 2025-05-16T16:39:43Z
body: @njhill @markmc ready. incorporated your feedbacks. 
author_association: NONE
user: login: njhill
created_at: 2025-05-16T19:50:20Z
body: Thanks for all of your help and patience with this @kouroshHakha!
author_association: OWNER