Repository: AlibabaPAI/llumnix
Pull request #171: [Misc] Add ability to specify the local paths for models and datasets
Author: login: KuilongCui
Created: 2025-04-17T08:15:14Z | Merged: 2025-04-21T09:18:36Z



Diff:
diff --git a/examples/offline_inference.py b/examples/offline_inference.py
index ad4f9e4bf..22d6aacb4 100644
--- a/examples/offline_inference.py
+++ b/examples/offline_inference.py
@@ -10,7 +10,7 @@
 
 from llumnix import (Manager, launch_ray_cluster, connect_to_ray_cluster, init_manager,
                      ManagerArgs, InstanceArgs, Llumlet, ServerInfo, QueueType, BackendType)
-from llumnix.utils import random_uuid
+from llumnix.utils import random_uuid, try_convert_to_local_path
 from llumnix.queue.ray_queue_server import RayQueueServer
 
 from tests.conftest import cleanup_ray_env_func
@@ -38,7 +38,7 @@
 # Set manager args and engine args.
 manager_args = ManagerArgs()
 instance_args = InstanceArgs()
-engine_args = EngineArgs(model="facebook/opt-125m", download_dir="/mnt/model", worker_use_ray=True,
+engine_args = EngineArgs(model=try_convert_to_local_path("facebook/opt-125m"), download_dir="/mnt/model", worker_use_ray=True,
                          trust_remote_code=True, max_model_len=370, enforce_eager=True)
 node_id = ray.get_runtime_context().get_node_id()
 
diff --git a/llumnix/constants.py b/llumnix/constants.py
index 739f3c7de..7d1cd251e 100644
--- a/llumnix/constants.py
+++ b/llumnix/constants.py
@@ -14,6 +14,10 @@
 import torch
 
 
+# examples/offline_inference.py, tests/*
+MODEL_PATH: str = '/mnt/model'
+DATASET_PATH: str = '/mnt/dataset'
+
 # llumnix/manager.py
 CLEAR_REQUEST_INSTANCE_INTERVAL: float = 1000.0
 NO_INSTANCE_RETRY_GENERATE_INTERVAL: float = 1.0
diff --git a/llumnix/envs.py b/llumnix/envs.py
index 877bf68ff..ce18ce8a5 100644
--- a/llumnix/envs.py
+++ b/llumnix/envs.py
@@ -21,6 +21,8 @@
     LLUMNIX_LOGGING_PREFIX: str = "Llumnix"
     LLUMNIX_LOG_STREAM: int = 1
     LLUMNIX_LOG_NODE_PATH: str = ""
+    MODEL_PATH: str = ""
+    DATASET_PATH: str = ""
 
 
 environment_variables: Dict[str, Callable[[], Any]] = {
@@ -49,6 +51,11 @@
     # if set, llumnix will routing all node logs to this path
     "LLUMNIX_LOG_NODE_PATH":
     lambda: os.getenv("LLUMNIX_LOG_NODE_PATH", ""),
+
+    "MODEL_PATH":
+    lambda: os.getenv("MODEL_PATH", ""),
+    "DATASET_PATH":
+    lambda: os.getenv("DATASET_PATH", ""),
 }
 
 
diff --git a/llumnix/utils.py b/llumnix/utils.py
index 29b1ed341..5cac21a8a 100644
--- a/llumnix/utils.py
+++ b/llumnix/utils.py
@@ -24,6 +24,7 @@
 
 from llumnix.logging.logger import init_logger
 from llumnix import envs as llumnix_envs
+from llumnix.constants import MODEL_PATH, DATASET_PATH
 
 logger = init_logger(__name__)
 
@@ -182,3 +183,22 @@ def check_free_port(host='0.0.0.0', port=8081):
             return False
         else:
             raise
+
+def try_convert_to_local_path(data_path: str) -> str:
+    if os.path.isabs(data_path):
+        return data_path
+
+    assert "/" in data_path
+    base_data_name = os.path.basename(data_path)
+
+    base_model_path: str = llumnix_envs.MODEL_PATH if llumnix_envs.MODEL_PATH else MODEL_PATH
+    local_model_path: str = os.path.join(base_model_path, base_data_name)
+    if os.path.exists(local_model_path):
+        return local_model_path
+
+    base_dataset_path: str = llumnix_envs.DATASET_PATH if llumnix_envs.DATASET_PATH else DATASET_PATH
+    local_dataset_path: str = os.path.join(base_dataset_path, base_data_name)
+    if os.path.exists(local_dataset_path):
+        return local_dataset_path
+
+    return data_path
diff --git a/tests/e2e_test/test_bench.py b/tests/e2e_test/test_bench.py
index d9fc78209..0710fb6db 100644
--- a/tests/e2e_test/test_bench.py
+++ b/tests/e2e_test/test_bench.py
@@ -21,7 +21,7 @@
 import torch
 import numpy as np
 
-from llumnix.utils import get_ip_address
+from llumnix.utils import get_ip_address, try_convert_to_local_path
 
 # pylint: disable=unused-import
 from tests.conftest import ray_env
@@ -78,7 +78,7 @@ def get_markdown_data(key: str, head_name: str):
 
 @pytest.mark.asyncio
 @pytest.mark.skipif(torch.cuda.device_count() < 4, reason="at least 4 gpus required for simple benchmark")
-@pytest.mark.parametrize("model", ['/mnt/model/Qwen-7B'])
+@pytest.mark.parametrize("model", [try_convert_to_local_path('Qwen/Qwen-7B')])
 @pytest.mark.parametrize("launch_mode", ['global', 'local'])
 @pytest.mark.parametrize("enable_pd_disagg", [False, True])
 @pytest.mark.parametrize("enable_simulator", [False, True])
diff --git a/tests/e2e_test/test_correctness.py b/tests/e2e_test/test_correctness.py
index 0e35b09ef..a205064d1 100644
--- a/tests/e2e_test/test_correctness.py
+++ b/tests/e2e_test/test_correctness.py
@@ -21,7 +21,7 @@
 import aiohttp
 import torch
 
-from llumnix.utils import get_ip_address
+from llumnix.utils import get_ip_address, try_convert_to_local_path
 
 # pylint: disable=unused-import
 from tests.conftest import ray_env, cleanup_ray_env_func
@@ -122,7 +122,7 @@ async def run_bladellm(model, enable_pd_disagg):
 
 @pytest.mark.asyncio
 @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="at least 2 gpus required for correctness test")
-@pytest.mark.parametrize("model", ['/mnt/model/Qwen-7B'])
+@pytest.mark.parametrize("model", [try_convert_to_local_path('Qwen/Qwen-7B')])
 @pytest.mark.parametrize("launch_mode", ['global', 'local'])
 @pytest.mark.parametrize("enable_pd_disagg", [False, True])
 @pytest.mark.parametrize("tensor_parallel_size", [1, 2])
diff --git a/tests/e2e_test/test_migration.py b/tests/e2e_test/test_migration.py
index 7d2b6ec89..cc476fb6a 100644
--- a/tests/e2e_test/test_migration.py
+++ b/tests/e2e_test/test_migration.py
@@ -22,7 +22,7 @@
 import torch
 import ray
 
-from llumnix.utils import get_ip_address
+from llumnix.utils import get_ip_address, try_convert_to_local_path
 
 # pylint: disable=unused-import
 from tests.conftest import ray_env
@@ -99,7 +99,7 @@ def get_instance_num_blocks():
 
 @pytest.mark.asyncio
 @pytest.mark.skipif(torch.cuda.device_count() < 4, reason="at least 4 gpus required for migration bench")
-@pytest.mark.parametrize("model", ['/mnt/model/Qwen-7B'])
+@pytest.mark.parametrize("model", [try_convert_to_local_path('Qwen/Qwen-7B')])
 @pytest.mark.parametrize("migration_backend", ['rayrpc', 'gloo', 'nccl', 'grpc', 'kvtransfer'])
 @pytest.mark.parametrize("migration_request_status", ['running', 'waiting'])
 @pytest.mark.parametrize("tensor_parallel_size", [1, 2])
diff --git a/tests/e2e_test/utils.py b/tests/e2e_test/utils.py
index a990fd346..87e794c28 100644
--- a/tests/e2e_test/utils.py
+++ b/tests/e2e_test/utils.py
@@ -18,7 +18,7 @@
 import pytest
 import requests
 
-from llumnix.utils import get_ip_address
+from llumnix.utils import get_ip_address, try_convert_to_local_path
 
 
 def generate_vllm_launch_command(
@@ -30,7 +30,7 @@ def generate_vllm_launch_command(
     instances_num: int = 1,
     dispatch_policy: str = "load",
     migration_backend: str = "gloo",
-    model: str = "facebook/opt-125m",
+    model: str = try_convert_to_local_path("facebook/opt-125m"),
     max_model_len: int = 4096,
     log_instance_info: bool = False,
     log_request_timestamps: bool = False,
@@ -86,7 +86,7 @@ def generate_vllm_serve_command(
     port: int = 37000,
     dispatch_policy: str = "load",
     migration_backend: str = "gloo",
-    model: str = "facebook/opt-125m",
+    model: str = try_convert_to_local_path("facebook/opt-125m"),
     max_model_len: int = 4096,
     log_instance_info: bool = False,
     log_request_timestamps: bool = True,
@@ -129,7 +129,6 @@ def generate_vllm_serve_command(
         f"{'--enable-pd-disagg ' if enable_pd_disagg else ''}"
         f"{'--simulator-mode ' if enable_simulator else ''}"
         f"--config-file {config_path} "
-        f"--max-instances 4 "
         f"{'--profiling-result-file-path /mnt/model/simulator/Qwen-7B.pkl ' if enable_simulator else ''}"
         f"{'--disable-async-output-proc ' if enable_simulator else ''}"
         f"{'> instance_'+result_filename if len(result_filename)> 0 else ''} 2>&1 &"
@@ -141,7 +140,7 @@ def generate_vllm_serve_command(
 def generate_bladellm_launch_command(
     config_file: str = "configs/bladellm.yml",
     result_filename: str = "",
-    model = "facebook/opt-125m",
+    model: str = try_convert_to_local_path("facebook/opt-125m"),
     HEAD_NODE_IP: str = "127.0.0.1",
     ip: str = get_ip_address(),
     port: int = 37000,
@@ -189,7 +188,7 @@ def generate_bladellm_launch_command(
 def generate_bladellm_serve_command(
     config_file: str = "configs/bladellm.yml",
     result_filename: str = "",
-    model = "facebook/opt-125m",
+    model: str = try_convert_to_local_path("facebook/opt-125m"),
     ip: str = get_ip_address(),
     port: int = 37000,
     max_num_batched_tokens: int = 16000,
diff --git a/tests/unit_test/backends/vllm/test_llm_engine.py b/tests/unit_test/backends/vllm/test_llm_engine.py
index 6bf7798a1..598debb58 100644
--- a/tests/unit_test/backends/vllm/test_llm_engine.py
+++ b/tests/unit_test/backends/vllm/test_llm_engine.py
@@ -31,6 +31,7 @@
 from llumnix.queue.queue_type import QueueType
 from llumnix.server_info import ServerInfo
 from llumnix.ray_utils import initialize_placement_group, get_placement_group_name
+from llumnix.utils import try_convert_to_local_path
 
 # pylint: disable=unused-import
 from tests.conftest import ray_env
@@ -57,7 +58,7 @@ def update_instance_info(self, instance_info):
 
 @pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Need at least 1 GPU to run the test.")
 def test_llm_engine_from_engine_args(ray_env):
-    engine_args = EngineArgs(model="facebook/opt-125m", download_dir="/mnt/model", worker_use_ray=True, enforce_eager=True)
+    engine_args = EngineArgs(model=try_convert_to_local_path("facebook/opt-125m"), download_dir="/mnt/model", worker_use_ray=True, enforce_eager=True)
     placement_group = initialize_placement_group(get_placement_group_name("0"), num_cpus=3, num_gpus=1, detached=True)
     llm_engine = MockEngine.from_engine_args(engine_args=engine_args, request_output_queue_type=QueueType.RAYQUEUE,
                                              instance_id="0", migration_config=None, placement_group=placement_group)
@@ -65,7 +66,7 @@ def test_llm_engine_from_engine_args(ray_env):
 
 def test_llm_engine_from_engine_args_sim(ray_env):
     latency_data = LatencyMemData({},{},{})
-    engine_args = EngineArgs(model="facebook/opt-125m", download_dir="/mnt/model", worker_use_ray=True, enforce_eager=True)
+    engine_args = EngineArgs(model=try_convert_to_local_path("facebook/opt-125m"), download_dir="/mnt/model", worker_use_ray=True, enforce_eager=True)
     placement_group = initialize_placement_group(get_placement_group_name("0"), num_cpus=2, num_gpus=1, detached=True)
     llm_engine = MockEngine.from_engine_args(engine_args=engine_args, request_output_queue_type=QueueType.RAYQUEUE,
                                              instance_id="0", migration_config=None, latency_mem=latency_data,
@@ -74,7 +75,7 @@ def test_llm_engine_from_engine_args_sim(ray_env):
 
 @pytest.mark.asyncio
 async def test_llm_engine_add_requset(ray_env):
-    engine_args = EngineArgs(model="facebook/opt-125m", download_dir="/mnt/model", worker_use_ray=True, enforce_eager=True)
+    engine_args = EngineArgs(model=try_convert_to_local_path("facebook/opt-125m"), download_dir="/mnt/model", worker_use_ray=True, enforce_eager=True)
     latency_data = LatencyMemData({},{},{})
     placement_group = initialize_placement_group(get_placement_group_name("0"), num_cpus=1, num_gpus=0, detached=True)
     llm_engine = LLMEngineLlumnix.from_engine_args(engine_args=engine_args,
diff --git a/tests/unit_test/backends/vllm/test_migration.py b/tests/unit_test/backends/vllm/test_migration.py
index 2bd07621c..ec28ea3d6 100644
--- a/tests/unit_test/backends/vllm/test_migration.py
+++ b/tests/unit_test/backends/vllm/test_migration.py
@@ -30,7 +30,7 @@
 from llumnix.llumlet.request import RequestInferenceType, RequestStatus
 from llumnix.queue.queue_type import QueueType
 from llumnix.arg_utils import InstanceArgs
-from llumnix.utils import get_llumnix_env_vars
+from llumnix.utils import get_llumnix_env_vars, try_convert_to_local_path
 from llumnix.ray_utils import (initialize_placement_group, get_placement_group_name,
                                remove_placement_group, kill_instance)
 
@@ -196,7 +196,8 @@ async def test_migration_correctness(migration_backend, migration_request_status
 
     ray.init(namespace="llumnix", ignore_reinit_error=True, runtime_env={"env_vars": get_llumnix_env_vars()})
 
-    engine_args = EngineArgs(model="facebook/opt-125m", download_dir="/mnt/model", worker_use_ray=True, tensor_parallel_size=tensor_parallel_size,
+    engine_args = EngineArgs(model=try_convert_to_local_path("facebook/opt-125m"), download_dir="/mnt/model",
+                             worker_use_ray=True, tensor_parallel_size=tensor_parallel_size,
                              enforce_eager=True, disable_async_output_proc=disable_async_output_proc)
     id_rank_map = {"0": 0, "1": 1}
     if migration_request_status == 'running':
@@ -364,7 +365,7 @@ async def test_correctness(prompt, origin_output):
 @pytest.mark.parametrize("migration_backend", ['rayrpc', 'gloo', 'nccl'])
 @pytest.mark.parametrize("disable_async_output_proc", [False, True])
 async def test_pd_diaggregation_correctness(ray_env, migration_backend, disable_async_output_proc):
-    engine_args = EngineArgs(model="facebook/opt-125m", download_dir="/mnt/model", worker_use_ray=True,
+    engine_args = EngineArgs(model=try_convert_to_local_path("facebook/opt-125m"), download_dir="/mnt/model", worker_use_ray=True,
                              enforce_eager=True, disable_async_output_proc=disable_async_output_proc)
     id_rank_map = {"0":0, "1":1}
 
diff --git a/tests/unit_test/backends/vllm/test_migration_backend.py b/tests/unit_test/backends/vllm/test_migration_backend.py
index 82752603d..a75390071 100644
--- a/tests/unit_test/backends/vllm/test_migration_backend.py
+++ b/tests/unit_test/backends/vllm/test_migration_backend.py
@@ -22,7 +22,7 @@
 
 from llumnix.backends.vllm.worker import MigrationWorker
 from llumnix.arg_utils import InstanceArgs
-from llumnix.utils import random_uuid
+from llumnix.utils import random_uuid, try_convert_to_local_path
 from llumnix.ray_utils import initialize_placement_group, get_placement_group_name
 
 # pylint: disable=unused-import
@@ -60,7 +60,8 @@ def get_worker_stage_metadata(self, request_id):
 @pytest.mark.parametrize("backend", ['rayrpc', 'gloo', 'nccl'])
 @pytest.mark.parametrize("send_worker_metadata", [True, False])
 def test_migrate_cache(ray_env, backend, send_worker_metadata):
-    engine_config = EngineArgs(model='facebook/opt-125m', download_dir="/mnt/model", max_model_len=8, enforce_eager=True).create_engine_config()
+    engine_config = EngineArgs(model=try_convert_to_local_path("facebook/opt-125m"), download_dir="/mnt/model",
+                               max_model_len=8, enforce_eager=True).create_engine_config()
     migraiton_config = InstanceArgs(migration_buffer_blocks=3, migration_num_layers=5).create_migration_config()
     migraiton_config.migration_backend = backend
 
diff --git a/tests/unit_test/backends/vllm/test_simulator.py b/tests/unit_test/backends/vllm/test_simulator.py
index 950e6ffd0..00958ffbb 100644
--- a/tests/unit_test/backends/vllm/test_simulator.py
+++ b/tests/unit_test/backends/vllm/test_simulator.py
@@ -13,6 +13,7 @@
 from llumnix.internal_config import MigrationConfig
 from llumnix.queue.queue_type import QueueType
 from llumnix.ray_utils import initialize_placement_group, get_placement_group_name
+from llumnix.utils import try_convert_to_local_path
 
 # pylint: disable=unused-import
 from tests.conftest import ray_env
@@ -31,7 +32,7 @@ def _get_lantecy_mem(self, *args, **kwargs):
 
 @pytest.mark.asyncio
 async def test_executor():
-    engine_args = EngineArgs(model="facebook/opt-125m", download_dir="/mnt/model", worker_use_ray=True,
+    engine_args = EngineArgs(model=try_convert_to_local_path("facebook/opt-125m"), download_dir="/mnt/model", worker_use_ray=True,
                              enforce_eager=True, disable_async_output_proc=True)
     engine_config = engine_args.create_engine_config()
     latency_mem = LatencyMemData({},{},{})
@@ -75,7 +76,7 @@ async def test_executor():
 async def test_backend(ray_env):
     # TODO(ZeldaHuang): add tests for BackendSimVLLM methods
     # (currently BackendSimVLLM is just a wrapper of BackendVLLM)
-    engine_args = EngineArgs(model="facebook/opt-125m", download_dir="/mnt/model", worker_use_ray=True,
+    engine_args = EngineArgs(model=try_convert_to_local_path("facebook/opt-125m"), download_dir="/mnt/model", worker_use_ray=True,
                              enforce_eager=True, disable_async_output_proc=True)
     migration_config = MigrationConfig("SR", "gloo", 16, 1, 4, 5, 20)
 
diff --git a/tests/unit_test/backends/vllm/test_worker.py b/tests/unit_test/backends/vllm/test_worker.py
index c5b291cc7..e123b7678 100644
--- a/tests/unit_test/backends/vllm/test_worker.py
+++ b/tests/unit_test/backends/vllm/test_worker.py
@@ -24,7 +24,7 @@
 from vllm.executor.ray_gpu_executor import RayWorkerWrapper
 
 from llumnix.arg_utils import InstanceArgs
-from llumnix.utils import random_uuid
+from llumnix.utils import random_uuid, try_convert_to_local_path
 from llumnix.ray_utils import initialize_placement_group, get_placement_group_name
 from llumnix.backends.vllm.worker import MigrationWorker
 
@@ -81,7 +81,8 @@ def create_worker(rank: int, local_rank: int, engine_config: EngineConfig,
 
 @pytest.mark.parametrize("backend", ['rayrpc', 'gloo', 'nccl'])
 def test_reserve_memory_for_migration(ray_env, backend):
-    engine_config = EngineArgs(model='facebook/opt-125m', download_dir="/mnt/model", max_model_len=8, enforce_eager=True).create_engine_config()
+    engine_config = EngineArgs(model=try_convert_to_local_path("facebook/opt-125m"), download_dir="/mnt/model",
+                               max_model_len=8, enforce_eager=True).create_engine_config()
     migration_config = InstanceArgs(migration_buffer_blocks=1).create_migration_config()
     migration_config.migration_backend = backend
     worker = create_worker(rank=0, local_rank=0, engine_config=engine_config)
@@ -102,7 +103,8 @@ def test_reserve_memory_for_migration(ray_env, backend):
 @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="Need at least 2 GPU to run the test.")
 @pytest.mark.parametrize("backend", ['rayrpc', 'gloo', 'nccl'])
 def test_rebuild_migration_backend(ray_env, backend):
-    engine_config = EngineArgs(model='facebook/opt-125m', download_dir="/mnt/model", max_model_len=8, enforce_eager=True).create_engine_config()
+    engine_config = EngineArgs(model=try_convert_to_local_path("facebook/opt-125m"), download_dir="/mnt/model",
+                               max_model_len=8, enforce_eager=True).create_engine_config()
     migration_config = InstanceArgs(migration_buffer_blocks=1).create_migration_config()
     migration_config.migration_backend = backend
 
@@ -151,7 +153,8 @@ def test_rebuild_migration_backend(ray_env, backend):
     assert ray.get(worker0.execute_method.remote('warmup'))
 
 def test_max_concurrency(ray_env):
-    engine_config = EngineArgs(model='facebook/opt-125m', download_dir="/mnt/model", max_model_len=8, enforce_eager=True).create_engine_config()
+    engine_config = EngineArgs(model=try_convert_to_local_path("facebook/opt-125m"), download_dir="/mnt/model",
+                               max_model_len=8, enforce_eager=True).create_engine_config()
     worker_no_concurrency = create_worker(rank=0, local_rank=0, engine_config=engine_config,
                                           worker_module_name="tests.unit_test.backends.vllm.test_worker",
                                           worker_class_name="MockMigrationWorker",
diff --git a/tests/unit_test/global_scheduler/test_manager.py b/tests/unit_test/global_scheduler/test_manager.py
index 5316ec759..efc43b491 100644
--- a/tests/unit_test/global_scheduler/test_manager.py
+++ b/tests/unit_test/global_scheduler/test_manager.py
@@ -34,7 +34,7 @@
 from llumnix.queue.queue_type import QueueType
 from llumnix.backends.backend_interface import BackendType
 from llumnix.entrypoints.utils import LaunchMode
-from llumnix.utils import random_uuid
+from llumnix.utils import random_uuid, try_convert_to_local_path
 from llumnix.ray_utils import (get_placement_group_name, get_server_name, get_instance_name,
                                remove_placement_group, INSTANCE_NAME_PREFIX, kill_server,
                                kill_instance, get_manager_name, get_scaler_name,
@@ -137,7 +137,7 @@ def init_manager_with_launch_mode(launch_mode, enable_pd_disagg=False, pd_ratio=
                                enable_pdd_node_affinity_scheduling=enable_pdd_node_affinity_scheduling)
     instance_args = InstanceArgs(migration_backend="rayrpc")
     entrypoints_args = EntrypointsArgs(host="127.0.0.1", port=8000)
-    engine_args = EngineArgs(model="facebook/opt-125m", download_dir="/mnt/model", worker_use_ray=True, enforce_eager=True)
+    engine_args = EngineArgs(model=try_convert_to_local_path("facebook/opt-125m"), download_dir="/mnt/model", worker_use_ray=True, enforce_eager=True)
     launch_args = LaunchArgs(launch_mode=launch_mode, backend_type=BackendType.VLLM)
 
     # As mock_manager can not be initialized to ray actor, it is initialized as local variable.
@@ -204,7 +204,7 @@ def test_init_llumlet(ray_env, llumlet):
     ray.get(llumlet.is_ready.remote())
 
 def test_init_instances(ray_env, manager):
-    engine_args = EngineArgs(model="facebook/opt-125m", download_dir="/mnt/model", worker_use_ray=True, enforce_eager=True)
+    engine_args = EngineArgs(model=try_convert_to_local_path("facebook/opt-125m"), download_dir="/mnt/model", worker_use_ray=True, enforce_eager=True)
     node_id = ray.get_runtime_context().get_node_id()
     _, instances = ray.get(manager.init_instances.remote(QueueType("rayqueue"), BackendType.VLLM, InstanceArgs(), engine_args,
                                                          node_id))
@@ -216,7 +216,8 @@ def test_init_instances_sim(ray_env, manager):
     # pylint: disable=import-outside-toplevel
     # cannot catch by pytest.raises
     try:
-        engine_args = EngineArgs(model="facebook/opt-125m", download_dir="/mnt/model", worker_use_ray=True, enforce_eager=True)
+        engine_args = EngineArgs(model=try_convert_to_local_path("facebook/opt-125m"), download_dir="/mnt/model",
+                                 worker_use_ray=True, enforce_eager=True)
         node_id = ray.get_runtime_context().get_node_id()
         _, _ = ray.get(manager.init_instances.remote(QueueType("rayqueue"), BackendType.SIM_VLLM,
                                                      InstanceArgs(profiling_result_file_path="/"), engine_args, node_id))
@@ -547,7 +548,7 @@ async def test_pd_disagg_gloal_launch_deployment_and_auto_scale_up_loop(ray_env)
 @pytest.mark.skipif(torch.cuda.device_count() < 4, reason="at least 4 gpus required")
 async def test_pd_disagg_deployment_states(ray_env):
     manager_args = ManagerArgs(enable_migration=True, enable_pd_disagg=True, pd_ratio="1:2")
-    engine_args = EngineArgs(model="facebook/opt-125m", download_dir="/mnt/model", worker_use_ray=True, enforce_eager=True)
+    engine_args = EngineArgs(model=try_convert_to_local_path("facebook/opt-125m"), download_dir="/mnt/model", worker_use_ray=True, enforce_eager=True)
     manager = Manager.from_args(entrypoints_args=EntrypointsArgs(), manager_args=manager_args,
                                 instance_args=InstanceArgs(migration_backend="rayrpc"),
                                 engine_args=engine_args, launch_args=LaunchArgs(LaunchMode.LOCAL, BackendType.VLLM))
diff --git a/tests/unit_test/llumlet/test_engine_step_exception.py b/tests/unit_test/llumlet/test_engine_step_exception.py
index d3c5d840b..19964d64f 100644
--- a/tests/unit_test/llumlet/test_engine_step_exception.py
+++ b/tests/unit_test/llumlet/test_engine_step_exception.py
@@ -24,6 +24,7 @@
 from llumnix.llumlet.llumlet import Llumlet
 from llumnix.queue.queue_type import QueueType
 from llumnix.ray_utils import initialize_placement_group, get_placement_group_name
+from llumnix.utils import try_convert_to_local_path
 
 # pylint: disable=unused-import
 from tests.conftest import ray_env
@@ -54,7 +55,8 @@ async def raise_error_step():
 
 @pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Need at least 1 GPU to run the test.")
 def test_engine_step_exception(ray_env):
-    engine_args = EngineArgs(model="facebook/opt-125m", download_dir="/mnt/model", max_model_len=8, worker_use_ray=True, enforce_eager=True)
+    engine_args = EngineArgs(model=try_convert_to_local_path("facebook/opt-125m"), download_dir="/mnt/model",
+                             max_model_len=8, worker_use_ray=True, enforce_eager=True)
 
     # wait previous test to release the GPU memory
     time.sleep(5.0)


Reviews:
user: login: s5u13b
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-21T02:26:31Z
user: login: s5u13b
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-21T07:44:59Z
user: login: zhypku
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-21T08:22:04Z
user: login: s5u13b
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-21T08:30:50Z

Review comments:
diff_hunk: @@ -38,7 +38,7 @@
 # Set manager args and engine args.
 manager_args = ManagerArgs()
 instance_args = InstanceArgs()
-engine_args = EngineArgs(model="facebook/opt-125m", download_dir="/mnt/model", worker_use_ray=True,
+engine_args = EngineArgs(model=as_local("facebook/opt-125m"), download_dir="/mnt/model", worker_use_ray=True,
path: examples/offline_inference.py
user: login: s5u13b
body: as_local, this name is quire strange
created_at: 2025-04-21T02:20:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -182,3 +183,22 @@ def check_free_port(host='0.0.0.0', port=8081):
             return False
         else:
             raise
+
path: llumnix/utils.py
user: login: s5u13b
body: do we need dataset?
created_at: 2025-04-21T07:44:54Z
author_association: CONTRIBUTOR

PR comments:
user: login: github-actions[bot]
created_at: 2025-04-17T08:58:13Z
body: rayqueue
latency(ms) | p25      | p50       | p75       | p95       | p99       | mean     
----------- | -------- | --------- | --------- | --------- | --------- | ---------
decode      | 49.56    | 54.34     | 66.69     | 109.77    | 233.16    | 65.46    
prefill     | 71904.26 | 187216.60 | 346009.36 | 424282.91 | 444694.25 | 202275.02


-------------------------------------
zmq
latency(ms) | p25      | p50       | p75       | p95       | p99       | mean     
----------- | -------- | --------- | --------- | --------- | --------- | ---------
decode      | 49.70    | 54.88     | 70.33     | 127.49    | 335.48    | 73.62    
prefill     | 45098.16 | 159452.07 | 345828.82 | 416613.93 | 444921.60 | 187151.48


-------------------------------------

author_association: NONE
user: login: github-actions[bot]
created_at: 2025-04-17T09:16:25Z
body: migration_size     | 8.00 MB | 16.00 MB | 24.00 MB | 32.00 MB | 40.00 MB | 48.00 MB | 56.00 MB | 64.00 MB | 72.00 MB | 80.00 MB | 88.00 MB | 96.00 MB | 104.00 MB | 112.00 MB | 120.00 MB | 128.00 MB | 136.00 MB | 144.00 MB | 152.00 MB | 160.00 MB | 168.00 MB | 176.00 MB | 184.00 MB | 192.00 MB | 208.00 MB | 216.00 MB | 224.00 MB | 232.00 MB | 296.00 MB | 304.00 MB | 320.00 MB | 336.00 MB | 352.00 MB | 456.00 MB | 504.00 MB | 560.00 MB
------------------ | ------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | ---------
rayrpc_speed(GB/s) | 0.87    | 1.30     | 1.59     | 1.77     | 1.84     | 1.92     | 2.05     | 2.12     | 2.12     | 2.16     | 2.21     | 2.29     | 2.26      | 2.24      | 2.31      | 2.33      | 2.38      | 2.38      | 2.41      | 2.42      | 2.38      | 2.39      | 2.41      | 2.39      | 2.63      | 2.38      | 2.60      | 2.61      | 2.75      | 2.63      | 2.62      | 2.53      | 3.03      | 3.06      | 3.08      | 3.07     

migration_size   | 8.00 MB | 16.00 MB | 24.00 MB | 32.00 MB | 40.00 MB | 48.00 MB | 56.00 MB | 64.00 MB | 72.00 MB | 80.00 MB | 88.00 MB | 96.00 MB | 104.00 MB | 112.00 MB | 120.00 MB | 128.00 MB | 136.00 MB | 144.00 MB | 152.00 MB | 160.00 MB | 168.00 MB | 176.00 MB | 184.00 MB | 200.00 MB | 208.00 MB | 248.00 MB | 288.00 MB | 296.00 MB | 312.00 MB | 328.00 MB | 480.00 MB
---------------- | ------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | ---------
gloo_speed(GB/s) | 0.80    | 1.30     | 1.61     | 1.86     | 2.06     | 2.25     | 2.38     | 2.40     | 2.45     | 2.58     | 2.50     | 2.39     | 2.52      | 2.64      | 2.70      | 2.58      | 2.29      | 2.50      | 2.11      | 2.36      | 2.32      | 2.25      | 2.32      | 2.42      | 2.46      | 2.41      | 2.25      | 1.41      | 1.13      | 1.03      | 2.14     

migration_size   | 8.00 MB | 16.00 MB | 24.00 MB | 32.00 MB | 40.00 MB | 48.00 MB | 56.00 MB | 64.00 MB | 72.00 MB | 80.00 MB | 88.00 MB | 96.00 MB | 104.00 MB | 112.00 MB | 120.00 MB | 128.00 MB | 136.00 MB | 144.00 MB | 152.00 MB | 160.00 MB | 168.00 MB | 176.00 MB | 184.00 MB | 192.00 MB | 200.00 MB | 208.00 MB | 216.00 MB | 224.00 MB | 232.00 MB | 272.00 MB | 288.00 MB | 352.00 MB | 448.00 MB | 464.00 MB | 504.00 MB
---------------- | ------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | ---------
nccl_speed(GB/s) | 0.28    | 0.54     | 0.73     | 0.92     | 1.08     | 1.30     | 1.41     | 1.50     | 1.88     | 1.91     | 1.93     | 1.83     | 2.42      | 1.95      | 2.10      | 2.28      | 3.04      | 2.31      | 1.33      | 1.43      | 1.45      | 1.60      | 1.40      | 1.52      | 2.13      | 1.87      | 1.70      | 1.94      | 1.65      | 1.18      | 1.20      | 1.21      | 1.27      | 1.39      | 1.35     


author_association: NONE
user: login: github-actions[bot]
created_at: 2025-04-18T11:07:13Z
body: rayqueue
latency(ms) | p25      | p50       | p75       | p95       | p99       | mean     
----------- | -------- | --------- | --------- | --------- | --------- | ---------
decode      | 49.53    | 54.73     | 65.55     | 121.01    | 293.96    | 67.49    
prefill     | 70251.40 | 183485.19 | 331641.56 | 410862.29 | 436215.52 | 199999.61


-------------------------------------
zmq
latency(ms) | p25      | p50       | p75       | p95       | p99       | mean     
----------- | -------- | --------- | --------- | --------- | --------- | ---------
decode      | 49.46    | 53.83     | 64.50     | 127.54    | 370.17    | 147.30   
prefill     | 50595.93 | 182672.94 | 321572.84 | 413210.03 | 445664.65 | 191707.42


-------------------------------------

author_association: NONE
user: login: github-actions[bot]
created_at: 2025-04-18T12:29:46Z
body: migration_size     | 8.00 MB | 16.00 MB | 24.00 MB | 32.00 MB | 40.00 MB | 48.00 MB | 56.00 MB | 64.00 MB | 72.00 MB | 80.00 MB | 88.00 MB | 96.00 MB | 104.00 MB | 112.00 MB | 120.00 MB | 128.00 MB | 136.00 MB | 144.00 MB | 152.00 MB | 160.00 MB | 168.00 MB | 176.00 MB | 184.00 MB | 192.00 MB | 208.00 MB | 232.00 MB | 240.00 MB | 280.00 MB | 312.00 MB | 328.00 MB | 472.00 MB | 488.00 MB
------------------ | ------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | ---------
rayrpc_speed(GB/s) | 0.86    | 1.29     | 1.59     | 1.76     | 1.86     | 1.96     | 2.04     | 2.13     | 2.14     | 2.18     | 2.24     | 2.28     | 2.30      | 2.21      | 2.29      | 2.27      | 2.39      | 2.39      | 2.44      | 2.38      | 2.45      | 2.51      | 2.48      | 2.47      | 2.55      | 2.39      | 2.54      | 2.65      | 2.69      | 2.77      | 2.98      | 3.08     

migration_size   | 8.00 MB | 16.00 MB | 24.00 MB | 32.00 MB | 40.00 MB | 48.00 MB | 56.00 MB | 64.00 MB | 72.00 MB | 80.00 MB | 88.00 MB | 96.00 MB | 104.00 MB | 112.00 MB | 120.00 MB | 128.00 MB | 136.00 MB | 144.00 MB | 152.00 MB | 160.00 MB | 168.00 MB | 176.00 MB | 184.00 MB | 192.00 MB | 200.00 MB | 208.00 MB | 224.00 MB | 232.00 MB | 288.00 MB | 480.00 MB | 488.00 MB
---------------- | ------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | ---------
gloo_speed(GB/s) | 0.83    | 1.32     | 1.63     | 1.97     | 2.00     | 2.31     | 2.29     | 2.45     | 2.52     | 2.48     | 2.83     | 2.57     | 2.79      | 2.75      | 2.78      | 2.53      | 2.03      | 2.39      | 2.68      | 2.21      | 2.20      | 2.55      | 2.63      | 2.15      | 2.10      | 2.60      | 1.84      | 2.09      | 1.69      | 2.22      | 2.18     

migration_size   | 8.00 MB | 16.00 MB | 24.00 MB | 32.00 MB | 40.00 MB | 48.00 MB | 56.00 MB | 64.00 MB | 72.00 MB | 80.00 MB | 88.00 MB | 96.00 MB | 104.00 MB | 112.00 MB | 120.00 MB | 128.00 MB | 136.00 MB | 144.00 MB | 152.00 MB | 160.00 MB | 168.00 MB | 176.00 MB | 184.00 MB | 192.00 MB | 200.00 MB | 208.00 MB | 216.00 MB | 232.00 MB | 296.00 MB | 328.00 MB | 336.00 MB | 456.00 MB | 480.00 MB | 488.00 MB
---------------- | ------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | ---------
nccl_speed(GB/s) | 0.27    | 0.51     | 0.73     | 0.98     | 1.14     | 1.28     | 1.31     | 1.52     | 1.97     | 1.90     | 1.96     | 1.96     | 2.24      | 2.28      | 2.19      | 2.31      | 2.63      | 2.30      | 1.39      | 1.61      | 1.35      | 1.67      | 1.67      | 1.19      | 1.79      | 1.33      | 1.71      | 1.42      | 1.31      | 0.82      | 1.16      | 0.92      | 1.14      | 1.48     


author_association: NONE
user: login: github-actions[bot]
created_at: 2025-04-21T03:23:25Z
body: rayqueue
latency(ms) | p25      | p50       | p75       | p95       | p99       | mean     
----------- | -------- | --------- | --------- | --------- | --------- | ---------
decode      | 50.34    | 55.43     | 65.59     | 118.45    | 320.67    | 70.03    
prefill     | 52604.96 | 228901.47 | 322289.61 | 419826.40 | 436422.74 | 203350.08


-------------------------------------
zmq
latency(ms) | p25      | p50       | p75       | p95       | p99       | mean     
----------- | -------- | --------- | --------- | --------- | --------- | ---------
decode      | 50.29    | 55.79     | 71.73     | 155.63    | 470.94    | 79.73    
prefill     | 42741.56 | 192105.00 | 304017.43 | 415910.65 | 435774.51 | 182558.62


-------------------------------------

author_association: NONE
user: login: github-actions[bot]
created_at: 2025-04-21T04:45:37Z
body: migration_size     | 8.00 MB | 16.00 MB | 24.00 MB | 32.00 MB | 40.00 MB | 48.00 MB | 56.00 MB | 64.00 MB | 72.00 MB | 80.00 MB | 88.00 MB | 96.00 MB | 104.00 MB | 112.00 MB | 120.00 MB | 128.00 MB | 136.00 MB | 144.00 MB | 152.00 MB | 160.00 MB | 168.00 MB | 176.00 MB | 184.00 MB | 192.00 MB | 200.00 MB | 208.00 MB | 232.00 MB | 256.00 MB | 288.00 MB | 296.00 MB | 312.00 MB | 320.00 MB | 416.00 MB | 464.00 MB | 496.00 MB
------------------ | ------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | ---------
rayrpc_speed(GB/s) | 0.86    | 1.30     | 1.57     | 1.75     | 1.86     | 1.96     | 1.99     | 2.11     | 2.18     | 2.23     | 2.31     | 2.25     | 2.34      | 2.20      | 2.33      | 2.36      | 2.37      | 2.40      | 2.55      | 2.50      | 2.42      | 2.51      | 2.38      | 2.46      | 2.54      | 2.59      | 2.56      | 2.06      | 2.72      | 2.71      | 2.89      | 2.66      | 2.99      | 3.01      | 3.17     

migration_size   | 8.00 MB | 16.00 MB | 24.00 MB | 32.00 MB | 40.00 MB | 48.00 MB | 56.00 MB | 64.00 MB | 72.00 MB | 80.00 MB | 88.00 MB | 96.00 MB | 104.00 MB | 112.00 MB | 120.00 MB | 128.00 MB | 136.00 MB | 144.00 MB | 152.00 MB | 160.00 MB | 168.00 MB | 176.00 MB | 184.00 MB | 192.00 MB | 200.00 MB | 208.00 MB | 216.00 MB | 232.00 MB | 288.00 MB | 304.00 MB | 312.00 MB | 416.00 MB | 552.00 MB | 568.00 MB
---------------- | ------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | ---------
gloo_speed(GB/s) | 0.82    | 1.37     | 1.66     | 1.95     | 2.06     | 2.27     | 2.33     | 2.45     | 2.57     | 2.68     | 2.57     | 2.70     | 2.89      | 2.61      | 2.83      | 2.60      | 2.58      | 2.44      | 2.38      | 2.37      | 2.25      | 2.55      | 2.40      | 2.16      | 2.08      | 2.25      | 2.04      | 1.89      | 1.86      | 2.42      | 0.95      | 2.57      | 2.51      | 2.63     

migration_size   | 8.00 MB | 16.00 MB | 24.00 MB | 32.00 MB | 40.00 MB | 48.00 MB | 56.00 MB | 64.00 MB | 72.00 MB | 80.00 MB | 88.00 MB | 96.00 MB | 104.00 MB | 112.00 MB | 120.00 MB | 128.00 MB | 136.00 MB | 144.00 MB | 152.00 MB | 160.00 MB | 168.00 MB | 176.00 MB | 184.00 MB | 192.00 MB | 208.00 MB | 216.00 MB | 232.00 MB | 296.00 MB | 312.00 MB | 320.00 MB | 344.00 MB | 352.00 MB | 416.00 MB | 472.00 MB | 552.00 MB
---------------- | ------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | ---------
nccl_speed(GB/s) | 0.25    | 0.47     | 0.68     | 0.92     | 1.11     | 1.16     | 1.44     | 1.58     | 1.44     | 1.93     | 1.85     | 2.06     | 2.15      | 2.42      | 2.02      | 2.19      | 2.42      | 2.89      | 1.68      | 1.45      | 1.64      | 1.44      | 1.53      | 1.55      | 1.41      | 2.22      | 1.91      | 0.82      | 1.29      | 1.06      | 1.14      | 1.55      | 1.63      | 1.37      | 1.14     


author_association: NONE
user: login: github-actions[bot]
created_at: 2025-04-21T07:24:49Z
body: rayqueue
latency(ms) | p25      | p50       | p75       | p95       | p99       | mean     
----------- | -------- | --------- | --------- | --------- | --------- | ---------
decode      | 50.02    | 54.74     | 68.92     | 118.94    | 297.56    | 70.31    
prefill     | 78212.98 | 210178.26 | 312099.13 | 423356.73 | 442995.37 | 202448.94


-------------------------------------
zmq
latency(ms) | p25      | p50       | p75       | p95       | p99       | mean     
----------- | -------- | --------- | --------- | --------- | --------- | ---------
decode      | 49.63    | 54.78     | 67.67     | 123.50    | 398.98    | 77.85    
prefill     | 47986.88 | 177752.08 | 312809.93 | 413344.62 | 442571.30 | 185523.42


-------------------------------------

author_association: NONE
user: login: github-actions[bot]
created_at: 2025-04-21T07:59:33Z
body: rayqueue
latency(ms) | p25      | p50       | p75       | p95       | p99       | mean     
----------- | -------- | --------- | --------- | --------- | --------- | ---------
decode      | 49.60    | 54.11     | 65.43     | 127.13    | 335.34    | 68.65    
prefill     | 55517.99 | 176684.42 | 353838.47 | 428096.42 | 444952.70 | 198731.40


-------------------------------------
zmq
latency(ms) | p25      | p50       | p75       | p95       | p99       | mean     
----------- | -------- | --------- | --------- | --------- | --------- | ---------
decode      | 50.12    | 55.60     | 68.82     | 135.01    | 461.36    | 105.43   
prefill     | 42772.57 | 159216.44 | 342108.72 | 422470.61 | 444748.39 | 184356.29


-------------------------------------

author_association: NONE
user: login: github-actions[bot]
created_at: 2025-04-21T09:17:34Z
body: migration_size     | 8.00 MB | 16.00 MB | 24.00 MB | 32.00 MB | 40.00 MB | 48.00 MB | 56.00 MB | 64.00 MB | 72.00 MB | 80.00 MB | 88.00 MB | 96.00 MB | 104.00 MB | 112.00 MB | 120.00 MB | 128.00 MB | 136.00 MB | 144.00 MB | 152.00 MB | 160.00 MB | 168.00 MB | 176.00 MB | 184.00 MB | 192.00 MB | 200.00 MB | 208.00 MB | 216.00 MB | 224.00 MB | 232.00 MB | 288.00 MB | 312.00 MB | 336.00 MB | 376.00 MB | 464.00 MB | 496.00 MB
------------------ | ------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | ---------
rayrpc_speed(GB/s) | 0.84    | 1.29     | 1.54     | 1.71     | 1.89     | 1.97     | 2.06     | 2.13     | 2.17     | 2.27     | 2.36     | 2.29     | 2.34      | 2.26      | 2.29      | 2.31      | 2.44      | 2.43      | 2.32      | 2.40      | 2.53      | 2.41      | 2.48      | 2.52      | 2.47      | 2.70      | 2.51      | 2.63      | 2.69      | 2.55      | 2.78      | 2.88      | 2.92      | 3.23      | 3.13     

migration_size   | 8.00 MB | 16.00 MB | 24.00 MB | 32.00 MB | 40.00 MB | 48.00 MB | 56.00 MB | 64.00 MB | 72.00 MB | 80.00 MB | 88.00 MB | 96.00 MB | 104.00 MB | 112.00 MB | 120.00 MB | 128.00 MB | 136.00 MB | 144.00 MB | 152.00 MB | 160.00 MB | 168.00 MB | 176.00 MB | 184.00 MB | 200.00 MB | 208.00 MB | 216.00 MB | 224.00 MB | 232.00 MB | 240.00 MB | 280.00 MB | 296.00 MB | 352.00 MB | 480.00 MB | 560.00 MB
---------------- | ------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | ---------
gloo_speed(GB/s) | 0.82    | 1.31     | 1.66     | 1.97     | 2.05     | 2.18     | 2.31     | 2.45     | 2.46     | 2.58     | 2.61     | 2.47     | 2.66      | 2.75      | 2.66      | 2.46      | 2.40      | 2.37      | 2.43      | 2.18      | 2.00      | 2.36      | 2.53      | 2.40      | 1.77      | 2.53      | 2.24      | 2.41      | 2.14      | 2.19      | 2.11      | 0.93      | 2.55      | 1.72     

migration_size   | 8.00 MB | 16.00 MB | 24.00 MB | 32.00 MB | 40.00 MB | 48.00 MB | 56.00 MB | 64.00 MB | 72.00 MB | 80.00 MB | 88.00 MB | 96.00 MB | 104.00 MB | 112.00 MB | 120.00 MB | 128.00 MB | 136.00 MB | 144.00 MB | 152.00 MB | 160.00 MB | 168.00 MB | 176.00 MB | 184.00 MB | 192.00 MB | 200.00 MB | 208.00 MB | 216.00 MB | 224.00 MB | 272.00 MB | 280.00 MB | 328.00 MB | 336.00 MB | 352.00 MB | 384.00 MB | 456.00 MB | 496.00 MB
---------------- | ------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | --------- | ---------
nccl_speed(GB/s) | 0.28    | 0.50     | 0.76     | 0.92     | 1.20     | 1.34     | 1.42     | 1.52     | 1.62     | 1.80     | 1.91     | 1.99     | 2.25      | 2.15      | 2.10      | 2.35      | 2.55      | 2.63      | 1.47      | 1.50      | 1.53      | 1.43      | 1.32      | 1.98      | 1.57      | 1.77      | 1.40      | 1.71      | 1.24      | 1.23      | 0.90      | 1.16      | 1.50      | 1.67      | 1.37      | 1.54     


author_association: NONE