Repository: gpustack/gpustack

Issue:
Failed to get derived_max_model_len of ModelScope/ZhipuAI/glm-4-9b-chat-1m and caused OOM
Deploy ModelScope/ZhipuAI/glm-4-9b-chat-1m from modelscope, failed to get derived_max_model_len and the max_model_len 8192 will not be set, which caused OOM.

Hints from issue comments:
@Finenyaco: Verified on main 79b4e55

<img width="2967" alt="Image" src="https://github.com/user-attachments/assets/86bf7b25-2c6e-450d-a136-8d3192322db8" />

Solution patch:
diff --git a/gpustack/utils/hub.py b/gpustack/utils/hub.py
index df5f40f..87a91a9 100644
--- a/gpustack/utils/hub.py
+++ b/gpustack/utils/hub.py
@@ -5,6 +5,7 @@ import fnmatch
 from huggingface_hub import HfFileSystem
 from huggingface_hub.utils import validate_repo_id
 from modelscope.hub.api import HubApi
+from transformers import PretrainedConfig
 
 from gpustack.config.config import get_global_config
 from gpustack.schemas.models import Model, SourceEnum
@@ -126,7 +127,7 @@ def get_pretrained_config(model: Model, **kwargs):
 # Simplified from vllm.config._get_and_verify_max_len
 # Keep in our codebase to avoid dependency on vllm's internal
 # APIs which may change unexpectedly.
-# https://github.com/vllm-project/vllm/blob/v0.6.2/vllm/config.py#L1668
+# https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/config.py#L2453
 def get_max_model_len(pretrained_config) -> int:  # noqa: C901
     """Get the model's maximum length."""
     derived_max_model_len = float("inf")
@@ -141,6 +142,8 @@ def get_max_model_len(pretrained_config) -> int:  # noqa: C901
         "seq_length",
         # Command-R
         "model_max_length",
+        # Whisper
+        "max_target_positions",
         # Others
         "max_sequence_length",
         "max_seq_length",
@@ -188,3 +191,17 @@ def get_max_model_len(pretrained_config) -> int:  # noqa: C901
 
     logger.debug(f"Derived max model length: {derived_max_model_len}")
     return int(derived_max_model_len)
+
+
+# Similar to https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/transformers_utils/config.py#L700,
+# But we don't assert and fail if num_attention_heads is missing.
+def get_hf_text_config(config: PretrainedConfig):
+    """Get the "sub" config relevant to llm for multi modal models.
+    No op for pure text models.
+    """
+    if hasattr(config, "text_config") and hasattr(
+        config.text_config, "num_attention_heads"
+    ):
+        return config.text_config
+    else:
+        return config
diff --git a/gpustack/worker/backends/base.py b/gpustack/worker/backends/base.py
index 533da12..e16d02e 100644
--- a/gpustack/worker/backends/base.py
+++ b/gpustack/worker/backends/base.py
@@ -8,7 +8,7 @@ from typing import Dict, List, Optional
 from abc import ABC, abstractmethod
 
 from gpustack.client.generated_clientset import ClientSet
-from gpustack.config.config import Config
+from gpustack.config.config import Config, set_global_config
 from gpustack.logging import setup_logging
 from gpustack.schemas.models import (
     BackendEnum,
@@ -147,6 +147,7 @@ class InferenceServer(ABC):
         cfg: Config,
     ):
         setup_logging(debug=cfg.debug)
+        set_global_config(cfg)
 
         try:
             self._clientset = clientset
diff --git a/gpustack/worker/backends/vllm.py b/gpustack/worker/backends/vllm.py
index 04fcf00..b036ce5 100644
--- a/gpustack/worker/backends/vllm.py
+++ b/gpustack/worker/backends/vllm.py
@@ -6,7 +6,11 @@ import sysconfig
 from typing import Optional
 from gpustack.schemas.models import ModelInstanceStateEnum
 from gpustack.utils.command import find_parameter, get_versioned_command
-from gpustack.utils.hub import get_max_model_len, get_pretrained_config
+from gpustack.utils.hub import (
+    get_hf_text_config,
+    get_max_model_len,
+    get_pretrained_config,
+)
 from gpustack.worker.backends.base import InferenceServer
 
 logger = logging.getLogger(__name__)
@@ -98,7 +102,8 @@ class VLLMServer(InferenceServer):
         """
         try:
             pretrained_config = get_pretrained_config(self._model)
-            return get_max_model_len(pretrained_config)
+            pretrained_or_hf_text_config = get_hf_text_config(pretrained_config)
+            return get_max_model_len(pretrained_or_hf_text_config)
         except Exception as e:
             logger.error(f"Failed to derive max model length: {e}")
 
diff --git a/pyproject.toml b/pyproject.toml
index 50333fb..57b89aa 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -54,6 +54,7 @@ tenacity = "^9.0.0"
 aiofiles = "^23.2.1"
 aiohttp = "^3.11.2"
 bitsandbytes = {version = "^0.45.2", optional = true}
+python-json-logger = "^3.3.0"
 [tool.poetry.group.dev.dependencies]
 coverage = {extras = ["toml"], version = "^7.5.1"}
 flake8 = "^7.0.0"


Test patch:
diff --git a/tests/pretrained_config/test_pretrained_config.py b/tests/pretrained_config/test_pretrained_config.py
index 8aaa58c..e5a175b 100644
--- a/tests/pretrained_config/test_pretrained_config.py
+++ b/tests/pretrained_config/test_pretrained_config.py
@@ -1,4 +1,4 @@
-from gpustack.worker.backends.vllm import get_max_model_len
+from gpustack.worker.backends.vllm import get_hf_text_config, get_max_model_len
 from transformers import AutoConfig
 
 
@@ -8,11 +8,14 @@ def test_get_max_model_len():
         "Qwen/Qwen2-VL-7B-Instruct": 32768,
         "tiiuae/falcon-7b": 2048,
         "microsoft/Phi-3.5-mini-instruct": 131072,
-        "liuhaotian/llava-v1.6-34b": 4096,
+        "llava-hf/llava-v1.6-mistral-7b-hf": 32768,
+        "unsloth/Llama-3.2-11B-Vision-Instruct": 131072,
+        "THUDM/glm-4-9b-chat-1m": 1048576,
     }
 
     for model_name, expected_max_model_len in hf_model_lengths.items():
         config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
+        pretrained_or_hf_text_config = get_hf_text_config(config)
         assert (
-            get_max_model_len(config) == expected_max_model_len
+            get_max_model_len(pretrained_or_hf_text_config) == expected_max_model_len
         ), f"max_model_len mismatch for {model_name}"