Repository: xorbitsai/inference

Issue:
BUG: chatglm hang



Discussion:

Resolving patch:
diff --git a/examples/chat.py b/examples/chat.py
--- a/examples/chat.py
+++ b/examples/chat.py
@@ -30,13 +30,7 @@
     print(f"Quantization: {quantization}")
 
     client = Client(endpoint)
-    model_uid = client.launch_model(
-        model_name=model_name,
-        model_size_in_billions=model_size_in_billions,
-        model_format=model_format,
-        quantization=quantization,
-        n_ctx=2048
-    )
+    model_uid = client.launch_model(model_name, n_ctx=2048)
     model = client.get_model(model_uid)
 
     chat_history: List["ChatCompletionMessage"] = []
diff --git a/xinference/model/llm/chatglm.py b/xinference/model/llm/chatglm.py
--- a/xinference/model/llm/chatglm.py
+++ b/xinference/model/llm/chatglm.py
@@ -22,6 +22,8 @@
 from .core import Model
 
 if TYPE_CHECKING:
+    from chatglm_cpp import Pipeline
+
     from .. import ModelSpec
 
 logger = logging.getLogger(__name__)
@@ -47,16 +49,8 @@ def __init__(
         model_config: Optional[ChatglmCppModelConfig] = None,
     ):
         super().__init__(model_uid, model_spec)
-        self._llm = None
+        self._llm: Optional["Pipeline"] = None
         self._model_path = model_path
-        if model_spec.model_name == "chatglm":
-            self.max_context_length = 2048
-        elif model_spec.model_name == "chatglm2":
-            self.max_context_length = 8192
-        else:
-            raise ValueError(
-                f"Invalid model name '{model_spec.model_name}'. Expected chatglm or chatglm2."
-            )
 
         # just a placeholder for now as the chatglm_cpp repo doesn't support model config.
         self._model_config = model_config
@@ -68,9 +62,6 @@ def _sanitize_generate_config(
     ) -> ChatglmCppGenerateConfig:
         if chatglmcpp_generate_config is None:
             chatglmcpp_generate_config = ChatglmCppGenerateConfig()
-        chatglmcpp_generate_config.setdefault("max_tokens", 256)
-        chatglmcpp_generate_config.setdefault("temperature", 0.95)
-        chatglmcpp_generate_config.setdefault("top_p", 0.8)
         chatglmcpp_generate_config.setdefault("stream", False)
         return chatglmcpp_generate_config
 
@@ -174,20 +165,12 @@ def chat(
         if generate_config.get("stream", False):
             it = self._llm.stream_chat(
                 chat_history_list,
-                max_context_length=self.max_context_length,
-                max_length=generate_config["max_tokens"],
-                temperature=generate_config["temperature"],
-                top_p=generate_config["top_p"],
             )
             assert not isinstance(it, str)
             return self._convert_raw_text_chunks_to_chat(it, self.model_uid)
         else:
             c = self._llm.chat(
                 chat_history_list,
-                max_context_length=self.max_context_length,
-                max_length=generate_config["max_tokens"],
-                temperature=generate_config["temperature"],
-                top_p=generate_config["top_p"],
             )
             assert not isinstance(c, Iterator)
             return self._convert_raw_text_completion_to_chat(c, self.model_uid)


Test patch:
diff --git a/xinference/model/tests/test_chatglm.py b/xinference/model/tests/test_chatglm.py
--- a/xinference/model/tests/test_chatglm.py
+++ b/xinference/model/tests/test_chatglm.py
@@ -78,19 +78,9 @@ def test_model_init(model_spec):
     assert model._model_path == path
     assert model.model_spec == model_spec
 
-    if model_spec.model_name == "chatglm":
-        assert model.max_context_length == 2048
-    elif model_spec.model_name == "chatglm2":
-        assert model.max_context_length == 8192
-
     assert model._model_config is None
     model._model_config = model._sanitize_generate_config(None)
-    assert model._model_config == {
-        "max_tokens": 256,
-        "temperature": 0.95,
-        "top_p": 0.8,
-        "stream": False,
-    }
+    assert model._model_config["stream"] == False
 
 
 @pytest.mark.parametrize("model_spec", [mock_model_spec1, mock_model_spec2])