Repository: xorbitsai/inference

Issue:
FEAT: Add generate method for `ChatglmCppModel`
### Is your feature request related to a problem? Please describe
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]

### Describe the solution you'd like
A clear and concise description of what you want to happen.

### Describe alternatives you've considered
A clear and concise description of any alternative solutions or features you've considered.

### Additional context
Add any other context or screenshots about the feature request here.



Discussion:

Resolving patch:
diff --git a/xinference/client/restful/restful_client.py b/xinference/client/restful/restful_client.py
--- a/xinference/client/restful/restful_client.py
+++ b/xinference/client/restful/restful_client.py
@@ -389,6 +389,58 @@ def chat(
         return response_data
 
 
+class RESTfulChatglmCppGenerateModelHandle(RESTfulChatglmCppChatModelHandle):
+    def generate(
+        self,
+        prompt: str,
+        generate_config: Optional["ChatglmCppGenerateConfig"] = None,
+    ) -> Union["Completion", Iterator["CompletionChunk"]]:
+        """
+        Given a prompt, the ChatGLM model will generate a response via RESTful APIs.
+
+        Parameters
+        ----------
+        prompt: str
+            The user's input.
+        generate_config: Optional["ChatglmCppGenerateConfig"]
+            Additional configuration for ChatGLM chat generation.
+
+        Returns
+        -------
+        Union["Completion", Iterator["CompletionChunk"]]
+            Stream is a parameter in generate_config.
+            When stream is set to True, the function will return Iterator["CompletionChunk"].
+            When stream is set to False, the function will return "Completion".
+
+        Raises
+        ------
+        RuntimeError
+            Report the failure to generate the content from the server. Detailed information provided in error message.
+
+        """
+
+        url = f"{self._base_url}/v1/completions"
+
+        request_body: Dict[str, Any] = {"model": self._model_uid, "prompt": prompt}
+        if generate_config is not None:
+            for key, value in generate_config.items():
+                request_body[key] = value
+
+        stream = bool(generate_config and generate_config.get("stream"))
+
+        response = requests.post(url, json=request_body, stream=stream)
+        if response.status_code != 200:
+            raise RuntimeError(
+                f"Failed to generate completion, detail: {response.json()['detail']}"
+            )
+
+        if stream:
+            return streaming_response_iterator(response.iter_lines())
+
+        response_data = response.json()
+        return response_data
+
+
 class Client:
     def __init__(self, base_url):
         self.base_url = base_url
@@ -612,7 +664,7 @@ def get_model(self, model_uid: str) -> RESTfulModelHandle:
 
         if desc["model_type"] == "LLM":
             if desc["model_format"] == "ggmlv3" and "chatglm" in desc["model_name"]:
-                return RESTfulChatglmCppChatModelHandle(model_uid, self.base_url)
+                return RESTfulChatglmCppGenerateModelHandle(model_uid, self.base_url)
             elif "chat" in desc["model_ability"]:
                 return RESTfulChatModelHandle(model_uid, self.base_url)
             elif "generate" in desc["model_ability"]:
diff --git a/xinference/model/llm/ggml/chatglm.py b/xinference/model/llm/ggml/chatglm.py
--- a/xinference/model/llm/ggml/chatglm.py
+++ b/xinference/model/llm/ggml/chatglm.py
@@ -25,6 +25,8 @@
     ChatCompletionMessage,
     ChatglmCppGenerateConfig,
     ChatglmCppModelConfig,
+    Completion,
+    CompletionChunk,
 )
 from .. import LLMFamilyV1, LLMSpecV1
 from ..core import LLM
@@ -187,6 +189,7 @@ def chat(
             "top_k": generate_config.get("top_k"),
             "top_p": generate_config.get("top_p"),
             "temperature": generate_config.get("temperature"),
+            "stream": generate_config.get("stream", False),
         }
 
         # Remove None values to exclude missing keys from params
@@ -195,7 +198,7 @@ def chat(
         assert self._llm is not None
 
         if generate_config["stream"]:
-            it = self._llm.stream_chat(
+            it = self._llm.chat(
                 chat_history_list,
                 **params,
             )
@@ -208,3 +211,73 @@ def chat(
             )
             assert not isinstance(c, Iterator)
             return self._convert_raw_text_completion_to_chat(c, self.model_uid)
+
+    @staticmethod
+    def _convert_str_to_completion(data: str, model_name: str) -> Completion:
+        return {
+            "id": "generate" + f"-{str(uuid.uuid4())}",
+            "model": model_name,
+            "object": "text_completion",
+            "created": int(time.time()),
+            "choices": [
+                {"index": 0, "text": data, "finish_reason": None, "logprobs": None}
+            ],
+            "usage": {
+                "prompt_tokens": -1,
+                "completion_tokens": -1,
+                "total_tokens": -1,
+            },
+        }
+
+    @staticmethod
+    def _convert_str_to_completion_chunk(
+        tokens: Iterator[str], model_name: str
+    ) -> Iterator[CompletionChunk]:
+        for token in tokens:
+            yield {
+                "id": "generate" + f"-{str(uuid.uuid4())}",
+                "model": model_name,
+                "object": "text_completion",
+                "created": int(time.time()),
+                "choices": [
+                    {"index": 0, "text": token, "finish_reason": None, "logprobs": None}
+                ],
+            }
+
+    def generate(
+        self,
+        prompt: str,
+        generate_config: Optional[ChatglmCppGenerateConfig] = None,
+    ) -> Union[Completion, Iterator[CompletionChunk]]:
+        logger.debug(f"Prompt for generate:\n{prompt}")
+
+        generate_config = self._sanitize_generate_config(generate_config)
+
+        params = {
+            "max_length": generate_config.get("max_tokens"),
+            "max_context_length": generate_config.get("max_tokens"),
+            "top_k": generate_config.get("top_k"),
+            "top_p": generate_config.get("top_p"),
+            "temperature": generate_config.get("temperature"),
+            "stream": generate_config.get("stream", False),
+        }
+
+        # Remove None values to exclude missing keys from params
+        params = {k: v for k, v in params.items() if v is not None}
+
+        assert self._llm is not None
+
+        if generate_config["stream"]:
+            it = self._llm.generate(
+                prompt,
+                **params,
+            )
+            assert not isinstance(it, str)
+            return self._convert_str_to_completion_chunk(it, self.model_uid)
+        else:
+            c = self._llm.generate(
+                prompt,
+                **params,
+            )
+            assert not isinstance(c, Iterator)
+            return self._convert_str_to_completion(c, self.model_uid)


Test patch:
diff --git a/xinference/model/llm/ggml/tests/test_chatglm.py b/xinference/model/llm/ggml/tests/test_chatglm.py
--- a/xinference/model/llm/ggml/tests/test_chatglm.py
+++ b/xinference/model/llm/ggml/tests/test_chatglm.py
@@ -14,7 +14,7 @@
 
 import random
 import string
-from typing import Iterator
+from typing import Iterator, Union
 
 import pytest
 
@@ -26,13 +26,21 @@ class MockPipeline:
     def __init__(self) -> None:
         pass
 
-    def stream_chat(self, *args, **kwargs) -> Iterator[str]:
-        res = [f"chatglm_test_stream_{i}" for i in range(5)]
-        return iter(res)
+    def chat(self, *args, **kwargs) -> Union[str, Iterator[str]]:
+        stream = kwargs.get("stream", False)
+        return (
+            iter([f"chatglm_test_stream_{i}" for i in range(5)])
+            if stream
+            else "chatglm_test_chat"
+        )
 
-    def chat(self, *args, **kwargs) -> str:
-        res = "chatglm_test_chat"
-        return res
+    def generate(self, *args, **kwargs) -> Union[str, Iterator[str]]:
+        stream = kwargs.get("stream", False)
+        return (
+            "chatglm_test_generate"
+            if not stream
+            else iter([f"chatglm_test_stream_generate_{i}" for i in range(5)])
+        )
 
 
 class MockChatglmCppChatModel(ChatglmCppChatModel):
@@ -182,3 +190,35 @@ def test_model_chat(model_spec, model_family):
         "role": "assistant",
         "content": "chatglm_test_chat",
     }
+
+
+@pytest.mark.parametrize(
+    "model_spec, model_family", [(mock_model_spec, mock_model_family)]
+)
+def test_model_generate(model_spec, model_family):
+    quantization = "q2_k"
+    uid = "".join(random.choice(string.digits) for i in range(100))
+    path = "".join(
+        random.choice(string.ascii_letters + string.punctuation) for i in range(100)
+    )
+    model = MockChatglmCppChatModel(
+        model_uid=uid,
+        model_family=model_family,
+        model_spec=model_spec,
+        quantization=quantization,
+        model_path=path,
+    )
+    assert model._llm is None
+
+    model.load()
+    assert isinstance(model._llm, MockPipeline)
+
+    responses_stream = list(model.generate("Hello", generate_config={"stream": True}))
+    for i in range(5):
+        assert (
+            responses_stream[i]["choices"][0]["text"]
+            == f"chatglm_test_stream_generate_{i}"
+        )
+
+    responses_non_stream = model.generate("Hello", generate_config={"stream": False})
+    assert responses_non_stream["choices"][0]["text"] == "chatglm_test_generate"