Repository: xorbitsai/inference

Issue:
FEAT: support chatglm-6b
We can leverage https://github.com/li-plus/chatglm.cpp to serve chatglm-6b.

1. download chatglm2 pytorch model from huggingface
2. convert it to ggml using `chatglm.cpp/convert.py`
3. run the demo
4. verify the python bindings
5. impl chatglm model
6. add chatglm as one of our builtin models


Discussion:
take

Resolving patch:
diff --git a/plexar/core/gradio.py b/plexar/core/gradio.py
--- a/plexar/core/gradio.py
+++ b/plexar/core/gradio.py
@@ -90,6 +90,10 @@ async def generate(
             if window_size != 0:
                 history = history[-(window_size // 2) :]
 
+            # chatglm only support even number of conversation history.
+            if len(history) % 2 != 0:
+                history = history[1:]
+
             generate_config = dict(
                 max_tokens=max_token,
                 temperature=temperature,
diff --git a/plexar/model/llm/__init__.py b/plexar/model/llm/__init__.py
--- a/plexar/model/llm/__init__.py
+++ b/plexar/model/llm/__init__.py
@@ -14,9 +14,9 @@
 
 
 def install():
-    from plexar.model.llm.core import LlamaCppModel
-
     from .. import MODEL_FAMILIES, ModelFamily
+    from .chatglm import ChatglmCppChatModel
+    from .core import LlamaCppModel
     from .orca import OrcaMiniGgml
     from .vicuna import VicunaCensoredGgml
     from .wizardlm import WizardlmGgml
@@ -166,3 +166,24 @@ def install():
             cls=OrcaMiniGgml,
         )
     )
+
+    chatglm_url_generator = lambda model_size, quantization: (
+        f"https://huggingface.co/Xorbits/chatglm2-{model_size}B-GGML/blob/main/"
+        f"chatglm2-ggml-{quantization}.bin"
+    )
+    MODEL_FAMILIES.append(
+        ModelFamily(
+            model_name="chatglm2",
+            model_sizes_in_billions=[6],
+            model_format="ggmlv3",
+            quantizations=[
+                "q4_0",
+                "q4_1",
+                "q5_0",
+                "q5_1",
+                "q8_0",
+            ],
+            url_generator=chatglm_url_generator,
+            cls=ChatglmCppChatModel,
+        )
+    )
diff --git a/plexar/model/llm/chatglm.py b/plexar/model/llm/chatglm.py
new file mode 100644
--- /dev/null
+++ b/plexar/model/llm/chatglm.py
@@ -0,0 +1,181 @@
+# Copyright 2022-2023 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import time
+import uuid
+from pathlib import Path
+from typing import Iterator, List, Optional, TypedDict, Union
+
+from .core import Model
+from .types import ChatCompletion, ChatCompletionChunk, ChatCompletionMessage
+
+logger = logging.getLogger(__name__)
+
+
+class ChatglmCppModelConfig(TypedDict, total=False):
+    pass
+
+
+class ChatglmCppGenerateConfig(TypedDict, total=False):
+    max_tokens: int
+    top_p: float
+    temperature: float
+    stream: bool
+
+
+class ChatglmCppChatModel(Model):
+    def __init__(
+        self,
+        model_path: str,
+        model_config: Optional[ChatglmCppModelConfig] = None,
+    ):
+        super().__init__()
+        self._llm = None
+        self._model_path = model_path
+        self._model_name = "-".join(self._model_path.split("/")[-2].split("-")[:-3])
+
+        # just a placeholder for now as the chatglm_cpp repo doesn't support model config
+        self._model_config = model_config
+
+    @classmethod
+    def _sanitize_generate_config(
+        cls,
+        chatglmcpp_generate_config: Optional[ChatglmCppGenerateConfig],
+    ) -> ChatglmCppGenerateConfig:
+        if chatglmcpp_generate_config is None:
+            chatglmcpp_generate_config = ChatglmCppGenerateConfig()
+        chatglmcpp_generate_config.setdefault("max_tokens", 8192)
+        chatglmcpp_generate_config.setdefault("temperature", 0.95)
+        chatglmcpp_generate_config.setdefault("top_p", 0.8)
+        chatglmcpp_generate_config.setdefault("stream", False)
+        return chatglmcpp_generate_config
+
+    def load(self):
+        try:
+            import chatglm_cpp
+        except ImportError:
+            error_message = "Failed to import module 'chatglm_cpp'"
+            installation_guide = [
+                "Please make sure 'chatglm_cpp' is installed. ",
+                "You can install it by running the following command in the terminal:\n",
+                "pip install git+https://github.com/li-plus/chatglm.cpp.git@main\n\n",
+                "Or visit the original git repo if the above command fails:\n",
+                "https://github.com/li-plus/chatglm.cpp",
+            ]
+
+            raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}")
+
+        self._llm = chatglm_cpp.Pipeline(Path(self._model_path))
+
+    @staticmethod
+    def _convert_raw_text_chunks_to_chat(
+        tokens: Iterator[str], model_name: str
+    ) -> Iterator[ChatCompletionChunk]:
+        yield {
+            "id": "chat" + f"cmpl-{str(uuid.uuid4())}",
+            "model": model_name,
+            "object": "chat.completion.chunk",
+            "created": int(time.time()),
+            "choices": [
+                {
+                    "index": 0,
+                    "delta": {
+                        "role": "assistant",
+                    },
+                    "finish_reason": None,
+                }
+            ],
+        }
+        for token in enumerate(tokens):
+            yield {
+                "id": "chat" + f"cmpl-{str(uuid.uuid4())}",
+                "model": model_name,
+                "object": "chat.completion.chunk",
+                "created": int(time.time()),
+                "choices": [
+                    {
+                        "index": 0,
+                        "delta": {
+                            "content": token[1],
+                        },
+                        "finish_reason": None,
+                    }
+                ],
+            }
+
+    @staticmethod
+    def _convert_raw_text_completion_to_chat(
+        text: str, model_name: str
+    ) -> ChatCompletion:
+        return {
+            "id": "chat" + f"cmpl-{str(uuid.uuid4())}",
+            "model": model_name,
+            "object": "chat.completion",
+            "created": int(time.time()),
+            "choices": [
+                {
+                    "index": 0,
+                    "message": {
+                        "role": "assistant",
+                        "content": text,
+                    },
+                    "finish_reason": None,
+                }
+            ],
+            "usage": {
+                "prompt_tokens": -1,
+                "completion_tokens": -1,
+                "total_tokens": -1,
+            },
+        }
+
+    def chat(
+        self,
+        prompt: str,
+        chat_history: Optional[List[ChatCompletionMessage]] = None,
+        generate_config: Optional[ChatglmCppGenerateConfig] = None,
+    ) -> Union[ChatCompletion, Iterator[ChatCompletionChunk]]:
+        if chat_history is not None:
+            chat_history_list = [message["content"] for message in chat_history]
+        else:
+            chat_history_list = []
+
+        chat_history_list.append(prompt)
+        logger.debug("Full conversation history:\n%s", str(chat_history_list))
+
+        generate_config = self._sanitize_generate_config(generate_config)
+
+        assert self._llm is not None
+
+        if generate_config.get("stream", False):
+            it = self._llm.stream_chat(
+                chat_history_list,
+                max_context_length=8192,
+                max_length=generate_config["max_tokens"],
+                temperature=generate_config["temperature"],
+                top_p=generate_config["top_p"],
+            )
+            assert not isinstance(it, str)
+            return self._convert_raw_text_chunks_to_chat(it, self._model_name)
+        else:
+            c = self._llm.chat(
+                chat_history_list,
+                max_context_length=8192,
+                max_length=generate_config["max_tokens"],
+                temperature=generate_config["temperature"],
+                top_p=generate_config["top_p"],
+            )
+            assert not isinstance(c, Iterator)
+            return self._convert_raw_text_completion_to_chat(c, self._model_name)


Test patch: