:gem: [Feature] Support new model: openchat-3.5-0106

apis/chat_api.py

@@ -40,6 +40,13 @@ class ChatAPIApp:
                     "created": 1700000000,
                     "owned_by": "mistralai",
                 },
+                {
+                    "id": "nous-mixtral-8x7b",
+                    "description": "[NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO]: https://huggingface.co/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
+                    "object": "model",
+                    "created": 1700000000,
+                    "owned_by": "NousResearch",
+                },
                 {
                     "id": "mistral-7b",
                     "description": "[mistralai/Mistral-7B-Instruct-v0.2]: https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2",
@@ -48,11 +55,11 @@ class ChatAPIApp:
                     "owned_by": "mistralai",
                 },
                 {
-                    "id": "nous-mixtral-8x7b",
-                    "description": "[NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO]: https://huggingface.co/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
+                    "id": "openchat-3.5",
+                    "description": "[openchat/openchat-3.5-0106]: https://huggingface.co/openchat/openchat-3.5-0106",
                     "object": "model",
                     "created": 1700000000,
-                    "owned_by": "NousResearch",
+                    "owned_by": "openchat",
                 },
                 {
                     "id": "gemma-7b",

messagers/message_composer.py

@@ -1,6 +1,7 @@
 import re
 from pprint import pprint
 from utils.logger import logger
+from transformers import AutoTokenizer
 
 
 class MessageComposer:
@@ -8,8 +9,8 @@ class MessageComposer:
     AVALAIBLE_MODELS = [
         "mixtral-8x7b",
         "mistral-7b",
-        "openchat-3.5",
         "nous-mixtral-8x7b",
+        "openchat-3.5",
         "gemma-7b",
     ]
 
@@ -102,26 +103,30 @@ class MessageComposer:
             self.merged_str = "\n".join(self.merged_str_list)
         # https://huggingface.co/openchat/openchat-3.5-0106
         elif self.model in ["openchat-3.5"]:
-            self.messages = self.concat_messages_by_role(messages)
-            self.merged_str_list = []
-            self.end_of_turn = "<|end_of_turn|>"
-            for message in self.messages:
-                role = message["role"]
-                content = message["content"]
-                if role in self.inst_roles:
-                    self.merged_str_list.append(
-                        f"GPT4 Correct User:\n{content}{self.end_of_turn}"
-                    )
-                elif role in self.answer_roles:
-                    self.merged_str_list.append(
-                        f"GPT4 Correct Assistant:\n{content}{self.end_of_turn}"
-                    )
-                else:
-                    self.merged_str_list.append(
-                        f"GPT4 Correct User: {content}{self.end_of_turn}"
-                    )
-            self.merged_str_list.append(f"GPT4 Correct Assistant:\n")
-            self.merged_str = "\n".join(self.merged_str_list)
+            tokenizer = AutoTokenizer.from_pretrained("openchat/openchat-3.5-0106")
+            self.merged_str = tokenizer.apply_chat_template(
+                messages, tokenize=False, add_generation_prompt=True
+            )
+            # self.messages = self.concat_messages_by_role(messages)
+            # self.merged_str_list = []
+            # self.end_of_turn = "<|end_of_turn|>"
+            # for message in self.messages:
+            #     role = message["role"]
+            #     content = message["content"]
+            #     if role in self.inst_roles:
+            #         self.merged_str_list.append(
+            #             f"GPT4 Correct User:\n{content}{self.end_of_turn}"
+            #         )
+            #     elif role in self.answer_roles:
+            #         self.merged_str_list.append(
+            #             f"GPT4 Correct Assistant:\n{content}{self.end_of_turn}"
+            #         )
+            #     else:
+            #         self.merged_str_list.append(
+            #             f"GPT4 Correct User: {content}{self.end_of_turn}"
+            #         )
+            # self.merged_str_list.append(f"GPT4 Correct Assistant:\n")
+            # self.merged_str = "\n".join(self.merged_str_list)
         # https://huggingface.co/google/gemma-7b-it#chat-template
         elif self.model in ["gemma-7b"]:
             self.messages = self.concat_messages_by_role(messages)
@@ -265,7 +270,8 @@ class MessageComposer:
 if __name__ == "__main__":
     # model = "mixtral-8x7b"
     # model = "nous-mixtral-8x7b"
-    model = "gemma-7b"
+    # model = "gemma-7b"
+    model = "openchat-3.5"
     composer = MessageComposer(model)
     messages = [
         {
@@ -291,3 +297,5 @@ if __name__ == "__main__":
     pprint(composer.split(merged_str))
     # logger.note("merged merged_str:")
     # logger.mesg(composer.merge(composer.split(merged_str)))
+
+    # python -m messagers.message_composer

networks/message_streamer.py

@@ -5,6 +5,7 @@ from tiktoken import get_encoding as tiktoken_get_encoding
 from messagers.message_outputer import OpenaiStreamOutputer
 from utils.logger import logger
 from utils.enver import enver
+from transformers import AutoTokenizer
 
 
 class MessageStreamer:
@@ -12,8 +13,8 @@ class MessageStreamer:
         "mixtral-8x7b": "mistralai/Mixtral-8x7B-Instruct-v0.1",  # 72.62, fast [Recommended]
         "mistral-7b": "mistralai/Mistral-7B-Instruct-v0.2",  # 65.71, fast
         "nous-mixtral-8x7b": "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
+        "openchat-3.5": "openchat/openchat-3.5-0106",
         "gemma-7b": "google/gemma-7b-it",
-        # "openchat-3.5": "openchat/openchat-3.5-1210",  # 68.89, fast
         # "zephyr-7b-beta": "HuggingFaceH4/zephyr-7b-beta",  # ❌ Too Slow
         # "llama-70b": "meta-llama/Llama-2-70b-chat-hf",  # ❌ Require Pro User
         # "codellama-34b": "codellama/CodeLlama-34b-Instruct-hf",  # ❌ Low Score
@@ -43,7 +44,8 @@ class MessageStreamer:
             self.model = "default"
         self.model_fullname = self.MODEL_MAP[self.model]
         self.message_outputer = OpenaiStreamOutputer()
-        self.tokenizer = tiktoken_get_encoding("cl100k_base")
+        # self.tokenizer = tiktoken_get_encoding("cl100k_base")
+        self.tokenizer = AutoTokenizer.from_pretrained(self.model_fullname)
 
     def parse_line(self, line):
         line = line.decode("utf-8")

requirements.txt

@@ -8,5 +8,6 @@ requests
 sse_starlette
 termcolor
 tiktoken
+transformers
 uvicorn
 websockets