update

models/cpp_qwen2.py

@@ -6,6 +6,7 @@ https://github.com/awinml/llama-cpp-python-bindings
 from simulator import Simulator
 from llama_cpp import Llama
 import llama_cpp.llama_tokenizer
+from transformers import AutoTokenizer
 
 
 class Qwen2Simulator(Simulator):
@@ -20,14 +21,13 @@ class Qwen2Simulator(Simulator):
         #     verbose=False,
         # )
 
+        self.hf_tokenizer = AutoTokenizer.from_pretrained("/workspace/czy/model_weights/Qwen1.5-0.5B-Chat/")
         self.llm = Llama(
             model_path="/workspace/xusong/huggingface/models/Qwen1.5-0.5B-Chat-GGUF/qwen1_5-0_5b-chat-q8_0.gguf",
             # n_gpu_layers=-1, # Uncomment to use GPU acceleration
             # seed=1337, # Uncomment to set a specific seed
             # n_ctx=2048, # Uncomment to increase the context window
-            tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer.from_pretrained(
-                "/workspace/czy/model_weights/Qwen1.5-0.5B-Chat/"
-            ),
+            tokenizer=self.hf_tokenizer,
             verbose=False,
         )
 
@@ -38,7 +38,7 @@ class Qwen2Simulator(Simulator):
         :return:
         """
         assert messages[-1]["role"] != "user"
-        inputs = self.tokenizer.apply_chat_template(
+        inputs = self.hf_tokenizer.apply_chat_template(
             messages,
             tokenize=False,
             add_generation_prompt=False,
@@ -50,7 +50,7 @@ class Qwen2Simulator(Simulator):
 
     def generate_response(self, messages):
         assert messages[-1]["role"] == "user"
-        inputs = self.tokenizer.apply_chat_template(
+        inputs = self.hf_tokenizer.apply_chat_template(
             messages,
             tokenize=False,
             add_generation_prompt=True