Update app.py

app.py

@@ -2,9 +2,11 @@ import gradio as gr
 import spaces
 from huggingface_hub import hf_hub_download
 from llama_cpp import Llama
+from transformers import AutoTokenizer
 
 MAX_INPUT_LIMIT = 3584
 MAX_NEW_TOKENS = 1536
+MODEL_HF = "Azure99/blossom-v5.1-34b"
 MODEL_REPO = "Azure99/blossom-v5.1-34b-gguf"
 MODEL_FILE = "model-q6_k.gguf"
 MODEL_LOCAL_DIR = "./"
@@ -16,22 +18,24 @@ hf_hub_download(
 )
 
 llm: Llama = None
+tokenizer = AutoTokenizer.from_pretrained(MODEL_HF)
 
 
-def get_input_ids(input, history):
-    prefix = "A chat between a human and an artificial intelligence bot. The bot gives helpful, detailed, and polite answers to the human's questions."
+def get_input_ids(inst, history):
+    prefix = ("A chat between a human and an artificial intelligence bot. "
+              "The bot gives helpful, detailed, and polite answers to the human's questions.")
     patterns = []
     for conv in history:
         patterns.append(f'\n|Human|: {conv[0]}\n|Bot|: ')
         patterns.append(f'{conv[1]}')
-    patterns.append(f'\n|Human|: {input}\n|Bot|: ')
+    patterns.append(f'\n|Human|: {inst}\n|Bot|: ')
     patterns[0] = prefix + patterns[0]
 
     input_ids = []
     for i, pattern in enumerate(patterns):
-        input_ids += llm.tokenize(pattern.encode("utf-8"), add_bos=False, special=True)
+        input_ids += tokenizer.encode(pattern, add_special_tokens=(i == 0))
         if i % 2 == 1:
-            input_ids += [llm.token_eos()]
+            input_ids += [tokenizer.eos_token_id]
     return input_ids