Update app.py

app.py

@@ -3,7 +3,7 @@ import torch
 import os
 import gradio as gr
 import sentencepiece
-from tokenization_yi import YiTokenizer
+# from tokenization_yi import YiTokenizer
 
 
 from transformers import AutoModelForCausalLM, GPTQConfig, AutoTokenizer, AutoModelForCausalLM
@@ -16,9 +16,13 @@ from tokenization_yi import YiTokenizer
 os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'max_split_size_mb:50'
 model_id = "TheBloke/Yi-34B-200K-Llamafied-GPTQ"
 
-gptq_config = GPTQConfig(bits=4, exllama_config={"version": 2})
-tokenizer = YiTokenizer.from_pretrained("./")
-model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", torch_dtype="auto", trust_remote_code=True, quantization_config=gptq_config)
+
+tokenizer = AutoTokenizer.from_pretrained("larryvrh/Yi-34B-200K-Llamafied")
+model = AutoModelForCausalLM.from_pretrained("larryvrh/Yi-34B-200K-Llamafied", device_map="auto", torch_dtype="bfloat16", trust_remote_code=True)
+
+# gptq_config = GPTQConfig(bits=4, exllama_config={"version": 2})
+# tokenizer = YiTokenizer.from_pretrained("./")
+# model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", torch_dtype="auto", trust_remote_code=True, quantization_config=gptq_config)
 
 def predict(message, max_new_tokens=4056, temperature=3.5, top_p=0.9, top_k=800):
     prompt = message.strip()