Spaces:
Running
Running
| import gradio as gr | |
| import random | |
| import time | |
| from ctransformers import AutoModelForCausalLM | |
| from dl_hf_model import dl_hf_model | |
| params = { | |
| "max_new_tokens":512, | |
| "stop":["<end>" ,"<|endoftext|>","["], | |
| "temperature":0.7, | |
| "top_p":0.8, | |
| "stream":True, | |
| "batch_size": 8} | |
| #url = "https://huggingface.co/Aspik101/trurl-2-7b-GGML/blob/main/trurl-2-7b.ggmlv3.q8_0.bin" | |
| #model_loc, file_size = dl_hf_model(url) | |
| llm = AutoModelForCausalLM.from_pretrained("Aspik101/trurl-2-13b-GGML", model_type="llama") | |
| with gr.Blocks() as demo: | |
| chatbot = gr.Chatbot() | |
| msg = gr.Textbox() | |
| clear = gr.Button("Clear") | |
| def user(user_message, history): | |
| return "", history + [[user_message, None]] | |
| def parse_history(hist): | |
| history_ = "" | |
| for q, a in hist: | |
| history_ += f"<user>: {q } \n" | |
| if a: | |
| history_ += f"<assistant>: {a} \n" | |
| return history_ | |
| def bot(history): | |
| print("history: ",history) | |
| prompt = f"Jesteś AI assystentem. Odpowiadaj po polsku. {parse_history(history)}. <assistant>:" | |
| print("prompt: ",prompt) | |
| stream = llm(prompt, **params) | |
| history[-1][1] = "" | |
| answer_save = "" | |
| for character in stream: | |
| history[-1][1] += character | |
| answer_save += character | |
| time.sleep(0.005) | |
| yield history | |
| print("answer_save: ",answer_save) | |
| msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then( | |
| bot, chatbot, chatbot | |
| ) | |
| clear.click(lambda: None, None, chatbot, queue=False) | |
| demo.queue() | |
| demo.launch() |