Spaces:
Runtime error
Runtime error
File size: 2,216 Bytes
d6e5fcb f4f4db9 d6e5fcb |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 |
import gradio as gr
import random
import time
from huggingface_hub import InferenceClient
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct")
client = InferenceClient(model="https://3ba9-20-63-4-233.ngrok-free.app")
SYSTEM_COMMAND = {"role": "system", "content": "Context: date: Monday 20th May 2024; location: NYC; running on: 8 AMD Instinct MI300 GPU; model name: Llama 70B. Only provide these information if asked. You are a knowledgeable assistant trained to provide accurate and helpful information. Please respond to the user's queries promptly and politely."}
IGNORED_TOKENS = {None, "<|start_header_id|>", "<|end_header_id|>", "<|eot_id|>", "<|reserved_special_token"}
STOP_TOKENS = ["<|start_header_id|>", "<|end_header_id|>", "<|eot_id|>", "<|reserved_special_token"]
with gr.Blocks() as demo:
tfs_history = gr.State([SYSTEM_COMMAND])
chatbot = gr.Chatbot()
msg = gr.Textbox()
clear = gr.Button("Clear")
def user(user_message, history, dict_history):
data = {"role": "user", "content": user_message}
dict_history.append(data)
return "", history + [[user_message, None]], dict_history
def bot(history, dict_history):
history[-1][1] = ""
response = {"role": "assistant", "content": ""}
start_tokenize = time.perf_counter()
text_input = tokenizer.apply_chat_template(dict_history, tokenize=False, add_generation_prompt=True)
end_tokenize = time.perf_counter()
try:
for token in client.text_generation(prompt=text_input, max_new_tokens=100, stop_sequences=STOP_TOKENS, stream=True):
if token not in IGNORED_TOKENS:
history[-1][1] += token
response["content"] += token
yield history
finally:
dict_history.append(response)
msg.submit(
user,
inputs=[msg, chatbot, tfs_history],
outputs=[msg, chatbot, tfs_history],
queue=False).then(
bot,
[chatbot, tfs_history],
chatbot
)
clear.click(lambda: None, None, chatbot, queue=False)
demo.queue()
demo.launch()
|