import gradio as gr import random import time from huggingface_hub import InferenceClient from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct") client = InferenceClient(model="https://770c-20-63-4-233.ngrok-free.app") SYSTEM_COMMAND = {"role": "system", "content": "Context: date: Monday 20th May 2024; location: NYC; running on: 8 AMD Instinct MI300 GPU; model name: Llama 70B. Only provide these information if asked. You are a knowledgeable assistant trained to provide accurate and helpful information. Please respond to the user's queries promptly and politely."} IGNORED_TOKENS = {None, "<|start_header_id|>", "<|end_header_id|>", "<|eot_id|>", "<|reserved_special_token"} STOP_TOKENS = ["<|start_header_id|>", "<|end_header_id|>", "<|eot_id|>", "<|reserved_special_token"] with gr.Blocks() as demo: tfs_history = gr.State([SYSTEM_COMMAND]) chatbot = gr.Chatbot() msg = gr.Textbox() clear = gr.Button("Clear") def user(user_message, history, dict_history): data = {"role": "user", "content": user_message} dict_history.append(data) return "", history + [[user_message, None]], dict_history def bot(history, dict_history): history[-1][1] = "" response = {"role": "assistant", "content": ""} start_tokenize = time.perf_counter() text_input = tokenizer.apply_chat_template(dict_history, tokenize=False, add_generation_prompt=True) end_tokenize = time.perf_counter() try: for token in client.text_generation(prompt=text_input, max_new_tokens=100, stop_sequences=STOP_TOKENS, stream=True): if token not in IGNORED_TOKENS: history[-1][1] += token response["content"] += token yield history finally: dict_history.append(response) msg.submit( user, inputs=[msg, chatbot, tfs_history], outputs=[msg, chatbot, tfs_history], queue=False).then( bot, [chatbot, tfs_history], chatbot ) clear.click(lambda: None, None, chatbot, queue=False) demo.queue() demo.launch()