Spaces:
Sleeping
Sleeping
from threading import Thread # Import the Thread class from the threading module | |
import torch # Import the PyTorch library | |
import gradio as gr # Import Gradio for creating a UI | |
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, TextIteratorStreamer # Import Hugging Face Transformers | |
# Define the Hugging Face model ID and check for available GPU (cuda) | |
model_id = "declare-lab/flan-alpaca-large" | |
torch_device = "cuda" if torch.cuda.is_available() else "cpu" | |
print("Running on device:", torch_device) | |
print("CPU threads:", torch.get_num_threads()) | |
# Load the pre-trained model based on the device | |
if torch_device == "cuda": | |
model = AutoModelForSeq2SeqLM.from_pretrained(model_id, load_in_8bit=True, device_map="auto") | |
else: | |
model = AutoModelForSeq2SeqLM.from_pretrained(model_id) | |
tokenizer = AutoTokenizer.from_pretrained(model_id) | |
# Define a function to run model text generation | |
def run_generation(user_text, top_p, temperature, top_k, max_new_tokens): | |
# Get the model and tokenizer, and tokenize the user text. | |
model_inputs = tokenizer([user_text], return_tensors="pt").to(torch_device) | |
# Start generation on a separate thread, so that we don't block the UI. | |
# Adds timeout to the streamer to handle exceptions in the generation thread. | |
streamer = TextIteratorStreamer(tokenizer, timeout=10., skip_prompt=True, skip_special_tokens=True) | |
generate_kwargs = dict( | |
model_inputs, | |
streamer=streamer, | |
max_new_tokens=max_new_tokens, | |
do_sample=True, | |
top_p=top_p, | |
temperature=float(temperature), | |
top_k=top_k | |
) | |
# Create a new thread for model generation | |
t = Thread(target=model.generate, kwargs=generate_kwargs) | |
t.start() | |
model_output = "" | |
for new_text in streamer: | |
model_output += new_text | |
yield model_output | |
return model_output | |
# Define a function to reset the user input textbox | |
def reset_textbox(): | |
return gr.update(value='') | |
# Create a Gradio UI interface | |
with gr.Blocks() as demo: | |
# Display a title | |
gr.Markdown( | |
"# Testing ALPACA Model \n" | |
) | |
with gr.Row(): | |
with gr.Column(scale=4): | |
# Create a textbox for user input | |
user_text = gr.Textbox( | |
placeholder="Ask Me Anything ... ", | |
label="User input" | |
) | |
# Create a textbox for model output | |
model_output = gr.Textbox(label="Model output", lines=10, interactive=False) | |
# Create a submit button | |
button_submit = gr.Button(value="Submit") | |
with gr.Column(scale=1): | |
# Create sliders for adjusting generation parameters | |
max_new_tokens = gr.Slider( | |
minimum=1, maximum=1000, value=250, step=1, interactive=True, label="Max New Tokens", | |
) | |
top_p = gr.Slider( | |
minimum=0.05, maximum=1.0, value=0.95, step=0.05, interactive=True, label="Top-p (nucleus sampling)", | |
) | |
top_k = gr.Slider( | |
minimum=1, maximum=50, value=50, step=1, interactive=True, label="Top-k", | |
) | |
temperature = gr.Slider( | |
minimum=0.1, maximum=5.0, value=0.8, step=0.1, interactive=True, label="Temperature", | |
) | |
# Set up the submission of user input | |
user_text.submit(run_generation, [user_text, top_p, temperature, top_k, max_new_tokens], model_output) | |
button_submit.click(run_generation, [user_text, top_p, temperature, top_k, max_new_tokens], model_output) | |
# Launch the Gradio interface | |
demo.queue(max_size=32).launch(enable_queue=True) |