Spaces:
Running
on
Zero
Running
on
Zero
import gradio as gr | |
import torch | |
from transformers import ( | |
AutoModelForCausalLM, | |
AutoTokenizer, | |
TextIteratorStreamer, | |
) | |
import os | |
from threading import Thread | |
import spaces | |
import time | |
import subprocess | |
PLACEHOLDER = """ | |
<div style="padding: 40px; text-align: center; display: flex; flex-direction: column; align-items: center;"> | |
<img src="https://i.imgur.com/3MUzS0kl.jpg" style="width: 90%; max-width: 650px; height: auto; opacity: 0.8; border-radius: 20px;"> | |
<h1 style="font-size: 28px; margin-top: 20px; margin-bottom: 2px; opacity: 0.55;">mii-llm / Maestrale</h1> | |
<p style="font-size: 18px; margin-bottom: 2px; opacity: 0.65;">Chiedi pure...</p> | |
</div> | |
""" | |
DESCRIPTION = """<div> | |
<p>๐ฎ๐น Italian LLM <a href="https://huggingface.co/mii-llm/maestrale-chat-v0.4-alpha"><b>Maestrale Chat v0.4 beta</b></a>. Maestrale is a powerful language model for Italian, trained by mii-llm, based on Mistral 7B.</p> | |
<p>๐ For more details about Maestrale and how to use it with <code>transformers</code>, visit the <a href="https://huggingface.co/mii-llm/maestrale-chat-v0.4-alpha">model card</a>.</p> | |
</div>""" | |
_token = os.environ["HUGGINGFACE_API_KEY"] | |
tokenizer = AutoTokenizer.from_pretrained("mii-llm/maestrale-chat-v0.4-alpha-v2", token=_token) | |
model = AutoModelForCausalLM.from_pretrained("mii-llm/maestrale-chat-v0.4-alpha-v2", torch_dtype=torch.bfloat16, device_map="auto", token=_token) | |
terminators = [ | |
tokenizer.eos_token_id, | |
tokenizer.convert_tokens_to_ids("<|im_end|>") | |
] | |
if torch.cuda.is_available(): | |
device = torch.device("cuda") | |
print(f"Using GPU: {torch.cuda.get_device_name(device)}") | |
else: | |
device = torch.device("cpu") | |
print("Using CPU") | |
model = model.to(device) | |
def chat(message, history, system, temperature, do_sample, max_tokens): | |
chat = [{"role": "system", "content": system}] if system else [] | |
chat.extend( | |
{"role": role, "content": content} | |
for user, assistant in history | |
for role, content in [("user", user), ("assistant", assistant)] | |
) | |
chat.append({"role": "user", "content": message}) | |
messages = tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True) | |
model_inputs = tokenizer([messages], return_tensors="pt").to(device) | |
streamer = TextIteratorStreamer( | |
tokenizer, timeout=20.0, skip_prompt=True, skip_special_tokens=True | |
) | |
generate_kwargs = { | |
**model_inputs, | |
"streamer": streamer, | |
"max_new_tokens": max_tokens, | |
"do_sample": do_sample, | |
"temperature": temperature, | |
"eos_token_id": terminators, | |
"pad_token_id": tokenizer.eos_token_id | |
} | |
thread = Thread(target=model.generate, kwargs=generate_kwargs) | |
thread.start() | |
partial_text = "" | |
for new_text in streamer: | |
partial_text += new_text | |
yield partial_text | |
yield partial_text | |
chatbot = gr.Chatbot(height=550, placeholder=PLACEHOLDER, label='Conversazione', show_copy_button=True) | |
demo = gr.ChatInterface( | |
fn=chat, | |
chatbot=chatbot, | |
fill_height=True, | |
theme=gr.themes.Soft(), | |
additional_inputs_accordion=gr.Accordion( | |
label="โ๏ธ Parametri", open=False, render=False | |
), | |
additional_inputs=[ | |
gr.Textbox( | |
label="System", | |
value="Sei un assistente utile.", | |
), | |
gr.Slider( | |
minimum=0, maximum=1, step=0.1, value=0.7, label="Temperature", render=False | |
), | |
gr.Checkbox(label="Sampling", value=True), | |
gr.Slider( | |
minimum=128, | |
maximum=4096, | |
step=1, | |
value=1024, | |
label="Max new tokens", | |
render=False, | |
), | |
], | |
stop_btn="Stop Generation", | |
cache_examples=False, | |
title="Maestrale Chat v0.4 Alpha", | |
description=DESCRIPTION | |
) | |
demo.launch() |