Hugging Face's logo

Spaces:
mii-llm
/
maestrale-chat-v0.4-beta
Running on Zero

App Files Community
maestrale-chat-v0.4-beta
File size: 3,692 Bytes 
5e3a4bc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
 
import gradio as gr
import torch
from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
    TextIteratorStreamer,
)
import os
from threading import Thread
import spaces
import time
import subprocess

PLACEHOLDER = """
<div style="padding: 40px; text-align: center; display: flex; flex-direction: column; align-items: center;">
   <img src="https://i.imgur.com/dgSNbTl.jpg" style="width: 90%; max-width: 650px; height: auto; opacity: 0.8;  "> 
   <h1 style="font-size: 28px; margin-top: 20px; margin-bottom: 2px; opacity: 0.55;">mii-llm / Maestrale</h1>
   <p style="font-size: 18px; margin-bottom: 2px; opacity: 0.65;">Chiedi pure...</p>
</div>
"""

DESCRIPTION = """<div>
<p>🇮🇹 Italian LLM <a href="https://huggingface.co/mii-llm/maestrale-chat-v0.3-beta"><b>Maestrale Chat v0.3 beta</b></a>. Maestrale is a powerful language model for Italian, trained by mii-llm, based on Mistral 7B.</p>
<p>🔎 For more details about the Maestrale model and how to use it with <code>transformers</code>, visit the <a href="https://huggingface.co/mii-llm/maestrale-chat-v0.3-beta">model card</a>.</p>
</div>"""

tokenizer = AutoTokenizer.from_pretrained("mii-llm/maestrale-chat-v0.4-alpha")
model = AutoModelForCausalLM.from_pretrained("mii-llm/maestrale-chat-v0.4-alpha", device_map="auto")

terminators = [
    tokenizer.eos_token_id,
    tokenizer.convert_tokens_to_ids("<|im_end|>")
]

if torch.cuda.is_available():
    device = torch.device("cuda")
    print(f"Using GPU: {torch.cuda.get_device_name(device)}")
else:
    device = torch.device("cpu")
    print("Using CPU")


model = model.to(device)


@spaces.GPU(duration=60)
def chat(message, history, system, temperature, do_sample, max_tokens):
    chat = [{"role": "system", "content": system}] if system else []
    chat.extend(
        {"role": role, "content": content}
        for user, assistant in history
        for role, content in [("user", user), ("assistant", assistant)]
    )
    chat.append({"role": "user", "content": message})

    messages = tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)
    model_inputs = tokenizer([messages], return_tensors="pt").to(device)

    streamer = TextIteratorStreamer(
        tokenizer, timeout=20.0, skip_prompt=True, skip_special_tokens=True
    )

    generate_kwargs = {
        **model_inputs,
        "streamer": streamer,
        "max_new_tokens": max_tokens,
        "do_sample": do_sample,
        "temperature": temperature,
        "eos_token_id": terminators,
    }

    thread = Thread(target=model.generate, kwargs=generate_kwargs)
    thread.start()

    partial_text = ""
    for new_text in streamer:
        partial_text += new_text
        yield partial_text

    yield partial_text


chatbot = gr.Chatbot(height=550, placeholder=PLACEHOLDER, label='Conversazione')

demo = gr.ChatInterface(
    fn=chat,
    chatbot=chatbot,
    fill_height=True,
    theme=gr.themes.Soft(),
    additional_inputs_accordion=gr.Accordion(
        label="⚙️ Parametri", open=False, render=False
    ),
    additional_inputs=[
        gr.Textbox(
            label="System",
            value="sei un assistente utile.",
        ),
        gr.Slider(
            minimum=0, maximum=1, step=0.1, value=0.7, label="Temperature", render=False
        ),
        gr.Checkbox(label="Sampling", value=True),
        gr.Slider(
            minimum=128,
            maximum=4096,
            step=1,
            value=512,
            label="Max new tokens",
            render=False,
        ),
    ],
    stop_btn="Stop Generation",
    cache_examples=False,
    title="Maestrale Chat v0.3 beta",
    description=DESCRIPTION
)

demo.launch()