Spaces:
Running
on
Zero
Running
on
Zero
File size: 4,666 Bytes
51a7d9e bd34f0b 51a7d9e edb9e8a 9a6b8ed 51a7d9e 18aaf72 1ec2e60 f201032 51a7d9e 23d16e2 51a7d9e bd34f0b 18aaf72 bd34f0b 51a7d9e 2024746 651de2e 2024746 8830af9 9a6b8ed 51a7d9e 9a6b8ed 3f6e58a 51a7d9e 76f3d4e 9a6b8ed 76f3d4e 651de2e 9a6b8ed 51a7d9e bd34f0b fd6304d 51a7d9e bd34f0b 3b9cb87 bd34f0b 639e063 edb9e8a bd34f0b edb9e8a bd34f0b 51a7d9e 9a6b8ed 51a7d9e edb9e8a 51a7d9e edb9e8a 51a7d9e a3e36c2 51a7d9e 781217c 51a7d9e 579ca70 51a7d9e ef2eb9e 51a7d9e bd34f0b 51a7d9e 9a6b8ed 51a7d9e 9a6b8ed |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 |
import torch
from PIL import Image
import gradio as gr
import spaces
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
import os
from threading import Thread
import random
from datasets import load_dataset
HF_TOKEN = os.environ.get("HF_TOKEN", None)
MODEL_ID = "TeamDelta/mistral-yuki-7B"
MODELS = os.environ.get("MODELS")
MODEL_NAME = MODEL_ID.split("/")[-1]
TITLE = "<h1><center>New japanese LLM model webui</center></h1>"
DESCRIPTION = f"""
<h3>MODEL: <a href="https://hf.co/{MODELS}">{MODEL_NAME}</a></h3>
<center>
<p>TeamDelta/mistral-yuki-7B is the large language model built by Teamdelta.
<br>
Feel free to test without log.
</p>
</center>
"""
CSS = """
.duplicate-button {
margin: auto !important;
color: white !important;
background: black !important;
border-radius: 100vh !important;
}
h3 {
text-align: center;
}
.chatbox .messages .message.user {
background-color: #e1f5fe;
}
.chatbox .messages .message.bot {
background-color: #eeeeee;
}
"""
# モデルとトークナイザーの読み込み
model = AutoModelForCausalLM.from_pretrained(
MODEL_ID,
torch_dtype=torch.float16,
device_map="auto",
)
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
# データセットをロードしてスプリットを確認
dataset = load_dataset("elyza/ELYZA-tasks-100")
print(dataset)
# 使用するスプリット名を確認
split_name = "train" if "train" in dataset else "test" # デフォルトをtrainにし、なければtestにフォールバック
# 適切なスプリットから10個の例を取得
examples_list = list(dataset[split_name]) # スプリットをリストに変換
examples = random.sample(examples_list, 10) # リストからランダムに10個選択
example_inputs = [example['input'] for example in examples]
@spaces.GPU
def stream_chat(message: str, history: list, temperature: float, max_new_tokens: int, top_p: float, top_k: int, penalty: float):
print(f'message is - {message}')
print(f'history is - {history}')
conversation = []
for prompt, answer in history:
conversation.extend([{"role": "user", "content": prompt}, {"role": "assistant", "content": answer}])
conversation.append({"role": "user", "content": message})
input_ids = tokenizer.apply_chat_template(conversation, tokenize=False, add_generation_prompt=True)
inputs = tokenizer(input_ids, return_tensors="pt").to(0)
streamer = TextIteratorStreamer(tokenizer, timeout=10., skip_prompt=True, skip_special_tokens=True)
generate_kwargs = dict(
inputs,
streamer=streamer,
top_k=top_k,
top_p=top_p,
repetition_penalty=penalty,
max_new_tokens=max_new_tokens,
do_sample=True,
temperature=temperature,
eos_token_id=[128001, 128009],
)
thread = Thread(target=model.generate, kwargs=generate_kwargs)
thread.start()
buffer = ""
for new_text in streamer:
buffer += new_text
yield buffer
chatbot = gr.Chatbot(height=500)
with gr.Blocks(css=CSS) as demo:
gr.HTML(TITLE)
gr.HTML(DESCRIPTION)
gr.ChatInterface(
fn=stream_chat,
chatbot=chatbot,
fill_height=True,
theme="soft",
retry_btn=None,
undo_btn="Delete Previous",
clear_btn="Clear",
additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False),
additional_inputs=[
gr.Slider(
minimum=0,
maximum=1,
step=0.1,
value=0.8,
label="Temperature",
render=False,
),
gr.Slider(
minimum=128,
maximum=4096,
step=1,
value=1024,
label="Max new tokens",
render=False,
),
gr.Slider(
minimum=0.0,
maximum=1.0,
step=0.1,
value=0.8,
label="top_p",
render=False,
),
gr.Slider(
minimum=1,
maximum=20,
step=1,
value=20,
label="top_k",
render=False,
),
gr.Slider(
minimum=0.0,
maximum=2.0,
step=0.1,
value=1.0,
label="Repetition penalty",
render=False,
),
],
examples=example_inputs,
cache_examples=False,
)
if __name__ == "__main__":
demo.launch() |