Spaces:
Runtime error
Runtime error
Commit
•
fffd084
0
Parent(s):
Duplicate from mosaicml/mpt-7b-chat
Browse filesCo-authored-by: Sam <sam-mosaic@users.noreply.huggingface.co>
- .gitattributes +34 -0
- .gitignore +4 -0
- README.md +13 -0
- app.py +312 -0
- requirements.txt +9 -0
.gitattributes
ADDED
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
28 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
29 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
30 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
31 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
32 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
33 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
34 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
.gitignore
ADDED
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
1 |
+
venv/
|
2 |
+
.venv/
|
3 |
+
env/
|
4 |
+
.env/
|
README.md
ADDED
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
---
|
2 |
+
title: MPT-7B-Chat
|
3 |
+
emoji: 🤖
|
4 |
+
colorFrom: purple
|
5 |
+
colorTo: yellow
|
6 |
+
sdk: gradio
|
7 |
+
sdk_version: 3.27.0
|
8 |
+
app_file: app.py
|
9 |
+
pinned: false
|
10 |
+
duplicated_from: mosaicml/mpt-7b-chat
|
11 |
+
---
|
12 |
+
|
13 |
+
Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
|
app.py
ADDED
@@ -0,0 +1,312 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Copyright 2023 MosaicML spaces authors
|
2 |
+
# SPDX-License-Identifier: Apache-2.0
|
3 |
+
import datetime
|
4 |
+
import os
|
5 |
+
from threading import Event, Thread
|
6 |
+
from uuid import uuid4
|
7 |
+
|
8 |
+
import gradio as gr
|
9 |
+
import requests
|
10 |
+
import torch
|
11 |
+
from transformers import (
|
12 |
+
AutoModelForCausalLM,
|
13 |
+
AutoTokenizer,
|
14 |
+
StoppingCriteria,
|
15 |
+
StoppingCriteriaList,
|
16 |
+
TextIteratorStreamer,
|
17 |
+
)
|
18 |
+
|
19 |
+
|
20 |
+
model_name = "mosaicml/mpt-7b-chat"
|
21 |
+
max_new_tokens = 1536
|
22 |
+
|
23 |
+
# # small testing model:
|
24 |
+
# model_name = "gpt2"
|
25 |
+
# max_new_tokens = 128
|
26 |
+
|
27 |
+
auth_token = os.getenv("HF_TOKEN", None)
|
28 |
+
|
29 |
+
print(f"Starting to load the model {model_name} into memory")
|
30 |
+
|
31 |
+
m = AutoModelForCausalLM.from_pretrained(
|
32 |
+
model_name,
|
33 |
+
torch_dtype=torch.bfloat16,
|
34 |
+
trust_remote_code=True,
|
35 |
+
use_auth_token=auth_token,
|
36 |
+
max_seq_len=8192,
|
37 |
+
).cuda()
|
38 |
+
tok = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True, use_auth_token=auth_token)
|
39 |
+
|
40 |
+
stop_token_ids = tok.convert_tokens_to_ids(["<|im_end|>", "<|endoftext|>"])
|
41 |
+
|
42 |
+
print(f"Successfully loaded the model {model_name} into memory")
|
43 |
+
|
44 |
+
|
45 |
+
start_message = """<|im_start|>system
|
46 |
+
- You are a helpful assistant chatbot trained by MosaicML.
|
47 |
+
- You answer questions.
|
48 |
+
- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.
|
49 |
+
- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>
|
50 |
+
"""
|
51 |
+
|
52 |
+
|
53 |
+
class StopOnTokens(StoppingCriteria):
|
54 |
+
def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
|
55 |
+
for stop_id in stop_token_ids:
|
56 |
+
if input_ids[0][-1] == stop_id:
|
57 |
+
return True
|
58 |
+
return False
|
59 |
+
|
60 |
+
|
61 |
+
def convert_history_to_text(history):
|
62 |
+
text = start_message + "".join(
|
63 |
+
[
|
64 |
+
"".join(
|
65 |
+
[
|
66 |
+
f"<|im_start|>user\n{item[0]}<|im_end|>",
|
67 |
+
f"<|im_start|>assistant\n{item[1]}<|im_end|>",
|
68 |
+
]
|
69 |
+
)
|
70 |
+
for item in history[:-1]
|
71 |
+
]
|
72 |
+
)
|
73 |
+
text += "".join(
|
74 |
+
[
|
75 |
+
"".join(
|
76 |
+
[
|
77 |
+
f"<|im_start|>user\n{history[-1][0]}<|im_end|>",
|
78 |
+
f"<|im_start|>assistant\n{history[-1][1]}",
|
79 |
+
]
|
80 |
+
)
|
81 |
+
]
|
82 |
+
)
|
83 |
+
return text
|
84 |
+
|
85 |
+
|
86 |
+
def log_conversation(conversation_id, history, messages, generate_kwargs):
|
87 |
+
logging_url = os.getenv("LOGGING_URL", None)
|
88 |
+
if logging_url is None:
|
89 |
+
return
|
90 |
+
|
91 |
+
timestamp = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
|
92 |
+
|
93 |
+
data = {
|
94 |
+
"conversation_id": conversation_id,
|
95 |
+
"timestamp": timestamp,
|
96 |
+
"history": history,
|
97 |
+
"messages": messages,
|
98 |
+
"generate_kwargs": generate_kwargs,
|
99 |
+
}
|
100 |
+
|
101 |
+
try:
|
102 |
+
requests.post(logging_url, json=data)
|
103 |
+
except requests.exceptions.RequestException as e:
|
104 |
+
print(f"Error logging conversation: {e}")
|
105 |
+
|
106 |
+
|
107 |
+
def user(message, history):
|
108 |
+
# Append the user's message to the conversation history
|
109 |
+
return "", history + [[message, ""]]
|
110 |
+
|
111 |
+
|
112 |
+
def bot(history, temperature, top_p, top_k, repetition_penalty, conversation_id):
|
113 |
+
print(f"history: {history}")
|
114 |
+
# Initialize a StopOnTokens object
|
115 |
+
stop = StopOnTokens()
|
116 |
+
|
117 |
+
# Construct the input message string for the model by concatenating the current system message and conversation history
|
118 |
+
messages = convert_history_to_text(history)
|
119 |
+
|
120 |
+
# Tokenize the messages string
|
121 |
+
input_ids = tok(messages, return_tensors="pt").input_ids
|
122 |
+
input_ids = input_ids.to(m.device)
|
123 |
+
streamer = TextIteratorStreamer(tok, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
|
124 |
+
generate_kwargs = dict(
|
125 |
+
input_ids=input_ids,
|
126 |
+
max_new_tokens=max_new_tokens,
|
127 |
+
temperature=temperature,
|
128 |
+
do_sample=temperature > 0.0,
|
129 |
+
top_p=top_p,
|
130 |
+
top_k=top_k,
|
131 |
+
repetition_penalty=repetition_penalty,
|
132 |
+
streamer=streamer,
|
133 |
+
stopping_criteria=StoppingCriteriaList([stop]),
|
134 |
+
)
|
135 |
+
|
136 |
+
stream_complete = Event()
|
137 |
+
|
138 |
+
def generate_and_signal_complete():
|
139 |
+
m.generate(**generate_kwargs)
|
140 |
+
stream_complete.set()
|
141 |
+
|
142 |
+
def log_after_stream_complete():
|
143 |
+
stream_complete.wait()
|
144 |
+
log_conversation(
|
145 |
+
conversation_id,
|
146 |
+
history,
|
147 |
+
messages,
|
148 |
+
{
|
149 |
+
"top_k": top_k,
|
150 |
+
"top_p": top_p,
|
151 |
+
"temperature": temperature,
|
152 |
+
"repetition_penalty": repetition_penalty,
|
153 |
+
},
|
154 |
+
)
|
155 |
+
|
156 |
+
t1 = Thread(target=generate_and_signal_complete)
|
157 |
+
t1.start()
|
158 |
+
|
159 |
+
t2 = Thread(target=log_after_stream_complete)
|
160 |
+
t2.start()
|
161 |
+
|
162 |
+
# Initialize an empty string to store the generated text
|
163 |
+
partial_text = ""
|
164 |
+
for new_text in streamer:
|
165 |
+
partial_text += new_text
|
166 |
+
history[-1][1] = partial_text
|
167 |
+
yield history
|
168 |
+
|
169 |
+
|
170 |
+
def get_uuid():
|
171 |
+
return str(uuid4())
|
172 |
+
|
173 |
+
|
174 |
+
with gr.Blocks(
|
175 |
+
theme=gr.themes.Soft(),
|
176 |
+
css=".disclaimer {font-variant-caps: all-small-caps;}",
|
177 |
+
) as demo:
|
178 |
+
conversation_id = gr.State(get_uuid)
|
179 |
+
gr.Markdown(
|
180 |
+
"""<h1><center>MosaicML MPT-7B-Chat</center></h1>
|
181 |
+
|
182 |
+
This demo is of [MPT-7B-Chat](https://huggingface.co/mosaicml/mpt-7b-chat). It is based on [MPT-7B](https://huggingface.co/mosaicml/mpt-7b) fine-tuned with approximately [171,000 conversation samples from this dataset](https://huggingface.co/datasets/sam-mosaic/vicuna_alpaca_hc3_chatml) and another [217,000 from this dataset](https://huggingface.co/datasets/sam-mosaic/hhrlhf_evol_chatml).
|
183 |
+
|
184 |
+
If you're interested in [training](https://www.mosaicml.com/training) and [deploying](https://www.mosaicml.com/inference) your own MPT or LLMs, [sign up](https://forms.mosaicml.com/demo?utm_source=huggingface&utm_medium=referral&utm_campaign=mpt-7b) for MosaicML platform.
|
185 |
+
|
186 |
+
This is running on a smaller, shared GPU, so it may take a few seconds to respond. If you want to run it on your own GPU, you can [download the model from HuggingFace](https://huggingface.co/mosaicml/mpt-7b-chat) and run it locally. Or [Duplicate the Space](https://huggingface.co/spaces/mosaicml/mpt-7b-chat?duplicate=true) to skip the queue and run in a private space.
|
187 |
+
"""
|
188 |
+
)
|
189 |
+
chatbot = gr.Chatbot().style(height=500)
|
190 |
+
with gr.Row():
|
191 |
+
with gr.Column():
|
192 |
+
msg = gr.Textbox(
|
193 |
+
label="Chat Message Box",
|
194 |
+
placeholder="Chat Message Box",
|
195 |
+
show_label=False,
|
196 |
+
).style(container=False)
|
197 |
+
with gr.Column():
|
198 |
+
with gr.Row():
|
199 |
+
submit = gr.Button("Submit")
|
200 |
+
stop = gr.Button("Stop")
|
201 |
+
clear = gr.Button("Clear")
|
202 |
+
with gr.Row():
|
203 |
+
with gr.Accordion("Advanced Options:", open=False):
|
204 |
+
with gr.Row():
|
205 |
+
with gr.Column():
|
206 |
+
with gr.Row():
|
207 |
+
temperature = gr.Slider(
|
208 |
+
label="Temperature",
|
209 |
+
value=0.1,
|
210 |
+
minimum=0.0,
|
211 |
+
maximum=1.0,
|
212 |
+
step=0.1,
|
213 |
+
interactive=True,
|
214 |
+
info="Higher values produce more diverse outputs",
|
215 |
+
)
|
216 |
+
with gr.Column():
|
217 |
+
with gr.Row():
|
218 |
+
top_p = gr.Slider(
|
219 |
+
label="Top-p (nucleus sampling)",
|
220 |
+
value=1.0,
|
221 |
+
minimum=0.0,
|
222 |
+
maximum=1,
|
223 |
+
step=0.01,
|
224 |
+
interactive=True,
|
225 |
+
info=(
|
226 |
+
"Sample from the smallest possible set of tokens whose cumulative probability "
|
227 |
+
"exceeds top_p. Set to 1 to disable and sample from all tokens."
|
228 |
+
),
|
229 |
+
)
|
230 |
+
with gr.Column():
|
231 |
+
with gr.Row():
|
232 |
+
top_k = gr.Slider(
|
233 |
+
label="Top-k",
|
234 |
+
value=0,
|
235 |
+
minimum=0.0,
|
236 |
+
maximum=200,
|
237 |
+
step=1,
|
238 |
+
interactive=True,
|
239 |
+
info="Sample from a shortlist of top-k tokens — 0 to disable and sample from all tokens.",
|
240 |
+
)
|
241 |
+
with gr.Column():
|
242 |
+
with gr.Row():
|
243 |
+
repetition_penalty = gr.Slider(
|
244 |
+
label="Repetition Penalty",
|
245 |
+
value=1.1,
|
246 |
+
minimum=1.0,
|
247 |
+
maximum=2.0,
|
248 |
+
step=0.1,
|
249 |
+
interactive=True,
|
250 |
+
info="Penalize repetition — 1.0 to disable.",
|
251 |
+
)
|
252 |
+
with gr.Row():
|
253 |
+
gr.Markdown(
|
254 |
+
"Disclaimer: MPT-7B can produce factually incorrect output, and should not be relied on to produce "
|
255 |
+
"factually accurate information. MPT-7B was trained on various public datasets; while great efforts "
|
256 |
+
"have been taken to clean the pretraining data, it is possible that this model could generate lewd, "
|
257 |
+
"biased, or otherwise offensive outputs.",
|
258 |
+
elem_classes=["disclaimer"],
|
259 |
+
)
|
260 |
+
with gr.Row():
|
261 |
+
gr.Markdown(
|
262 |
+
"[Privacy policy](https://gist.github.com/samhavens/c29c68cdcd420a9aa0202d0839876dac)",
|
263 |
+
elem_classes=["disclaimer"],
|
264 |
+
)
|
265 |
+
|
266 |
+
submit_event = msg.submit(
|
267 |
+
fn=user,
|
268 |
+
inputs=[msg, chatbot],
|
269 |
+
outputs=[msg, chatbot],
|
270 |
+
queue=False,
|
271 |
+
).then(
|
272 |
+
fn=bot,
|
273 |
+
inputs=[
|
274 |
+
chatbot,
|
275 |
+
temperature,
|
276 |
+
top_p,
|
277 |
+
top_k,
|
278 |
+
repetition_penalty,
|
279 |
+
conversation_id,
|
280 |
+
],
|
281 |
+
outputs=chatbot,
|
282 |
+
queue=True,
|
283 |
+
)
|
284 |
+
submit_click_event = submit.click(
|
285 |
+
fn=user,
|
286 |
+
inputs=[msg, chatbot],
|
287 |
+
outputs=[msg, chatbot],
|
288 |
+
queue=False,
|
289 |
+
).then(
|
290 |
+
fn=bot,
|
291 |
+
inputs=[
|
292 |
+
chatbot,
|
293 |
+
temperature,
|
294 |
+
top_p,
|
295 |
+
top_k,
|
296 |
+
repetition_penalty,
|
297 |
+
conversation_id,
|
298 |
+
],
|
299 |
+
outputs=chatbot,
|
300 |
+
queue=True,
|
301 |
+
)
|
302 |
+
stop.click(
|
303 |
+
fn=None,
|
304 |
+
inputs=None,
|
305 |
+
outputs=None,
|
306 |
+
cancels=[submit_event, submit_click_event],
|
307 |
+
queue=False,
|
308 |
+
)
|
309 |
+
clear.click(lambda: None, None, chatbot, queue=False)
|
310 |
+
|
311 |
+
demo.queue(max_size=128, concurrency_count=2)
|
312 |
+
demo.launch()
|
requirements.txt
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
einops
|
2 |
+
gradio
|
3 |
+
torch
|
4 |
+
transformers
|
5 |
+
numpy
|
6 |
+
sentencepiece
|
7 |
+
# triton==2.0.0.dev20221202
|
8 |
+
# -e git+https://github.com/samhavens/just-triton-flash.git#egg=flash_attn
|
9 |
+
# RuntimeError: Triton requires CUDA 11.4+
|