Younes Belkada sam-mosaic commited on
Commit
73b7f0f
0 Parent(s):

Duplicate from mosaicml/mpt-7b-chat

Browse files

Co-authored-by: Sam <sam-mosaic@users.noreply.huggingface.co>

Files changed (5) hide show
  1. .gitattributes +34 -0
  2. .gitignore +4 -0
  3. README.md +13 -0
  4. app.py +312 -0
  5. requirements.txt +9 -0
.gitattributes ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tflite filter=lfs diff=lfs merge=lfs -text
29
+ *.tgz filter=lfs diff=lfs merge=lfs -text
30
+ *.wasm filter=lfs diff=lfs merge=lfs -text
31
+ *.xz filter=lfs diff=lfs merge=lfs -text
32
+ *.zip filter=lfs diff=lfs merge=lfs -text
33
+ *.zst filter=lfs diff=lfs merge=lfs -text
34
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ venv/
2
+ .venv/
3
+ env/
4
+ .env/
README.md ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: MPT-7B-Chat
3
+ emoji: 🤖
4
+ colorFrom: purple
5
+ colorTo: yellow
6
+ sdk: gradio
7
+ sdk_version: 3.27.0
8
+ app_file: app.py
9
+ pinned: false
10
+ duplicated_from: mosaicml/mpt-7b-chat
11
+ ---
12
+
13
+ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py ADDED
@@ -0,0 +1,312 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2023 MosaicML spaces authors
2
+ # SPDX-License-Identifier: Apache-2.0
3
+ import datetime
4
+ import os
5
+ from threading import Event, Thread
6
+ from uuid import uuid4
7
+
8
+ import gradio as gr
9
+ import requests
10
+ import torch
11
+ from transformers import (
12
+ AutoModelForCausalLM,
13
+ AutoTokenizer,
14
+ StoppingCriteria,
15
+ StoppingCriteriaList,
16
+ TextIteratorStreamer,
17
+ )
18
+
19
+
20
+ model_name = "mosaicml/mpt-7b-chat"
21
+ max_new_tokens = 1536
22
+
23
+ # # small testing model:
24
+ # model_name = "gpt2"
25
+ # max_new_tokens = 128
26
+
27
+ auth_token = os.getenv("HF_TOKEN", None)
28
+
29
+ print(f"Starting to load the model {model_name} into memory")
30
+
31
+ m = AutoModelForCausalLM.from_pretrained(
32
+ model_name,
33
+ torch_dtype=torch.bfloat16,
34
+ trust_remote_code=True,
35
+ use_auth_token=auth_token,
36
+ max_seq_len=8192,
37
+ ).cuda()
38
+ tok = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True, use_auth_token=auth_token)
39
+
40
+ stop_token_ids = tok.convert_tokens_to_ids(["<|im_end|>", "<|endoftext|>"])
41
+
42
+ print(f"Successfully loaded the model {model_name} into memory")
43
+
44
+
45
+ start_message = """<|im_start|>system
46
+ - You are a helpful assistant chatbot trained by MosaicML.
47
+ - You answer questions.
48
+ - You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.
49
+ - You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>
50
+ """
51
+
52
+
53
+ class StopOnTokens(StoppingCriteria):
54
+ def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
55
+ for stop_id in stop_token_ids:
56
+ if input_ids[0][-1] == stop_id:
57
+ return True
58
+ return False
59
+
60
+
61
+ def convert_history_to_text(history):
62
+ text = start_message + "".join(
63
+ [
64
+ "".join(
65
+ [
66
+ f"<|im_start|>user\n{item[0]}<|im_end|>",
67
+ f"<|im_start|>assistant\n{item[1]}<|im_end|>",
68
+ ]
69
+ )
70
+ for item in history[:-1]
71
+ ]
72
+ )
73
+ text += "".join(
74
+ [
75
+ "".join(
76
+ [
77
+ f"<|im_start|>user\n{history[-1][0]}<|im_end|>",
78
+ f"<|im_start|>assistant\n{history[-1][1]}",
79
+ ]
80
+ )
81
+ ]
82
+ )
83
+ return text
84
+
85
+
86
+ def log_conversation(conversation_id, history, messages, generate_kwargs):
87
+ logging_url = os.getenv("LOGGING_URL", None)
88
+ if logging_url is None:
89
+ return
90
+
91
+ timestamp = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
92
+
93
+ data = {
94
+ "conversation_id": conversation_id,
95
+ "timestamp": timestamp,
96
+ "history": history,
97
+ "messages": messages,
98
+ "generate_kwargs": generate_kwargs,
99
+ }
100
+
101
+ try:
102
+ requests.post(logging_url, json=data)
103
+ except requests.exceptions.RequestException as e:
104
+ print(f"Error logging conversation: {e}")
105
+
106
+
107
+ def user(message, history):
108
+ # Append the user's message to the conversation history
109
+ return "", history + [[message, ""]]
110
+
111
+
112
+ def bot(history, temperature, top_p, top_k, repetition_penalty, conversation_id):
113
+ print(f"history: {history}")
114
+ # Initialize a StopOnTokens object
115
+ stop = StopOnTokens()
116
+
117
+ # Construct the input message string for the model by concatenating the current system message and conversation history
118
+ messages = convert_history_to_text(history)
119
+
120
+ # Tokenize the messages string
121
+ input_ids = tok(messages, return_tensors="pt").input_ids
122
+ input_ids = input_ids.to(m.device)
123
+ streamer = TextIteratorStreamer(tok, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
124
+ generate_kwargs = dict(
125
+ input_ids=input_ids,
126
+ max_new_tokens=max_new_tokens,
127
+ temperature=temperature,
128
+ do_sample=temperature > 0.0,
129
+ top_p=top_p,
130
+ top_k=top_k,
131
+ repetition_penalty=repetition_penalty,
132
+ streamer=streamer,
133
+ stopping_criteria=StoppingCriteriaList([stop]),
134
+ )
135
+
136
+ stream_complete = Event()
137
+
138
+ def generate_and_signal_complete():
139
+ m.generate(**generate_kwargs)
140
+ stream_complete.set()
141
+
142
+ def log_after_stream_complete():
143
+ stream_complete.wait()
144
+ log_conversation(
145
+ conversation_id,
146
+ history,
147
+ messages,
148
+ {
149
+ "top_k": top_k,
150
+ "top_p": top_p,
151
+ "temperature": temperature,
152
+ "repetition_penalty": repetition_penalty,
153
+ },
154
+ )
155
+
156
+ t1 = Thread(target=generate_and_signal_complete)
157
+ t1.start()
158
+
159
+ t2 = Thread(target=log_after_stream_complete)
160
+ t2.start()
161
+
162
+ # Initialize an empty string to store the generated text
163
+ partial_text = ""
164
+ for new_text in streamer:
165
+ partial_text += new_text
166
+ history[-1][1] = partial_text
167
+ yield history
168
+
169
+
170
+ def get_uuid():
171
+ return str(uuid4())
172
+
173
+
174
+ with gr.Blocks(
175
+ theme=gr.themes.Soft(),
176
+ css=".disclaimer {font-variant-caps: all-small-caps;}",
177
+ ) as demo:
178
+ conversation_id = gr.State(get_uuid)
179
+ gr.Markdown(
180
+ """<h1><center>MosaicML MPT-7B-Chat</center></h1>
181
+
182
+ This demo is of [MPT-7B-Chat](https://huggingface.co/mosaicml/mpt-7b-chat). It is based on [MPT-7B](https://huggingface.co/mosaicml/mpt-7b) fine-tuned with approximately [171,000 conversation samples from this dataset](https://huggingface.co/datasets/sam-mosaic/vicuna_alpaca_hc3_chatml) and another [217,000 from this dataset](https://huggingface.co/datasets/sam-mosaic/hhrlhf_evol_chatml).
183
+
184
+ If you're interested in [training](https://www.mosaicml.com/training) and [deploying](https://www.mosaicml.com/inference) your own MPT or LLMs, [sign up](https://forms.mosaicml.com/demo?utm_source=huggingface&utm_medium=referral&utm_campaign=mpt-7b) for MosaicML platform.
185
+
186
+ This is running on a smaller, shared GPU, so it may take a few seconds to respond. If you want to run it on your own GPU, you can [download the model from HuggingFace](https://huggingface.co/mosaicml/mpt-7b-chat) and run it locally. Or [Duplicate the Space](https://huggingface.co/spaces/mosaicml/mpt-7b-chat?duplicate=true) to skip the queue and run in a private space.
187
+ """
188
+ )
189
+ chatbot = gr.Chatbot().style(height=500)
190
+ with gr.Row():
191
+ with gr.Column():
192
+ msg = gr.Textbox(
193
+ label="Chat Message Box",
194
+ placeholder="Chat Message Box",
195
+ show_label=False,
196
+ ).style(container=False)
197
+ with gr.Column():
198
+ with gr.Row():
199
+ submit = gr.Button("Submit")
200
+ stop = gr.Button("Stop")
201
+ clear = gr.Button("Clear")
202
+ with gr.Row():
203
+ with gr.Accordion("Advanced Options:", open=False):
204
+ with gr.Row():
205
+ with gr.Column():
206
+ with gr.Row():
207
+ temperature = gr.Slider(
208
+ label="Temperature",
209
+ value=0.1,
210
+ minimum=0.0,
211
+ maximum=1.0,
212
+ step=0.1,
213
+ interactive=True,
214
+ info="Higher values produce more diverse outputs",
215
+ )
216
+ with gr.Column():
217
+ with gr.Row():
218
+ top_p = gr.Slider(
219
+ label="Top-p (nucleus sampling)",
220
+ value=1.0,
221
+ minimum=0.0,
222
+ maximum=1,
223
+ step=0.01,
224
+ interactive=True,
225
+ info=(
226
+ "Sample from the smallest possible set of tokens whose cumulative probability "
227
+ "exceeds top_p. Set to 1 to disable and sample from all tokens."
228
+ ),
229
+ )
230
+ with gr.Column():
231
+ with gr.Row():
232
+ top_k = gr.Slider(
233
+ label="Top-k",
234
+ value=0,
235
+ minimum=0.0,
236
+ maximum=200,
237
+ step=1,
238
+ interactive=True,
239
+ info="Sample from a shortlist of top-k tokens — 0 to disable and sample from all tokens.",
240
+ )
241
+ with gr.Column():
242
+ with gr.Row():
243
+ repetition_penalty = gr.Slider(
244
+ label="Repetition Penalty",
245
+ value=1.1,
246
+ minimum=1.0,
247
+ maximum=2.0,
248
+ step=0.1,
249
+ interactive=True,
250
+ info="Penalize repetition — 1.0 to disable.",
251
+ )
252
+ with gr.Row():
253
+ gr.Markdown(
254
+ "Disclaimer: MPT-7B can produce factually incorrect output, and should not be relied on to produce "
255
+ "factually accurate information. MPT-7B was trained on various public datasets; while great efforts "
256
+ "have been taken to clean the pretraining data, it is possible that this model could generate lewd, "
257
+ "biased, or otherwise offensive outputs.",
258
+ elem_classes=["disclaimer"],
259
+ )
260
+ with gr.Row():
261
+ gr.Markdown(
262
+ "[Privacy policy](https://gist.github.com/samhavens/c29c68cdcd420a9aa0202d0839876dac)",
263
+ elem_classes=["disclaimer"],
264
+ )
265
+
266
+ submit_event = msg.submit(
267
+ fn=user,
268
+ inputs=[msg, chatbot],
269
+ outputs=[msg, chatbot],
270
+ queue=False,
271
+ ).then(
272
+ fn=bot,
273
+ inputs=[
274
+ chatbot,
275
+ temperature,
276
+ top_p,
277
+ top_k,
278
+ repetition_penalty,
279
+ conversation_id,
280
+ ],
281
+ outputs=chatbot,
282
+ queue=True,
283
+ )
284
+ submit_click_event = submit.click(
285
+ fn=user,
286
+ inputs=[msg, chatbot],
287
+ outputs=[msg, chatbot],
288
+ queue=False,
289
+ ).then(
290
+ fn=bot,
291
+ inputs=[
292
+ chatbot,
293
+ temperature,
294
+ top_p,
295
+ top_k,
296
+ repetition_penalty,
297
+ conversation_id,
298
+ ],
299
+ outputs=chatbot,
300
+ queue=True,
301
+ )
302
+ stop.click(
303
+ fn=None,
304
+ inputs=None,
305
+ outputs=None,
306
+ cancels=[submit_event, submit_click_event],
307
+ queue=False,
308
+ )
309
+ clear.click(lambda: None, None, chatbot, queue=False)
310
+
311
+ demo.queue(max_size=128, concurrency_count=2)
312
+ demo.launch()
requirements.txt ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ einops
2
+ gradio
3
+ torch
4
+ transformers
5
+ numpy
6
+ sentencepiece
7
+ # triton==2.0.0.dev20221202
8
+ # -e git+https://github.com/samhavens/just-triton-flash.git#egg=flash_attn
9
+ # RuntimeError: Triton requires CUDA 11.4+