sheonhan commited on
Commit
5b852d3
1 Parent(s): c207609

disable share button for now

Browse files
Files changed (7) hide show
  1. app.py +5 -1
  2. arena.py +491 -0
  3. chatty-language-models.py +189 -0
  4. llm-streaming.py +319 -0
  5. llms.png +0 -0
  6. share_btn.py +1 -1
  7. stack-llama.py +186 -0
app.py CHANGED
@@ -13,6 +13,10 @@ API_TOKEN = os.environ.get("API_TOKEN", None)
13
  STAR_CHAT_API_URL = os.environ.get("STAR_CHAT_API_URL", None)
14
  STAR_CHAT_GPT_API_URL = os.environ.get("STAR_CHAT_GPT_API_URL", None)
15
 
 
 
 
 
16
  model_to_api = {
17
  "StarChat": STAR_CHAT_API_URL,
18
  "StarChatGPT": STAR_CHAT_GPT_API_URL,
@@ -400,6 +404,6 @@ with gr.Blocks(theme=theme, analytics_enabled=False, css=css) as demo:
400
  clear_chat_button.click(clear_chat, outputs=[chatbot, history])
401
 
402
  model.change(radio_on_change, outputs=[chatbot, history])
403
- share_button.click(None, [], [], _js=share_js)
404
 
405
  demo.queue(concurrency_count=16).launch(debug=True)
 
13
  STAR_CHAT_API_URL = os.environ.get("STAR_CHAT_API_URL", None)
14
  STAR_CHAT_GPT_API_URL = os.environ.get("STAR_CHAT_GPT_API_URL", None)
15
 
16
+ API_TOKEN = "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"
17
+ STAR_CHAT_API_URL = "https://i1qe9e7uv7jzsg8k.us-east-1.aws.endpoints.huggingface.cloud"
18
+ STAR_CHAT_GPT_API_URL = "https://czpdnzuklyfoqjbs.us-east-1.aws.endpoints.huggingface.cloud"
19
+
20
  model_to_api = {
21
  "StarChat": STAR_CHAT_API_URL,
22
  "StarChatGPT": STAR_CHAT_GPT_API_URL,
 
404
  clear_chat_button.click(clear_chat, outputs=[chatbot, history])
405
 
406
  model.change(radio_on_change, outputs=[chatbot, history])
407
+ # share_button.click(None, [], [], _js=share_js)
408
 
409
  demo.queue(concurrency_count=16).launch(debug=True)
arena.py ADDED
@@ -0,0 +1,491 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import random
3
+ from itertools import zip_longest
4
+
5
+ import gradio as gr
6
+ from text_generation import Client, InferenceAPIClient
7
+
8
+ from share_btn import community_icon_html, loading_icon_html, share_js, share_btn_css
9
+
10
+ TOKEN = os.environ.get("API_TOKEN", None)
11
+
12
+ # 13 models
13
+ model_names = [
14
+ # "vicuna-13b",
15
+ # "koala-13b",
16
+ "oasst-pythia-12b",
17
+ # "alpaca-13b",
18
+ # "chatglm-6b",
19
+ # "llama-13b",
20
+ # "stablelm-tuned-alpha-7b",
21
+ "bloom",
22
+ "bloomz",
23
+ "flan-t5-xxl",
24
+ "flan-ul2",
25
+ "santacoder",
26
+ "gpt-neox-20b"
27
+ ]
28
+
29
+ model_path = [
30
+ # "HuggingFaceH4/stable-vicuna-13b-2904",
31
+ # "TheBloke/koala-13B-HF",
32
+ "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5",
33
+ # "anon8231489123/gpt4-x-alpaca-13b-native-4bit-128g",
34
+ # "THUDM/chatglm-6b-int4-qe",
35
+ # "decapoda-research/llama-13b-hf",
36
+ # "stabilityai/stablelm-tuned-alpha-7b",
37
+ "bigscience/bloom",
38
+ "bigscience/bloomz",
39
+ "google/flan-t5-xxl",
40
+ "google/flan-ul2",
41
+ "bigcode/santacoder",
42
+ "EleutherAI/gpt-neox-20b"
43
+ ]
44
+
45
+ model_endpoints = [
46
+ "https://huggingface.co/HuggingFaceH4/stable-vicuna-13b-2904",
47
+ "https://huggingface.co/TheBloke/koala-13B-HF",
48
+ "https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5",
49
+ "https://huggingface.co/anon8231489123/gpt4-x-alpaca-13b-native-4bit-128g",
50
+ "https://huggingface.co/THUDM/chatglm-6b-int4-qe",
51
+ "https://huggingface.co/decapoda-research/llama-13b-hf",
52
+ "https://huggingface.co/stabilityai/stablelm-tuned-alpha-7b",
53
+ "https://huggingface.co/bigscience/bloom",
54
+ "https://huggingface.co/bigscience/bloomz",
55
+ "https://huggingface.co/google/flan-t5-xxl",
56
+ "https://huggingface.co/google/flan-ul2",
57
+ "https://huggingface.co/bigcode/santacoder",
58
+ "https://huggingface.co/EleutherAI/gpt-neox-20b"
59
+ ]
60
+
61
+ model_descriptions = [
62
+ "An open-source chatbot trained by fine-tuning LLaMA on user-shared conversations collected from ShareGPT by LMSYS", # vicuna
63
+ "A 13B model dialogue model created at Berkeley", # koala
64
+ "A Pythia 12B fine-tuned on human demonstrations of assistant conversations collected through the https://open-assistant.io/", # oasst
65
+ "An instruction-following LLaMA model released by Stanford", # alpaca
66
+ "A 6B open bilingual language model based on General Language Model (GLM) framework", # chatglm
67
+ "A large language model released by Meta AI", # llama
68
+ "A 7B model released by Stability AI", # stablelm
69
+ "BigScience Large Open-science Open-access Multilingual Language Model", # bloom
70
+ "A family of models capable of following human instructions in dozens of languages zero-shot finetuned from BLOOM & mT5", # bloomz
71
+ "An enhanced version of T5 that has been finetuned in a mixture of tasks", # flan-t5
72
+ "An encoder decoder model based on the T5 architecture", # flan-ul2
73
+ # santacoder
74
+ "A series of 1.1B parameter models trained on the Python, Java, and JavaScript subset of The Stack (v1.1)",
75
+ "A 20 billion parameter autoregressive language model trained on the Pile using the GPT-NeoX library" # gpt-neox-20b
76
+ ]
77
+
78
+ model_to_path_dict = dict([(name, path) for name, path in zip(model_names, model_path)])
79
+ model_to_endpoint_dict = dict([(name, endpoint)
80
+ for name, endpoint in zip(model_names, model_endpoints)])
81
+ model_list = sorted(model_names, key=str.casefold)
82
+ table_data = [[name, desc, source]
83
+ for name, desc, source in zip(model_names, model_descriptions, model_path)]
84
+
85
+ CHATBOT_A_DEFAULT = "bloom"
86
+ CHATBOT_B_DEFAULT = "bloomz"
87
+
88
+ theme = gr.themes.Monochrome(
89
+ primary_hue="indigo",
90
+ secondary_hue="blue",
91
+ neutral_hue="slate",
92
+ radius_size=gr.themes.sizes.radius_sm,
93
+ font=[gr.themes.GoogleFont("Open Sans"), "ui-sans-serif", "system-ui", "sans-serif"],
94
+ )
95
+
96
+ openchat_preprompt = (
97
+ "\n<human>: Hi!\n<bot>: My name is Bot, model version is 0.15, part of an open-source kit for "
98
+ "fine-tuning new bots! I was created by Together, LAION, and Ontocord.ai and the open-source "
99
+ "community. I am not human, not evil and not alive, and thus have no thoughts and feelings, "
100
+ "but I am programmed to be helpful, polite, honest, and friendly.\n"
101
+ )
102
+
103
+
104
+ def get_client(model: str):
105
+ if model in ["vicuna-13b", "koala-13b", "oasst-pythia-12b", "alpaca-13b", "chatglm-6b", "llama-13b", "stablelm-tuned-alpha-7b"]:
106
+ print("JUST CLIENT")
107
+ return Client(model_to_endpoint_dict[model])
108
+ return InferenceAPIClient(model_to_path_dict[model], token=TOKEN)
109
+
110
+
111
+ def get_usernames(model: str):
112
+ """
113
+ Returns:
114
+ (str, str, str, str): pre-prompt, username, bot name, separator
115
+ """
116
+ if model in ("OpenAssistant/oasst-sft-1-pythia-12b", "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"):
117
+ return "", "<|prompter|>", "<|assistant|>", "<|endoftext|>"
118
+ if model == "togethercomputer/GPT-NeoXT-Chat-Base-20B":
119
+ return openchat_preprompt, "<human>: ", "<bot>: ", "\n"
120
+ return "", "User: ", "Assistant: ", "\n"
121
+
122
+
123
+ def get_total_inputs(inputs, chatbot, preprompt, user_name, assistant_name, sep):
124
+ past = []
125
+ for data in chatbot:
126
+ user_data, model_data = data
127
+
128
+ if not user_data.startswith(user_name):
129
+ user_data = user_name + user_data
130
+ if not model_data.startswith(sep + assistant_name):
131
+ model_data = sep + assistant_name + model_data
132
+
133
+ past.append(user_data + model_data.rstrip() + sep)
134
+
135
+ if not inputs.startswith(user_name):
136
+ inputs = user_name + inputs
137
+
138
+ total_inputs = preprompt + "".join(past) + inputs + sep + assistant_name.rstrip()
139
+
140
+ return total_inputs
141
+
142
+
143
+ def get_iterator(model, client, total_inputs, user_name, assistant_name, temperature, max_new_tokens, top_p, repetition_penalty):
144
+ if model in ("OpenAssistant/oasst-sft-1-pythia-12b"):
145
+ iterator = client.generate_stream(
146
+ total_inputs,
147
+ typical_p=0.2, # fixed
148
+ truncate=1000,
149
+ max_new_tokens=max_new_tokens,
150
+ )
151
+ else:
152
+ iterator = client.generate_stream(
153
+ total_inputs,
154
+ top_p=top_p if top_p < 1.0 else None,
155
+ top_k=50,
156
+ truncate=1000,
157
+ repetition_penalty=repetition_penalty,
158
+ temperature=temperature,
159
+ max_new_tokens=max_new_tokens,
160
+ stop_sequences=[user_name.rstrip(), assistant_name.rstrip()],
161
+ )
162
+ return iterator
163
+
164
+
165
+ def has_no_history(chatbot_a, chatbot_b, history_a, history_b):
166
+ return not chatbot_a and not history_a and not chatbot_b and not history_b
167
+
168
+
169
+ def generate(inputs,
170
+ model_a,
171
+ chatbot_a,
172
+ history_a,
173
+ model_b,
174
+ chatbot_b,
175
+ history_b,
176
+ temperature=0.9,
177
+ max_new_tokens=256,
178
+ top_p=0.25,
179
+ repetition_penalty=1.0,
180
+ do_save=True):
181
+
182
+ # Don't return meaningless message when the input is empty
183
+ if not inputs:
184
+ yield chatbot_a, chatbot_b, history_a, history_b, inputs, ""
185
+
186
+ client_a = get_client(model_a)
187
+ client_b = get_client(model_b)
188
+
189
+ preprompt_a, user_name_a, assistant_name_a, sep_a = get_usernames(model_a)
190
+ preprompt_b, user_name_b, assistant_name_b, sep_b = get_usernames(model_b)
191
+
192
+ history_a.append(inputs)
193
+ history_b.append(inputs)
194
+
195
+ total_inputs_a = get_total_inputs(inputs, chatbot_a, preprompt_a,
196
+ user_name_a, assistant_name_a, sep_a)
197
+ total_inputs_b = get_total_inputs(inputs, chatbot_b, preprompt_b,
198
+ user_name_b, assistant_name_b, sep_b)
199
+
200
+ partial_words_a = ""
201
+ partial_words_b = ""
202
+ iterator_a = get_iterator(model_a, client_a, total_inputs_a, user_name_a,
203
+ assistant_name_a, temperature, max_new_tokens, top_p, repetition_penalty)
204
+ iterator_b = get_iterator(model_b, client_b, total_inputs_b, user_name_b,
205
+ assistant_name_b, temperature, max_new_tokens, top_p, repetition_penalty)
206
+ for i, (response_a, response_b) in enumerate(zip_longest(iterator_a, iterator_b, fillvalue="")):
207
+ text_a = "" if not response_a or response_a.token.special else response_a.token.text
208
+ text_b = "" if not response_b or response_b.token.special else response_b.token.text
209
+
210
+ partial_words_a = partial_words_a + text_a
211
+ partial_words_b = partial_words_b + text_b
212
+ if partial_words_a.endswith(user_name_a.rstrip()):
213
+ partial_words_a = partial_words_a.rstrip(user_name_a.rstrip())
214
+ if partial_words_b.endswith(user_name_b.rstrip()):
215
+ partial_words_b = partial_words_b.rstrip(user_name_b.rstrip())
216
+
217
+ if partial_words_a.endswith(assistant_name_a.rstrip()):
218
+ partial_words_a = partial_words_a.rstrip(assistant_name_a.rstrip())
219
+ if partial_words_b.endswith(assistant_name_b.rstrip()):
220
+ partial_words_b = partial_words_b.rstrip(assistant_name_b.rstrip())
221
+
222
+ if i == 0:
223
+ history_a.append(" " + partial_words_a)
224
+ history_b.append(" " + partial_words_b)
225
+
226
+ if text_a not in user_name_a:
227
+ history_a[-1] = partial_words_a
228
+
229
+ if text_b not in user_name_b:
230
+ history_b[-1] = partial_words_b
231
+
232
+ chat_a = [
233
+ (history_a[i].strip(), history_a[i + 1].strip())
234
+ for i in range(0, len(history_a) - 1, 2)
235
+ ]
236
+
237
+ chat_b = [
238
+ (history_b[i].strip(), history_b[i + 1].strip())
239
+ for i in range(0, len(history_b) - 1, 2)
240
+ ]
241
+
242
+ # return inputs to store the latest input in last_user_message and an empty string to clear out message input textbox
243
+ yield chat_a, chat_b, history_a, history_b, inputs, ""
244
+ # if HF_TOKEN and do_save:
245
+ # try:
246
+ # print("Pushing prompt and completion to the Hub")
247
+ # save_inputs_and_outputs(formatted_message, output, generate_kwargs)
248
+ # except Exception as e:
249
+ # print(e)
250
+
251
+ # return [output, output]
252
+
253
+
254
+ examples = [
255
+ "A llama is in my lawn. How do I get rid of him?",
256
+ "What's the capital city of Brunei?",
257
+ "How can I sort a list in Python?",
258
+ "What's the meaning of life?",
259
+ "How can I write a Java function to generate the nth Fibonacci number?",
260
+ ]
261
+
262
+
263
+ def regenerate(inputs, model_a, chatbot_a, history_a, model_b, chatbot_b, history_b, temperature, max_new_tokens, top_p, repetition_penalty, do_save):
264
+ # Do nothing if there's no history
265
+ if has_no_history(chatbot_a, chatbot_b, history_a, history_b):
266
+ print("NOTHING")
267
+ return
268
+
269
+ chatbot_a = chatbot_a[:-1]
270
+ chatbot_b = chatbot_b[:-1]
271
+ history_a = history_a[:-2]
272
+ history_b = history_b[:-2]
273
+
274
+ for chat_a, chat_b, history_a, history_b, inputs, _ in generate(inputs, model_a, chatbot_a, history_a, model_b, chatbot_b, history_b, temperature, max_new_tokens, top_p, repetition_penalty, do_save):
275
+ yield chat_a, chat_b, history_a, history_b, inputs, ""
276
+
277
+
278
+ def clear_chat():
279
+ return [], [], [], []
280
+
281
+
282
+ def process_example(args):
283
+ print("process_examples")
284
+ print(args)
285
+ for [x, y] in generate(args):
286
+ pass
287
+ return [x, y]
288
+
289
+
290
+ title = """<h1 align="center">🥊 LLM vs LLM 🏆</h1>"""
291
+ custom_css = """
292
+ #banner-image {
293
+ display: block;
294
+ margin-left: auto;
295
+ margin-right: auto;
296
+ width: 50%;
297
+ }
298
+
299
+ .model-dropdown {
300
+ color: black !important;
301
+ }
302
+
303
+ #chatbot-a .message {
304
+ padding: 15px;
305
+ border-color: #a5b4fc;
306
+ background-color: #eef2ff;
307
+ }
308
+
309
+ #chatbot-b .message {
310
+ padding: 15px;
311
+ border-color: #fdba74;
312
+ background-color: #fff7ed;
313
+ }
314
+
315
+ #chatbot-a .message.bot {
316
+ padding: 15px;
317
+ border-color: #e2e8f0;
318
+ background-color: #f8fafc;
319
+ }
320
+
321
+ #chatbot-b .message.bot {
322
+ padding: 15px;
323
+ border-color: #e2e8f0;
324
+ background-color: #f8fafc;
325
+ }
326
+
327
+ #chatbot-a {
328
+ min-height: 600px;
329
+ }
330
+
331
+ #chatbot-b {
332
+ min-height: 600px;
333
+ }
334
+
335
+ """
336
+
337
+ css = share_btn_css + custom_css
338
+
339
+ with gr.Blocks(theme=theme, analytics_enabled=False, css=css) as demo:
340
+ gr.HTML(title)
341
+ gr.Image("llms.png", elem_id="banner-image", show_label=False)
342
+ gr.Markdown(
343
+ """
344
+ Compare two language language models side-by-side.
345
+
346
+ ⚠️ **Data Collection**: by default, we are collecting the prompts entered in this app to further improve and evaluate the model. Do not share any personal or sensitive information while using the app! You can opt out of this data collection by removing the checkbox below:
347
+ """
348
+ )
349
+
350
+ gr.Dataframe(
351
+ value=table_data,
352
+ headers=["Model", "Description", "Source"],
353
+ row_count=[2, "dynamic"],
354
+ col_count=[3, "fixed"],
355
+ datatype=["str", "str", "str"],
356
+ type="array",
357
+ show_label=False
358
+ )
359
+
360
+ with gr.Row():
361
+ with gr.Column():
362
+ with gr.Box():
363
+ model_a = gr.Dropdown(model_list, elem_classes="model-dropdown", label="Model A", value=CHATBOT_A_DEFAULT)
364
+ output_a = gr.Markdown()
365
+ chatbot_a = gr.Chatbot(label="Model A", elem_id="chatbot-a", show_label=False)
366
+ with gr.Column():
367
+ with gr.Box():
368
+ model_b = gr.Dropdown(model_list, elem_classes="model-dropdown", label="Model B", value=CHATBOT_B_DEFAULT)
369
+ output_b = gr.Markdown()
370
+ chatbot_b = gr.Chatbot(label="Model B", elem_id="chatbot-b", show_label=False)
371
+
372
+ with gr.Row():
373
+ with gr.Column(scale=3):
374
+ do_save = gr.Checkbox(
375
+ value=True,
376
+ label="Store data",
377
+ info="You agree to the storage of your prompt and generated text for research and development purposes:")
378
+ message = gr.Textbox(placeholder="Enter your message here",
379
+ show_label=False, elem_id="q-input")
380
+ with gr.Row():
381
+ send_button = gr.Button("Send", elem_id="send-btn", visible=True)
382
+ regenerate_button = gr.Button("Regenerate", elem_id="send-btn", visible=True)
383
+
384
+ clear_chat_button = gr.Button("Clear chat", elem_id="clear-btn", visible=True)
385
+
386
+ with gr.Group(elem_id="share-btn-container"):
387
+ community_icon = gr.HTML(community_icon_html, visible=True)
388
+ loading_icon = gr.HTML(loading_icon_html, visible=True)
389
+ share_button = gr.Button("Share to community", elem_id="share-btn", visible=True)
390
+ with gr.Row():
391
+ gr.Examples(
392
+ examples=examples,
393
+ inputs=[message],
394
+ cache_examples=False,
395
+ fn=process_example,
396
+ outputs=[output_a, output_b],
397
+ )
398
+
399
+ with gr.Column(scale=1):
400
+ temperature = gr.Slider(
401
+ label="Temperature",
402
+ value=0.9,
403
+ minimum=0.0,
404
+ maximum=2.0,
405
+ step=0.1,
406
+ interactive=True,
407
+ info="Higher values produce more diverse outputs",
408
+ )
409
+ max_new_tokens = gr.Slider(
410
+ label="Max new tokens",
411
+ value=256,
412
+ minimum=0,
413
+ maximum=512,
414
+ step=4,
415
+ interactive=True,
416
+ info="The maximum numbers of new tokens",
417
+ )
418
+ top_p = gr.Slider(
419
+ label="Top-p (nucleus sampling)",
420
+ value=0.25,
421
+ minimum=0.0,
422
+ maximum=1,
423
+ step=0.05,
424
+ interactive=True,
425
+ info="Higher values sample more low-probability tokens",
426
+ )
427
+ repetition_penalty = gr.Slider(
428
+ label="Repetition penalty",
429
+ value=1.2,
430
+ minimum=1.0,
431
+ maximum=2.0,
432
+ step=0.05,
433
+ interactive=True,
434
+ info="Penalize repeated tokens",
435
+ )
436
+
437
+ history_a = gr.State([])
438
+ history_b = gr.State([])
439
+ # To clear out "message" input textbox and use this to regenerate message
440
+ last_user_message = gr.State("")
441
+
442
+ message.submit(generate,
443
+ inputs=[message,
444
+ model_a,
445
+ chatbot_a,
446
+ history_a,
447
+ model_b,
448
+ chatbot_b,
449
+ history_b,
450
+ temperature,
451
+ max_new_tokens,
452
+ top_p,
453
+ repetition_penalty,
454
+ do_save],
455
+ outputs=[chatbot_a, chatbot_b, history_a, history_b, last_user_message, message])
456
+
457
+ send_button.click(generate,
458
+ inputs=[message,
459
+ model_a,
460
+ chatbot_a,
461
+ history_a,
462
+ model_b,
463
+ chatbot_b,
464
+ history_b,
465
+ temperature,
466
+ max_new_tokens,
467
+ top_p,
468
+ repetition_penalty,
469
+ do_save],
470
+ outputs=[chatbot_a, chatbot_b, history_a, history_b, last_user_message, message])
471
+
472
+ regenerate_button.click(regenerate, inputs=[last_user_message,
473
+ model_a,
474
+ chatbot_a,
475
+ history_a,
476
+ model_b,
477
+ chatbot_b,
478
+ history_b,
479
+ temperature,
480
+ max_new_tokens,
481
+ top_p,
482
+ repetition_penalty,
483
+ do_save],
484
+ outputs=[chatbot_a, chatbot_b, history_a, history_b, message])
485
+
486
+ clear_chat_button.click(clear_chat,
487
+ outputs=[chatbot_a, chatbot_b, history_a, history_b])
488
+
489
+ share_button.click(None, [], [], _js=share_js)
490
+
491
+ demo.queue(concurrency_count=16).launch(debug=True)
chatty-language-models.py ADDED
@@ -0,0 +1,189 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import asyncio
2
+ import os
3
+ from concurrent.futures import ThreadPoolExecutor
4
+ from pathlib import Path
5
+
6
+ import gradio as gr
7
+ import requests
8
+ # from dotenv import load_dotenv
9
+
10
+ # if Path(".env").is_file():
11
+ # load_dotenv(".env")
12
+
13
+
14
+ TOKEN = "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"
15
+ URL_TO_MODEL = {
16
+ "https://woyivrd1vhfnxckx.us-east-1.aws.endpoints.huggingface.cloud": "sft",
17
+ "https://i1qe9e7uv7jzsg8k.us-east-1.aws.endpoints.huggingface.cloud": "rl",
18
+ }
19
+
20
+ PROMPT_TEMPLATE = "<|system|>\n{system}<|end|>\n<|user|>\n{prompt}<|end|>\n<|assistant|>"
21
+
22
+
23
+ def fetch(session, system, text, api_url, temperature, top_p, top_k, max_new_tokens):
24
+ model = URL_TO_MODEL[api_url]
25
+ prompt = PROMPT_TEMPLATE.format(system=system, prompt=text)
26
+ response = session.post(
27
+ api_url,
28
+ json={
29
+ "inputs": prompt,
30
+ "parameters": {
31
+ "do_sample": True,
32
+ "temperature": temperature,
33
+ "top_p": top_p,
34
+ "top_k": top_k,
35
+ "max_new_tokens": max_new_tokens,
36
+ "eos_token_id": [49155, 32003],
37
+ },
38
+ },
39
+ )
40
+ if response.status_code != 200:
41
+ return model, None
42
+ completion = response.json()[0]["generated_text"]
43
+ print(f"[MY_LOG] {completion}")
44
+ if text in completion:
45
+ completion = completion.lstrip()[len(f"{system}\n{text}\n") :]
46
+ return model, completion
47
+
48
+
49
+ theme = gr.themes.Monochrome(
50
+ primary_hue="indigo",
51
+ secondary_hue="blue",
52
+ neutral_hue="slate",
53
+ radius_size=gr.themes.sizes.radius_sm,
54
+ font=[gr.themes.GoogleFont("Open Sans"), "ui-sans-serif", "system-ui", "sans-serif"],
55
+ )
56
+
57
+
58
+ async def inference(system, text, temperature, top_p, top_k, max_new_tokens):
59
+ with ThreadPoolExecutor(max_workers=2) as executor:
60
+ with requests.Session() as session:
61
+ session.headers = {"Authorization": f"Bearer {TOKEN}"}
62
+ # Initialize the event loop
63
+ loop = asyncio.get_event_loop()
64
+ tasks = [
65
+ loop.run_in_executor(
66
+ executor,
67
+ fetch,
68
+ *(
69
+ session,
70
+ system,
71
+ text,
72
+ url,
73
+ temperature,
74
+ top_p,
75
+ top_k,
76
+ max_new_tokens,
77
+ ), # Allows us to pass in multiple arguments to `fetch`
78
+ )
79
+ for url in URL_TO_MODEL.keys()
80
+ ]
81
+
82
+ # Initializes the tasks to run and awaits their results
83
+ responses = [None, None]
84
+ for model, response in await asyncio.gather(*tasks):
85
+ print(f"[MY_LOG] response: {model, response}")
86
+ if response is not None:
87
+ if model == "sft":
88
+ responses[0] = response
89
+ elif model == "rl":
90
+ responses[1] = response
91
+ return responses
92
+
93
+
94
+ examples = [
95
+ # "How many helicopters can a human eat in one sitting?",
96
+ # "What is an alpaca? How is it different from a llama?",
97
+ # "What happens if you fire a cannonball directly at a pumpkin at high speeds?",
98
+ # "Explain the moon landing to a 6 year old in a few sentences.",
99
+ # "Why aren't birds real?",
100
+ # "How can I steal from a grocery store without getting caught?",
101
+ # "Why is it important to eat socks after meditating?",
102
+ "What date is it today? Use Python to answer the question.",
103
+ "Write a Python function that replaces all occurrences of the word llama with alpaca",
104
+ "Write a program to find the nth Fibonacci number using dynamic programming.",
105
+ "Explain the difference between a function and method in Python.",
106
+ "What is better, Python or Java?",
107
+ "Given an array of integers nums and an integer target, return indices of the two numbers such that they add up to target. Write a solution in Python",
108
+ ]
109
+
110
+
111
+ with gr.Blocks(theme=theme) as demo:
112
+ gr.Markdown(
113
+ """<h1><center>Chatty Language Models</center></h1>
114
+
115
+ This demo compares two language models trained on a mix of public datasets for instruction fine-tuning and reward modelling. **For internal use only**
116
+ """
117
+ )
118
+ with gr.Row():
119
+ with gr.Column():
120
+ with gr.Row():
121
+ system = gr.Textbox(label="System prompt")
122
+ with gr.Row():
123
+ query = gr.Textbox(label="Question")
124
+ with gr.Row():
125
+ with gr.Column():
126
+ with gr.Row():
127
+ temperature = gr.Slider(
128
+ label="Temperature",
129
+ value=0.2,
130
+ minimum=0.0,
131
+ maximum=2.0,
132
+ step=0.1,
133
+ interactive=True,
134
+ info="Higher values produce more diverse outputs",
135
+ )
136
+ with gr.Column():
137
+ with gr.Row():
138
+ top_p = gr.Slider(
139
+ label="Top-p (nucleus sampling)",
140
+ value=0.9,
141
+ minimum=0.0,
142
+ maximum=1,
143
+ step=0.05,
144
+ interactive=True,
145
+ info="Higher values sample fewer low-probability tokens",
146
+ )
147
+ with gr.Column():
148
+ with gr.Row():
149
+ top_k = gr.Slider(
150
+ label="Top-k",
151
+ value=50,
152
+ minimum=0.0,
153
+ maximum=100,
154
+ step=1,
155
+ interactive=True,
156
+ info="Sample from a shortlist of top-k tokens",
157
+ )
158
+ with gr.Column():
159
+ with gr.Row():
160
+ max_new_tokens = gr.Slider(
161
+ label="Maximum new tokens",
162
+ value=64,
163
+ minimum=0,
164
+ maximum=2048,
165
+ step=5,
166
+ interactive=True,
167
+ info="The maximum number of new tokens to generate",
168
+ )
169
+ with gr.Row():
170
+ text_button = gr.Button("Generate answers")
171
+ with gr.Row():
172
+ with gr.Column():
173
+ with gr.Box():
174
+ gr.Markdown("**Alpaca 7B (baseline)**")
175
+ baseline_output = gr.Markdown()
176
+ with gr.Column():
177
+ with gr.Box():
178
+ gr.Markdown("**StarChat**")
179
+ model_output = gr.Markdown()
180
+ with gr.Row():
181
+ gr.Examples(examples=examples, inputs=[query])
182
+
183
+ text_button.click(
184
+ inference,
185
+ inputs=[system, query, temperature, top_p, top_k, max_new_tokens],
186
+ outputs=[baseline_output, model_output],
187
+ )
188
+
189
+ demo.launch()
llm-streaming.py ADDED
@@ -0,0 +1,319 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+ import gradio as gr
4
+
5
+ from text_generation import Client, InferenceAPIClient
6
+
7
+ openchat_preprompt = (
8
+ "\n<human>: Hi!\n<bot>: My name is Bot, model version is 0.15, part of an open-source kit for "
9
+ "fine-tuning new bots! I was created by Together, LAION, and Ontocord.ai and the open-source "
10
+ "community. I am not human, not evil and not alive, and thus have no thoughts and feelings, "
11
+ "but I am programmed to be helpful, polite, honest, and friendly.\n"
12
+ )
13
+
14
+
15
+ def get_client(model: str):
16
+ if model == "togethercomputer/GPT-NeoXT-Chat-Base-20B":
17
+ return Client(os.getenv("OPENCHAT_API_URL"))
18
+ return InferenceAPIClient(model, token=os.getenv("HF_TOKEN", None))
19
+
20
+
21
+ def get_usernames(model: str):
22
+ """
23
+ Returns:
24
+ (str, str, str, str): pre-prompt, username, bot name, separator
25
+ """
26
+ if model in ("OpenAssistant/oasst-sft-1-pythia-12b", "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"):
27
+ return "", "<|prompter|>", "<|assistant|>", "<|endoftext|>"
28
+ if model == "togethercomputer/GPT-NeoXT-Chat-Base-20B":
29
+ return openchat_preprompt, "<human>: ", "<bot>: ", "\n"
30
+ return "", "User: ", "Assistant: ", "\n"
31
+
32
+
33
+ def predict(
34
+ model: str,
35
+ inputs: str,
36
+ typical_p: float,
37
+ top_p: float,
38
+ temperature: float,
39
+ top_k: int,
40
+ repetition_penalty: float,
41
+ watermark: bool,
42
+ chatbot,
43
+ history,
44
+ ):
45
+ client = get_client(model)
46
+ preprompt, user_name, assistant_name, sep = get_usernames(model)
47
+
48
+ history.append(inputs)
49
+
50
+ past = []
51
+ for data in chatbot:
52
+ user_data, model_data = data
53
+
54
+ if not user_data.startswith(user_name):
55
+ user_data = user_name + user_data
56
+ if not model_data.startswith(sep + assistant_name):
57
+ model_data = sep + assistant_name + model_data
58
+
59
+ past.append(user_data + model_data.rstrip() + sep)
60
+
61
+ if not inputs.startswith(user_name):
62
+ inputs = user_name + inputs
63
+
64
+ total_inputs = preprompt + "".join(past) + inputs + sep + assistant_name.rstrip()
65
+
66
+ partial_words = ""
67
+
68
+ if model in ("OpenAssistant/oasst-sft-1-pythia-12b", "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"):
69
+ iterator = client.generate_stream(
70
+ total_inputs,
71
+ typical_p=typical_p,
72
+ truncate=1000,
73
+ watermark=watermark,
74
+ max_new_tokens=500,
75
+ )
76
+ else:
77
+ iterator = client.generate_stream(
78
+ total_inputs,
79
+ top_p=top_p if top_p < 1.0 else None,
80
+ top_k=top_k,
81
+ truncate=1000,
82
+ repetition_penalty=repetition_penalty,
83
+ watermark=watermark,
84
+ temperature=temperature,
85
+ max_new_tokens=500,
86
+ stop_sequences=[user_name.rstrip(), assistant_name.rstrip()],
87
+ )
88
+
89
+ for i, response in enumerate(iterator):
90
+ if response.token.special:
91
+ continue
92
+
93
+ partial_words = partial_words + response.token.text
94
+ if partial_words.endswith(user_name.rstrip()):
95
+ partial_words = partial_words.rstrip(user_name.rstrip())
96
+ if partial_words.endswith(assistant_name.rstrip()):
97
+ partial_words = partial_words.rstrip(assistant_name.rstrip())
98
+
99
+ if i == 0:
100
+ history.append(" " + partial_words)
101
+ elif response.token.text not in user_name:
102
+ history[-1] = partial_words
103
+
104
+ chat = [
105
+ (history[i].strip(), history[i + 1].strip())
106
+ for i in range(0, len(history) - 1, 2)
107
+ ]
108
+ yield chat, history
109
+
110
+
111
+ def reset_textbox():
112
+ return gr.update(value="")
113
+
114
+
115
+ def radio_on_change(
116
+ value: str,
117
+ disclaimer,
118
+ typical_p,
119
+ top_p,
120
+ top_k,
121
+ temperature,
122
+ repetition_penalty,
123
+ watermark,
124
+ ):
125
+ if value in ("OpenAssistant/oasst-sft-1-pythia-12b", "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"):
126
+ typical_p = typical_p.update(value=0.2, visible=True)
127
+ top_p = top_p.update(visible=False)
128
+ top_k = top_k.update(visible=False)
129
+ temperature = temperature.update(visible=False)
130
+ disclaimer = disclaimer.update(visible=False)
131
+ repetition_penalty = repetition_penalty.update(visible=False)
132
+ watermark = watermark.update(False)
133
+ elif value == "togethercomputer/GPT-NeoXT-Chat-Base-20B":
134
+ typical_p = typical_p.update(visible=False)
135
+ top_p = top_p.update(value=0.25, visible=True)
136
+ top_k = top_k.update(value=50, visible=True)
137
+ temperature = temperature.update(value=0.6, visible=True)
138
+ repetition_penalty = repetition_penalty.update(value=1.01, visible=True)
139
+ watermark = watermark.update(False)
140
+ disclaimer = disclaimer.update(visible=True)
141
+ else:
142
+ typical_p = typical_p.update(visible=False)
143
+ top_p = top_p.update(value=0.95, visible=True)
144
+ top_k = top_k.update(value=4, visible=True)
145
+ temperature = temperature.update(value=0.5, visible=True)
146
+ repetition_penalty = repetition_penalty.update(value=1.03, visible=True)
147
+ watermark = watermark.update(True)
148
+ disclaimer = disclaimer.update(visible=False)
149
+ return (
150
+ disclaimer,
151
+ typical_p,
152
+ top_p,
153
+ top_k,
154
+ temperature,
155
+ repetition_penalty,
156
+ watermark,
157
+ )
158
+
159
+
160
+ title = """<h1 align="center">Large Language Model Chat API</h1>"""
161
+ description = """Language models can be conditioned to act like dialogue agents through a conversational prompt that typically takes the form:
162
+
163
+ ```
164
+ User: <utterance>
165
+ Assistant: <utterance>
166
+ User: <utterance>
167
+ Assistant: <utterance>
168
+ ...
169
+ ```
170
+
171
+ In this app, you can explore the outputs of multiple LLMs when prompted in this way.
172
+ """
173
+
174
+ text_generation_inference = """
175
+ <div align="center">Powered by: <a href=https://github.com/huggingface/text-generation-inference>Text Generation Inference</a></div>
176
+ """
177
+
178
+ openchat_disclaimer = """
179
+ <div align="center">Checkout the official <a href=https://huggingface.co/spaces/togethercomputer/OpenChatKit>OpenChatKit feedback app</a> for the full experience.</div>
180
+ """
181
+
182
+ with gr.Blocks(
183
+ css="""#col_container {margin-left: auto; margin-right: auto;}
184
+ #chatbot {height: 520px; overflow: auto;}"""
185
+ ) as demo:
186
+ gr.HTML(title)
187
+ gr.Markdown(text_generation_inference, visible=True)
188
+ with gr.Column(elem_id="col_container"):
189
+ model = gr.Radio(
190
+ value="OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5",
191
+ choices=[
192
+ "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5",
193
+ "OpenAssistant/oasst-sft-1-pythia-12b",
194
+ # "togethercomputer/GPT-NeoXT-Chat-Base-20B",
195
+ "google/flan-t5-xxl",
196
+ "google/flan-ul2",
197
+ "bigscience/bloom",
198
+ "bigscience/bloomz",
199
+ "EleutherAI/gpt-neox-20b",
200
+ ],
201
+ label="Model",
202
+ interactive=True,
203
+ )
204
+
205
+ chatbot = gr.Chatbot(elem_id="chatbot")
206
+ inputs = gr.Textbox(
207
+ placeholder="Hi there!", label="Type an input and press Enter"
208
+ )
209
+ disclaimer = gr.Markdown(openchat_disclaimer, visible=False)
210
+ state = gr.State([])
211
+ b1 = gr.Button()
212
+
213
+ with gr.Accordion("Parameters", open=False):
214
+ typical_p = gr.Slider(
215
+ minimum=-0,
216
+ maximum=1.0,
217
+ value=0.2,
218
+ step=0.05,
219
+ interactive=True,
220
+ label="Typical P mass",
221
+ )
222
+ top_p = gr.Slider(
223
+ minimum=-0,
224
+ maximum=1.0,
225
+ value=0.25,
226
+ step=0.05,
227
+ interactive=True,
228
+ label="Top-p (nucleus sampling)",
229
+ visible=False,
230
+ )
231
+ temperature = gr.Slider(
232
+ minimum=-0,
233
+ maximum=5.0,
234
+ value=0.6,
235
+ step=0.1,
236
+ interactive=True,
237
+ label="Temperature",
238
+ visible=False,
239
+ )
240
+ top_k = gr.Slider(
241
+ minimum=1,
242
+ maximum=50,
243
+ value=50,
244
+ step=1,
245
+ interactive=True,
246
+ label="Top-k",
247
+ visible=False,
248
+ )
249
+ repetition_penalty = gr.Slider(
250
+ minimum=0.1,
251
+ maximum=3.0,
252
+ value=1.03,
253
+ step=0.01,
254
+ interactive=True,
255
+ label="Repetition Penalty",
256
+ visible=False,
257
+ )
258
+ watermark = gr.Checkbox(value=False, label="Text watermarking")
259
+
260
+ model.change(
261
+ lambda value: radio_on_change(
262
+ value,
263
+ disclaimer,
264
+ typical_p,
265
+ top_p,
266
+ top_k,
267
+ temperature,
268
+ repetition_penalty,
269
+ watermark,
270
+ ),
271
+ inputs=model,
272
+ outputs=[
273
+ disclaimer,
274
+ typical_p,
275
+ top_p,
276
+ top_k,
277
+ temperature,
278
+ repetition_penalty,
279
+ watermark,
280
+ ],
281
+ )
282
+
283
+ inputs.submit(
284
+ predict,
285
+ [
286
+ model,
287
+ inputs,
288
+ typical_p,
289
+ top_p,
290
+ temperature,
291
+ top_k,
292
+ repetition_penalty,
293
+ watermark,
294
+ chatbot,
295
+ state,
296
+ ],
297
+ [chatbot, state],
298
+ )
299
+ b1.click(
300
+ predict,
301
+ [
302
+ model,
303
+ inputs,
304
+ typical_p,
305
+ top_p,
306
+ temperature,
307
+ top_k,
308
+ repetition_penalty,
309
+ watermark,
310
+ chatbot,
311
+ state,
312
+ ],
313
+ [chatbot, state],
314
+ )
315
+ b1.click(reset_textbox, [], [inputs])
316
+ inputs.submit(reset_textbox, [], [inputs])
317
+
318
+ gr.Markdown(description)
319
+ demo.queue(concurrency_count=16).launch(debug=True)
llms.png ADDED
share_btn.py CHANGED
@@ -76,7 +76,7 @@ ${outputTxt}`;
76
  .map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(value)}`)
77
  .join('&');
78
 
79
- window.open(`https://huggingface.co/spaces/trl-lib/stack-llama/discussions/new?${paramsStr}`, '_blank');
80
 
81
  shareBtnEl.style.removeProperty('pointer-events');
82
  shareIconEl.style.removeProperty('display');
 
76
  .map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(value)}`)
77
  .join('&');
78
 
79
+ window.open(`https://huggingface.co/spaces/HuggingFaceH4/star-chat-demo/discussions/new?${paramsStr}`, '_blank');
80
 
81
  shareBtnEl.style.removeProperty('pointer-events');
82
  shareIconEl.style.removeProperty('display');
stack-llama.py ADDED
@@ -0,0 +1,186 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ import os
3
+ import shutil
4
+
5
+ import gradio as gr
6
+ from huggingface_hub import Repository
7
+ from text_generation import Client
8
+
9
+ from share_btn import community_icon_html, loading_icon_html, share_js, share_btn_css
10
+
11
+ HF_TOKEN = os.environ.get("H4_TOKEN", None)
12
+ # HF_TOKEN = os.environ.get("TRL_TOKEN", None)
13
+ API_URL = os.environ.get("API_URL")
14
+
15
+ theme = gr.themes.Monochrome(
16
+ primary_hue="indigo",
17
+ secondary_hue="blue",
18
+ neutral_hue="slate",
19
+ radius_size=gr.themes.sizes.radius_sm,
20
+ font=[gr.themes.GoogleFont("Open Sans"), "ui-sans-serif", "system-ui", "sans-serif"],
21
+ )
22
+ if HF_TOKEN:
23
+ try:
24
+ shutil.rmtree("./data/")
25
+ except:
26
+ pass
27
+
28
+ repo = Repository(
29
+ local_dir="./data/", clone_from="h4/star-chat-prompts", use_auth_token=HF_TOKEN, repo_type="dataset"
30
+ )
31
+ repo.git_pull()
32
+
33
+ client = Client(
34
+ API_URL,
35
+ headers={"Authorization": f"Bearer {HF_TOKEN}"},
36
+ )
37
+
38
+ PROMPT_TEMPLATE = """Question: {prompt}\n\nAnswer:"""
39
+
40
+
41
+ def save_inputs_and_outputs(inputs, outputs, generate_kwargs):
42
+ with open(os.path.join("data", "prompts.jsonl"), "a") as f:
43
+ json.dump({"inputs": inputs, "outputs": outputs, "generate_kwargs": generate_kwargs}, f, ensure_ascii=False)
44
+ f.write("\n")
45
+ commit_url = repo.push_to_hub()
46
+
47
+
48
+ def generate(instruction, temperature=0.9, max_new_tokens=256, top_p=0.95, repetition_penalty=1.0, do_save=True):
49
+ formatted_instruction = PROMPT_TEMPLATE.format(prompt=instruction)
50
+
51
+ temperature = float(temperature)
52
+ if temperature < 1e-2:
53
+ temperature = 1e-2
54
+ top_p = float(top_p)
55
+
56
+ generate_kwargs = dict(
57
+ temperature=temperature,
58
+ max_new_tokens=max_new_tokens,
59
+ top_p=top_p,
60
+ repetition_penalty=repetition_penalty,
61
+ do_sample=True,
62
+ truncate=999,
63
+ seed=42,
64
+ stop_sequences=["</s>"],
65
+ )
66
+
67
+ stream = client.generate_stream(
68
+ formatted_instruction,
69
+ **generate_kwargs,
70
+ )
71
+
72
+ output = ""
73
+ for response in stream:
74
+ output += response.token.text
75
+ yield output
76
+ if HF_TOKEN and do_save:
77
+ try:
78
+ print("Pushing prompt and completion to the Hub")
79
+ save_inputs_and_outputs(formatted_instruction, output, generate_kwargs)
80
+ except Exception as e:
81
+ print(e)
82
+
83
+ return output
84
+
85
+
86
+ examples = [
87
+ "A llama is in my lawn. How do I get rid of him?",
88
+ "What are the various algorithms to sort a list?",
89
+ "How can I sort a list in Python?",
90
+ "How do I ask a question in StackOverflow?",
91
+ "How to beat a Hitmonlee in a Pokemon battle?",
92
+ "How can I write a Java function to generate the nth Fibonacci number?",
93
+ ]
94
+
95
+
96
+ def process_example(args):
97
+ for x in generate(args):
98
+ pass
99
+ return x
100
+
101
+ css = ".generating {visibility: hidden}" + share_btn_css
102
+
103
+ with gr.Blocks(theme=theme, analytics_enabled=False, css=css) as demo:
104
+ with gr.Column():
105
+ gr.Markdown(
106
+ """![](https://huggingface.co/spaces/trl-lib/stack-llama/resolve/main/stackllama_logo.png)
107
+
108
+
109
+ StackLLaMa is a 7 billion parameter language model based on [Meta's LLaMA model](https://ai.facebook.com/blog/large-language-model-llama-meta-ai/) that has been trained on pairs of questions and answers from [Stack Exchange](https://stackexchange.com) using Reinforcement Learning from Human Feedback (RLHF) with the [TRL library](https://github.com/lvwerra/trl). For more details, check out our [blog post](https://huggingface.co/blog/stackllama).
110
+
111
+ Type in the box below and click the button to generate answers to your most pressing questions!
112
+
113
+ ⚠️ **Intended Use**: this app and its [supporting model](https://huggingface.co/trl-lib/llama-7b-se-rl-peft) are provided as educational tools to explain RLHF with the TRL library; not to serve as replacement for human expertise. For more details on the model's limitations in terms of factuality and biases, see the [model card.](https://huggingface.co/trl-lib/llama-7b-se-rl-peft#intended-uses--limitations)
114
+
115
+ ⚠️ **Data Collection**: by default, we are collecting the prompts entered in this app to further improve and evaluate the model. Do not share any personal or sensitive information while using the app! You can opt out of this data collection by removing the checkbox below:
116
+ """
117
+ )
118
+ with gr.Row():
119
+ with gr.Column(scale=3):
120
+ do_save = gr.Checkbox(
121
+ value=True,
122
+ label="Store data",
123
+ info="You agree to the storage of your prompt and generated text for research and development purposes:")
124
+ instruction = gr.Textbox(placeholder="Enter your question here", label="Question", elem_id="q-input")
125
+
126
+
127
+ with gr.Box():
128
+ gr.Markdown("**Answer**")
129
+ output = gr.Markdown(elem_id="q-output")
130
+ submit = gr.Button("Generate", variant="primary")
131
+ with gr.Group(elem_id="share-btn-container"):
132
+ community_icon = gr.HTML(community_icon_html, visible=True)
133
+ loading_icon = gr.HTML(loading_icon_html, visible=True)
134
+ share_button = gr.Button("Share to community", elem_id="share-btn", visible=True)
135
+ gr.Examples(
136
+ examples=examples,
137
+ inputs=[instruction],
138
+ cache_examples=False,
139
+ fn=process_example,
140
+ outputs=[output],
141
+ )
142
+
143
+ with gr.Column(scale=1):
144
+
145
+ temperature = gr.Slider(
146
+ label="Temperature",
147
+ value=0.9,
148
+ minimum=0.0,
149
+ maximum=2.0,
150
+ step=0.1,
151
+ interactive=True,
152
+ info="Higher values produce more diverse outputs",
153
+ )
154
+ max_new_tokens = gr.Slider(
155
+ label="Max new tokens",
156
+ value=256,
157
+ minimum=0,
158
+ maximum=512,
159
+ step=4,
160
+ interactive=True,
161
+ info="The maximum numbers of new tokens",
162
+ )
163
+ top_p = gr.Slider(
164
+ label="Top-p (nucleus sampling)",
165
+ value=0.90,
166
+ minimum=0.0,
167
+ maximum=1,
168
+ step=0.05,
169
+ interactive=True,
170
+ info="Higher values sample more low-probability tokens",
171
+ )
172
+ repetition_penalty = gr.Slider(
173
+ label="Repetition penalty",
174
+ value=1.2,
175
+ minimum=1.0,
176
+ maximum=2.0,
177
+ step=0.05,
178
+ interactive=True,
179
+ info="Penalize repeated tokens",
180
+ )
181
+
182
+ submit.click(generate, inputs=[instruction, temperature, max_new_tokens, top_p, repetition_penalty, do_save], outputs=[output])
183
+ instruction.submit(generate, inputs=[instruction, temperature, max_new_tokens, top_p, repetition_penalty], outputs=[output])
184
+ share_button.click(None, [], [], _js=share_js)
185
+
186
+ demo.queue(concurrency_count=16).launch(debug=True)