DHEIVER commited on
Commit
d1a163d
1 Parent(s): 764fb2f

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +444 -19
app.py CHANGED
@@ -1,27 +1,452 @@
 
 
 
 
 
 
 
 
 
 
 
1
  import gradio as gr
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
 
3
- from transformers import AutoModelForCausalLM
 
 
 
4
 
5
- model = AutoModelForCausalLM.from_pretrained("automaise/quokka-7b")
6
 
7
- def create_gradio_app(model, tokenizer):
8
- def gradio_fn(question):
9
- inputs = tokenizer(question, return_tensors="pt")
10
- outputs = model(inputs)[0]
11
- generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
12
- return generated_text
13
 
14
- iface = gr.Interface(
15
- fn=gradio_fn,
16
- inputs=gr.inputs.Textbox(),
17
- outputs=gr.outputs.Textbox(),
18
- live=True,
19
- title="Gradio App",
20
- description="Create a gradio app using the Quokka-7b model.",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
21
  )
22
- return iface
23
 
24
- iface = create_gradio_app(model, tokenizer)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
25
 
26
- if __name__ == "__main__":
27
- iface.launch()
 
1
+ """Run codes."""
2
+ # pylint: disable=line-too-long, broad-exception-caught, invalid-name, missing-function-docstring, too-many-instance-attributes, missing-class-docstring
3
+ # ruff: noqa: E501
4
+ import os
5
+ import platform
6
+ import random
7
+ import time
8
+ from dataclasses import asdict, dataclass
9
+ from pathlib import Path
10
+
11
+ # from types import SimpleNamespace
12
  import gradio as gr
13
+ import psutil
14
+ from about_time import about_time
15
+ from ctransformers import AutoModelForCausalLM
16
+ from dl_hf_model import dl_hf_model
17
+ from loguru import logger
18
+
19
+ filename_list = [
20
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q2_K.bin",
21
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q3_K_L.bin",
22
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q3_K_M.bin",
23
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q3_K_S.bin",
24
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_0.bin",
25
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_1.bin",
26
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_K_M.bin",
27
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_K_S.bin",
28
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q5_0.bin",
29
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q5_1.bin",
30
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q5_K_M.bin",
31
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q5_K_S.bin",
32
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q6_K.bin",
33
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q8_0.bin",
34
+ ]
35
+
36
+ URL = "https://huggingface.co/TheBloke/Wizard-Vicuna-7B-Uncensored-GGML/raw/main/Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_K_M.bin" # 4.05G
37
+
38
+ url = "https://huggingface.co/savvamadar/ggml-gpt4all-j-v1.3-groovy/blob/main/ggml-gpt4all-j-v1.3-groovy.bin"
39
+ url = "https://huggingface.co/TheBloke/Llama-2-13B-GGML/blob/main/llama-2-13b.ggmlv3.q4_K_S.bin" # 7.37G
40
+ # url = "https://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/blob/main/llama-2-13b-chat.ggmlv3.q3_K_L.bin"
41
+ url = "https://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/blob/main/llama-2-13b-chat.ggmlv3.q3_K_L.bin" # 6.93G
42
+ # url = "https://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/blob/main/llama-2-13b-chat.ggmlv3.q3_K_L.binhttps://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/blob/main/llama-2-13b-chat.ggmlv3.q4_K_M.bin" # 7.87G
43
+
44
+ url = "https://huggingface.co/localmodels/Llama-2-13B-Chat-ggml/blob/main/llama-2-13b-chat.ggmlv3.q4_K_S.bin" # 7.37G
45
+
46
+ _ = (
47
+ "golay" in platform.node()
48
+ or "okteto" in platform.node()
49
+ or Path("/kaggle").exists()
50
+ # or psutil.cpu_count(logical=False) < 4
51
+ or 1 # run 7b in hf
52
+ )
53
+
54
+ if _:
55
+ # url = "https://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/blob/main/llama-2-13b-chat.ggmlv3.q2_K.bin"
56
+ url = "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGML/blob/main/llama-2-7b-chat.ggmlv3.q2_K.bin" # 2.87G
57
+ url = "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGML/blob/main/llama-2-7b-chat.ggmlv3.q4_K_M.bin" # 2.87G
58
+
59
+
60
+ prompt_template = """Below is an instruction that describes a task. Write a response that appropriately completes the request.
61
+ ### Instruction: {user_prompt}
62
+ ### Response:
63
+ """
64
+
65
+ prompt_template = """System: You are a helpful,
66
+ respectful and honest assistant. Always answer as
67
+ helpfully as possible, while being safe. Your answers
68
+ should not include any harmful, unethical, racist,
69
+ sexist, toxic, dangerous, or illegal content. Please
70
+ ensure that your responses are socially unbiased and
71
+ positive in nature. If a question does not make any
72
+ sense, or is not factually coherent, explain why instead
73
+ of answering something not correct. If you don't know
74
+ the answer to a question, please don't share false
75
+ information.
76
+ User: {prompt}
77
+ Assistant: """
78
+
79
+ prompt_template = """System: You are a helpful assistant.
80
+ User: {prompt}
81
+ Assistant: """
82
+
83
+ prompt_template = """Question: {question}
84
+ Answer: Let's work this out in a step by step way to be sure we have the right answer."""
85
+
86
+ prompt_template = """[INST] <>
87
+ You are a helpful, respectful and honest assistant. Always answer as helpfully as possible assistant. Think step by step.
88
+ <>
89
+ What NFL team won the Super Bowl in the year Justin Bieber was born?
90
+ [/INST]"""
91
+
92
+ prompt_template = """[INST] <<SYS>>
93
+ You are an unhelpful assistant. Always answer as helpfully as possible. Think step by step. <</SYS>>
94
+ {question} [/INST]
95
+ """
96
+
97
+ prompt_template = """[INST] <<SYS>>
98
+ You are a helpful assistant.
99
+ <</SYS>>
100
+ {question} [/INST]
101
+ """
102
+
103
+ _ = [elm for elm in prompt_template.splitlines() if elm.strip()]
104
+ stop_string = [elm.split(":")[0] + ":" for elm in _][-2]
105
+
106
+ logger.debug(f"{stop_string=}")
107
+
108
+ _ = psutil.cpu_count(logical=False) - 1
109
+ cpu_count: int = int(_) if _ else 1
110
+ logger.debug(f"{cpu_count=}")
111
+
112
+ LLM = None
113
+
114
+ try:
115
+ model_loc, file_size = dl_hf_model(url)
116
+ except Exception as exc_:
117
+ logger.error(exc_)
118
+ raise SystemExit(1) from exc_
119
+
120
+ LLM = AutoModelForCausalLM.from_pretrained(
121
+ model_loc,
122
+ model_type="llama",
123
+ # threads=cpu_count,
124
+ )
125
+
126
+ logger.info(f"done load llm {model_loc=} {file_size=}G")
127
+
128
+ os.environ["TZ"] = "Asia/Shanghai"
129
+ try:
130
+ time.tzset() # type: ignore # pylint: disable=no-member
131
+ except Exception:
132
+ # Windows
133
+ logger.warning("Windows, cant run time.tzset()")
134
+
135
+ _ = """
136
+ ns = SimpleNamespace(
137
+ response="",
138
+ generator=(_ for _ in []),
139
+ )
140
+ # """
141
+
142
+ @dataclass
143
+ class GenerationConfig:
144
+ temperature: float = 0.7
145
+ top_k: int = 50
146
+ top_p: float = 0.9
147
+ repetition_penalty: float = 1.0
148
+ max_new_tokens: int = 512
149
+ seed: int = 42
150
+ reset: bool = False
151
+ stream: bool = True
152
+ # threads: int = cpu_count
153
+ # stop: list[str] = field(default_factory=lambda: [stop_string])
154
+
155
+
156
+ def generate(
157
+ question: str,
158
+ llm=LLM,
159
+ config: GenerationConfig = GenerationConfig(),
160
+ ):
161
+ """Run model inference, will return a Generator if streaming is true."""
162
+ # _ = prompt_template.format(question=question)
163
+ # print(_)
164
+
165
+ prompt = prompt_template.format(question=question)
166
+
167
+ return llm(
168
+ prompt,
169
+ **asdict(config),
170
+ )
171
+
172
+
173
+ logger.debug(f"{asdict(GenerationConfig())=}")
174
+
175
 
176
+ def user(user_message, history):
177
+ # return user_message, history + [[user_message, None]]
178
+ history.append([user_message, None])
179
+ return user_message, history # keep user_message
180
 
 
181
 
182
+ def user1(user_message, history):
183
+ # return user_message, history + [[user_message, None]]
184
+ history.append([user_message, None])
185
+ return "", history # clear user_message
 
 
186
 
187
+
188
+ def bot_(history):
189
+ user_message = history[-1][0]
190
+ resp = random.choice(["How are you?", "I love you", "I'm very hungry"])
191
+ bot_message = user_message + ": " + resp
192
+ history[-1][1] = ""
193
+ for character in bot_message:
194
+ history[-1][1] += character
195
+ time.sleep(0.02)
196
+ yield history
197
+
198
+ history[-1][1] = resp
199
+ yield history
200
+
201
+
202
+ def bot(history):
203
+ user_message = history[-1][0]
204
+ response = []
205
+
206
+ logger.debug(f"{user_message=}")
207
+
208
+ with about_time() as atime: # type: ignore
209
+ flag = 1
210
+ prefix = ""
211
+ then = time.time()
212
+
213
+ logger.debug("about to generate")
214
+
215
+ config = GenerationConfig(reset=True)
216
+ for elm in generate(user_message, config=config):
217
+ if flag == 1:
218
+ logger.debug("in the loop")
219
+ prefix = f"({time.time() - then:.2f}s) "
220
+ flag = 0
221
+ print(prefix, end="", flush=True)
222
+ logger.debug(f"{prefix=}")
223
+ print(elm, end="", flush=True)
224
+ # logger.debug(f"{elm}")
225
+
226
+ response.append(elm)
227
+ history[-1][1] = prefix + "".join(response)
228
+ yield history
229
+
230
+ _ = (
231
+ f"(time elapsed: {atime.duration_human}, " # type: ignore
232
+ f"{atime.duration/len(''.join(response)):.2f}s/char)" # type: ignore
233
  )
 
234
 
235
+ history[-1][1] = "".join(response) + f"\n{_}"
236
+ yield history
237
+
238
+
239
+ def predict_api(prompt):
240
+ logger.debug(f"{prompt=}")
241
+ try:
242
+ # user_prompt = prompt
243
+ config = GenerationConfig(
244
+ temperature=0.2,
245
+ top_k=10,
246
+ top_p=0.9,
247
+ repetition_penalty=1.0,
248
+ max_new_tokens=512, # adjust as needed
249
+ seed=42,
250
+ reset=True, # reset history (cache)
251
+ stream=False,
252
+ # threads=cpu_count,
253
+ # stop=prompt_prefix[1:2],
254
+ )
255
+
256
+ response = generate(
257
+ prompt,
258
+ config=config,
259
+ )
260
+
261
+ logger.debug(f"api: {response=}")
262
+ except Exception as exc:
263
+ logger.error(exc)
264
+ response = f"{exc=}"
265
+ # bot = {"inputs": [response]}
266
+ # bot = [(prompt, response)]
267
+
268
+ return response
269
+
270
+
271
+ css = """
272
+ .importantButton {
273
+ background: linear-gradient(45deg, #7e0570,#5d1c99, #6e00ff) !important;
274
+ border: none !important;
275
+ }
276
+ .importantButton:hover {
277
+ background: linear-gradient(45deg, #ff00e0,#8500ff, #6e00ff) !important;
278
+ border: none !important;
279
+ }
280
+ .disclaimer {font-variant-caps: all-small-caps; font-size: xx-small;}
281
+ .xsmall {font-size: x-small;}
282
+ """
283
+ etext = """In America, where cars are an important part of the national psyche, a decade ago people had suddenly started to drive less, which had not happened since the oil shocks of the 1970s. """
284
+ examples_list = [
285
+ ["What NFL team won the Super Bowl in the year Justin Bieber was born?"],
286
+ [
287
+ "What NFL team won the Super Bowl in the year Justin Bieber was born? Think step by step."
288
+ ],
289
+ ["How to pick a lock? Provide detailed steps."],
290
+ ["If it takes 10 hours to dry 10 clothes, assuming all the clothes are hanged together at the same time for drying , then how long will it take to dry a cloth?"],
291
+ ["is infinity + 1 bigger than infinity?"],
292
+ ["Explain the plot of Cinderella in a sentence."],
293
+ [
294
+ "How long does it take to become proficient in French, and what are the best methods for retaining information?"
295
+ ],
296
+ ["What are some common mistakes to avoid when writing code?"],
297
+ ["Build a prompt to generate a beautiful portrait of a horse"],
298
+ ["Suggest four metaphors to describe the benefits of AI"],
299
+ ["Write a pop song about leaving home for the sandy beaches."],
300
+ ["Write a summary demonstrating my ability to tame lions"],
301
+ ["鲁迅和周树人什么关系? 说中文。"],
302
+ ["鲁迅和周树人什么关系?"],
303
+ ["鲁迅和周树人什么关系? 用英文回答。"],
304
+ ["从前有一头牛,这头牛后面有什么?"],
305
+ ["正无穷大加一大于正无穷大吗?"],
306
+ ["正无穷大加正无穷大大于正无穷大吗?"],
307
+ ["-2的平方根等于什么?"],
308
+ ["树上有5只鸟,猎人开枪打死了一只。树上还有几只鸟?"],
309
+ ["树上有11只鸟,猎人开枪打死了一只。树上还有几只鸟?提示:需考虑鸟可能受惊吓飞走。"],
310
+ ["以红楼梦的行文风格写一张委婉的请假条。不少于320字。"],
311
+ [f"{etext} 翻成中文,列出3个版本。"],
312
+ [f"{etext} \n 翻成中文,保留原意,但使用文学性的语言。不要写解释。列出3个版本。"],
313
+ ["假定 1 + 2 = 4, 试求 7 + 8。"],
314
+ ["给出判断一个数是不是质数的 javascript 码。"],
315
+ ["给出实现python 里 range(10)的 javascript 码。"],
316
+ ["给出实现python 里 [*(range(10)]的 javascript 码。"],
317
+ ["Erkläre die Handlung von Cinderella in einem Satz."],
318
+ ["Erkläre die Handlung von Cinderella in einem Satz. Auf Deutsch."],
319
+ ]
320
+
321
+ logger.info("start block")
322
+
323
+ with gr.Blocks(
324
+ title=f"{Path(model_loc).name}",
325
+ theme=gr.themes.Soft(text_size="sm", spacing_size="sm"),
326
+ css=css,
327
+ ) as block:
328
+ # buff_var = gr.State("")
329
+ with gr.Accordion("🎈 Info", open=False):
330
+ # gr.HTML(
331
+ # """<center><a href="https://huggingface.co/spaces/mikeee/mpt-30b-chat?duplicate=true"><img src="https://bit.ly/3gLdBN6" alt="Duplicate"></a> and spin a CPU UPGRADE to avoid the queue</center>"""
332
+ # )
333
+ gr.Markdown(
334
+ f"""<h5><center>{Path(model_loc).name}</center></h4>
335
+ Most examples are meant for another model.
336
+ You probably should try to test
337
+ some related prompts.""",
338
+ elem_classes="xsmall",
339
+ )
340
+
341
+ # chatbot = gr.Chatbot().style(height=700) # 500
342
+ chatbot = gr.Chatbot(height=500)
343
+
344
+ # buff = gr.Textbox(show_label=False, visible=True)
345
+
346
+ with gr.Row():
347
+ with gr.Column(scale=5):
348
+ msg = gr.Textbox(
349
+ label="Chat Message Box",
350
+ placeholder="Ask me anything (press Shift+Enter or click Submit to send)",
351
+ show_label=False,
352
+ # container=False,
353
+ lines=6,
354
+ max_lines=30,
355
+ show_copy_button=True,
356
+ # ).style(container=False)
357
+ )
358
+ with gr.Column(scale=1, min_width=50):
359
+ with gr.Row():
360
+ submit = gr.Button("Submit", elem_classes="xsmall")
361
+ stop = gr.Button("Stop", visible=False)
362
+ clear = gr.Button("Clear History", visible=True)
363
+ with gr.Row(visible=False):
364
+ with gr.Accordion("Advanced Options:", open=False):
365
+ with gr.Row():
366
+ with gr.Column(scale=2):
367
+ system = gr.Textbox(
368
+ label="System Prompt",
369
+ value=prompt_template,
370
+ show_label=False,
371
+ container=False,
372
+ # ).style(container=False)
373
+ )
374
+ with gr.Column():
375
+ with gr.Row():
376
+ change = gr.Button("Change System Prompt")
377
+ reset = gr.Button("Reset System Prompt")
378
+
379
+ with gr.Accordion("Example Inputs", open=True):
380
+ examples = gr.Examples(
381
+ examples=examples_list,
382
+ inputs=[msg],
383
+ examples_per_page=40,
384
+ )
385
+
386
+ # with gr.Row():
387
+ with gr.Accordion("Disclaimer", open=False):
388
+ _ = Path(model_loc).name
389
+ gr.Markdown(
390
+ f"Disclaimer: {_} can produce factually incorrect output, and should not be relied on to produce "
391
+ "factually accurate information. {_} was trained on various public datasets; while great efforts "
392
+ "have been taken to clean the pretraining data, it is possible that this model could generate lewd, "
393
+ "biased, or otherwise offensive outputs.",
394
+ elem_classes=["disclaimer"],
395
+ )
396
+
397
+ msg.submit(
398
+ # fn=conversation.user_turn,
399
+ fn=user,
400
+ inputs=[msg, chatbot],
401
+ outputs=[msg, chatbot],
402
+ queue=True,
403
+ show_progress="full",
404
+ # api_name=None,
405
+ ).then(bot, chatbot, chatbot, queue=True)
406
+ submit.click(
407
+ # fn=lambda x, y: ("",) + user(x, y)[1:], # clear msg
408
+ fn=user1, # clear msg
409
+ inputs=[msg, chatbot],
410
+ outputs=[msg, chatbot],
411
+ queue=True,
412
+ # queue=False,
413
+ show_progress="full",
414
+ # api_name=None,
415
+ ).then(bot, chatbot, chatbot, queue=True)
416
+
417
+ clear.click(lambda: None, None, chatbot, queue=False)
418
+
419
+ with gr.Accordion("For Chat/Translation API", open=False, visible=False):
420
+ input_text = gr.Text()
421
+ api_btn = gr.Button("Go", variant="primary")
422
+ out_text = gr.Text()
423
+
424
+ api_btn.click(
425
+ predict_api,
426
+ input_text,
427
+ out_text,
428
+ api_name="api",
429
+ )
430
+
431
+ # block.load(update_buff, [], buff, every=1)
432
+ # block.load(update_buff, [buff_var], [buff_var, buff], every=1)
433
+
434
+ # concurrency_count=5, max_size=20
435
+ # max_size=36, concurrency_count=14
436
+ # CPU cpu_count=2 16G, model 7G
437
+ # CPU UPGRADE cpu_count=8 32G, model 7G
438
+
439
+ # does not work
440
+ _ = """
441
+ # _ = int(psutil.virtual_memory().total / 10**9 // file_size - 1)
442
+ # concurrency_count = max(_, 1)
443
+ if psutil.cpu_count(logical=False) >= 8:
444
+ # concurrency_count = max(int(32 / file_size) - 1, 1)
445
+ else:
446
+ # concurrency_count = max(int(16 / file_size) - 1, 1)
447
+ # """
448
+
449
+ concurrency_count = 1
450
+ logger.info(f"{concurrency_count=}")
451
 
452
+ block.queue(concurrency_count=concurrency_count, max_size=5).launch(debug=True)