mikeee commited on
Commit
42dc8e4
0 Parent(s):

Duplicate from mikeee/llama2-7b-chat-ggml

Browse files
Files changed (8) hide show
  1. .gitattributes +35 -0
  2. .gitignore +12 -0
  3. .ruff.toml +17 -0
  4. .stignore +103 -0
  5. README.md +13 -0
  6. app.py +463 -0
  7. requirements.txt +8 -0
  8. run-app.sh +1 -0
.gitattributes ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ call-activate.bat
2
+ okteto.yml
3
+ okteto-up.bat
4
+ install-sw.sh
5
+ install-sw1.sh
6
+ start-sshd.sh
7
+ pyproject.toml
8
+ models
9
+ .ruff_cache
10
+ run-nodemon.sh
11
+ app-.py
12
+ nodemon.json
.ruff.toml ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Assume Python 3.10.
2
+ target-version = "py310"
3
+ # Decrease the maximum line length to 79 characters.
4
+ line-length = 300
5
+
6
+ # pyflakes, pycodestyle, isort
7
+ # flake8 YTT, pydocstyle D, pylint PLC
8
+ select = ["F", "E", "W", "I001", "YTT", "D", "PLC"]
9
+ # select = ["ALL"]
10
+
11
+ # D103 Missing docstring in public function
12
+ # D101 Missing docstring in public class
13
+ # `multi-line-summary-first-line` (D212)
14
+ # `one-blank-line-before-class` (D203)
15
+ extend-ignore = ["D103", "D101", "D212", "D203"]
16
+
17
+ exclude = [".venv"]
.stignore ADDED
@@ -0,0 +1,103 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ models
2
+ *.bin
3
+ .git
4
+ # Byte-compiled / optimized / DLL files
5
+ __pycache__
6
+ *.py[cod]
7
+ *$py.class
8
+
9
+ # C extensions
10
+ *.so
11
+
12
+ # Distribution / packaging
13
+ .Python
14
+ build
15
+ develop-eggs
16
+ dist
17
+ downloads
18
+ eggs
19
+ .eggs
20
+ lib
21
+ lib64
22
+ parts
23
+ sdist
24
+ var
25
+ wheels
26
+ pip-wheel-metadata
27
+ share/python-wheels
28
+ *.egg-info
29
+ .installed.cfg
30
+ *.egg
31
+ MANIFEST
32
+
33
+ # PyInstaller
34
+ # Usually these files are written by a python script from a template
35
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
36
+ *.manifest
37
+ *.spec
38
+
39
+ # Installer logs
40
+ pip-log.txt
41
+ pip-delete-this-directory.txt
42
+
43
+ # Translations
44
+ *.mo
45
+ *.pot
46
+
47
+ # Django stuff:
48
+ *.log
49
+ local_settings.py
50
+ db.sqlite3
51
+
52
+ # Flask stuff:
53
+ instance
54
+ .webassets-cache
55
+
56
+ # Scrapy stuff:
57
+ .scrapy
58
+
59
+ # Sphinx documentation
60
+ docs/_build
61
+
62
+ # PyBuilder
63
+ target
64
+
65
+ # Jupyter Notebook
66
+ .ipynb_checkpoints
67
+
68
+ # IPython
69
+ profile_default
70
+ ipython_config.py
71
+
72
+ # pyenv
73
+ .python-version
74
+
75
+ # celery beat schedule file
76
+ celerybeat-schedule
77
+
78
+ # SageMath parsed files
79
+ *.sage.py
80
+
81
+ # Environments
82
+ .env
83
+ .venv
84
+ env
85
+ venv
86
+ ENV
87
+ env.bak
88
+ venv.bak
89
+
90
+ # Spyder project settings
91
+ .spyderproject
92
+ .spyproject
93
+
94
+ # Rope project settings
95
+ .ropeproject
96
+
97
+ # mypy
98
+ .mypy_cache
99
+ .dmypy.json
100
+ dmypy.json
101
+
102
+ # Pyre type checker
103
+ .pyre
README.md ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: llama-2-7b-or-13b-ggml
3
+ emoji: 🚀
4
+ colorFrom: green
5
+ colorTo: green
6
+ sdk: gradio
7
+ sdk_version: 3.37.0
8
+ app_file: app.py
9
+ pinned: true
10
+ duplicated_from: mikeee/llama2-7b-chat-ggml
11
+ ---
12
+
13
+ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py ADDED
@@ -0,0 +1,463 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Run codes."""
2
+ # pylint: disable=line-too-long, broad-exception-caught, invalid-name, missing-function-docstring, too-many-instance-attributes, missing-class-docstring
3
+ # ruff: noqa: E501
4
+ import os
5
+ import platform
6
+ import random
7
+ import time
8
+ from dataclasses import asdict, dataclass
9
+ from pathlib import Path
10
+
11
+ # from types import SimpleNamespace
12
+ import gradio as gr
13
+ import psutil
14
+ from about_time import about_time
15
+ from ctransformers import AutoModelForCausalLM
16
+ from dl_hf_model import dl_hf_model
17
+ from loguru import logger
18
+
19
+ filename_list = [
20
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q2_K.bin",
21
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q3_K_L.bin",
22
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q3_K_M.bin",
23
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q3_K_S.bin",
24
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_0.bin",
25
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_1.bin",
26
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_K_M.bin",
27
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_K_S.bin",
28
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q5_0.bin",
29
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q5_1.bin",
30
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q5_K_M.bin",
31
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q5_K_S.bin",
32
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q6_K.bin",
33
+ "Wizard-Vicuna-7B-Uncensored.ggmlv3.q8_0.bin",
34
+ ]
35
+
36
+ URL = "https://huggingface.co/TheBloke/Wizard-Vicuna-7B-Uncensored-GGML/raw/main/Wizard-Vicuna-7B-Uncensored.ggmlv3.q4_K_M.bin" # 4.05G
37
+
38
+ url = "https://huggingface.co/savvamadar/ggml-gpt4all-j-v1.3-groovy/blob/main/ggml-gpt4all-j-v1.3-groovy.bin"
39
+ url = "https://huggingface.co/TheBloke/Llama-2-13B-GGML/blob/main/llama-2-13b.ggmlv3.q4_K_S.bin" # 7.37G
40
+ # url = "https://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/blob/main/llama-2-13b-chat.ggmlv3.q3_K_L.bin"
41
+ url = "https://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/blob/main/llama-2-13b-chat.ggmlv3.q3_K_L.bin" # 6.93G
42
+ # url = "https://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/blob/main/llama-2-13b-chat.ggmlv3.q3_K_L.binhttps://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/blob/main/llama-2-13b-chat.ggmlv3.q4_K_M.bin" # 7.87G
43
+
44
+ url = "https://huggingface.co/localmodels/Llama-2-13B-Chat-ggml/blob/main/llama-2-13b-chat.ggmlv3.q4_K_S.bin" # 7.37G
45
+
46
+ _ = (
47
+ "golay" in platform.node()
48
+ or "okteto" in platform.node()
49
+ or Path("/kaggle").exists()
50
+ # or psutil.cpu_count(logical=False) < 4
51
+ or 1 # run 7b in hf
52
+ )
53
+
54
+ if _:
55
+ # url = "https://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/blob/main/llama-2-13b-chat.ggmlv3.q2_K.bin"
56
+ url = "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGML/blob/main/llama-2-7b-chat.ggmlv3.q2_K.bin" # 2.87G
57
+ url = "https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGML/blob/main/llama-2-7b-chat.ggmlv3.q4_K_M.bin" # 2.87G
58
+
59
+
60
+ prompt_template = """Below is an instruction that describes a task. Write a response that appropriately completes the request.
61
+
62
+ ### Instruction: {user_prompt}
63
+
64
+ ### Response:
65
+ """
66
+
67
+ prompt_template = """System: You are a helpful,
68
+ respectful and honest assistant. Always answer as
69
+ helpfully as possible, while being safe. Your answers
70
+ should not include any harmful, unethical, racist,
71
+ sexist, toxic, dangerous, or illegal content. Please
72
+ ensure that your responses are socially unbiased and
73
+ positive in nature. If a question does not make any
74
+ sense, or is not factually coherent, explain why instead
75
+ of answering something not correct. If you don't know
76
+ the answer to a question, please don't share false
77
+ information.
78
+ User: {prompt}
79
+ Assistant: """
80
+
81
+ prompt_template = """System: You are a helpful assistant.
82
+ User: {prompt}
83
+ Assistant: """
84
+
85
+ prompt_template = """Question: {question}
86
+ Answer: Let's work this out in a step by step way to be sure we have the right answer."""
87
+
88
+ prompt_template = """[INST] <>
89
+ You are a helpful, respectful and honest assistant. Always answer as helpfully as possible assistant. Think step by step.
90
+ <>
91
+
92
+ What NFL team won the Super Bowl in the year Justin Bieber was born?
93
+ [/INST]"""
94
+
95
+ prompt_template = """[INST] <<SYS>>
96
+ You are an unhelpful assistant. Always answer as helpfully as possible. Think step by step. <</SYS>>
97
+
98
+ {question} [/INST]
99
+ """
100
+
101
+ prompt_template = """[INST] <<SYS>>
102
+ You are a helpful assistant.
103
+ <</SYS>>
104
+
105
+ {question} [/INST]
106
+ """
107
+
108
+ _ = [elm for elm in prompt_template.splitlines() if elm.strip()]
109
+ stop_string = [elm.split(":")[0] + ":" for elm in _][-2]
110
+
111
+ logger.debug(f"{stop_string=}")
112
+
113
+ _ = psutil.cpu_count(logical=False) - 1
114
+ cpu_count: int = int(_) if _ else 1
115
+ logger.debug(f"{cpu_count=}")
116
+
117
+ LLM = None
118
+
119
+ try:
120
+ model_loc, file_size = dl_hf_model(url)
121
+ except Exception as exc_:
122
+ logger.error(exc_)
123
+ raise SystemExit(1) from exc_
124
+
125
+ LLM = AutoModelForCausalLM.from_pretrained(
126
+ model_loc,
127
+ model_type="llama",
128
+ # threads=cpu_count,
129
+ )
130
+
131
+ logger.info(f"done load llm {model_loc=} {file_size=}G")
132
+
133
+ os.environ["TZ"] = "Asia/Shanghai"
134
+ try:
135
+ time.tzset() # type: ignore # pylint: disable=no-member
136
+ except Exception:
137
+ # Windows
138
+ logger.warning("Windows, cant run time.tzset()")
139
+
140
+ _ = """
141
+ ns = SimpleNamespace(
142
+ response="",
143
+ generator=(_ for _ in []),
144
+ )
145
+ # """
146
+
147
+ @dataclass
148
+ class GenerationConfig:
149
+ temperature: float = 0.7
150
+ top_k: int = 50
151
+ top_p: float = 0.9
152
+ repetition_penalty: float = 1.0
153
+ max_new_tokens: int = 512
154
+ seed: int = 42
155
+ reset: bool = False
156
+ stream: bool = True
157
+ # threads: int = cpu_count
158
+ # stop: list[str] = field(default_factory=lambda: [stop_string])
159
+
160
+
161
+ def generate(
162
+ question: str,
163
+ llm=LLM,
164
+ config: GenerationConfig = GenerationConfig(),
165
+ ):
166
+ """Run model inference, will return a Generator if streaming is true."""
167
+ # _ = prompt_template.format(question=question)
168
+ # print(_)
169
+
170
+ prompt = prompt_template.format(question=question)
171
+
172
+ return llm(
173
+ prompt,
174
+ **asdict(config),
175
+ )
176
+
177
+
178
+ logger.debug(f"{asdict(GenerationConfig())=}")
179
+
180
+
181
+ def user(user_message, history):
182
+ # return user_message, history + [[user_message, None]]
183
+ history.append([user_message, None])
184
+ return user_message, history # keep user_message
185
+
186
+
187
+ def user1(user_message, history):
188
+ # return user_message, history + [[user_message, None]]
189
+ history.append([user_message, None])
190
+ return "", history # clear user_message
191
+
192
+
193
+ def bot_(history):
194
+ user_message = history[-1][0]
195
+ resp = random.choice(["How are you?", "I love you", "I'm very hungry"])
196
+ bot_message = user_message + ": " + resp
197
+ history[-1][1] = ""
198
+ for character in bot_message:
199
+ history[-1][1] += character
200
+ time.sleep(0.02)
201
+ yield history
202
+
203
+ history[-1][1] = resp
204
+ yield history
205
+
206
+
207
+ def bot(history):
208
+ user_message = history[-1][0]
209
+ response = []
210
+
211
+ logger.debug(f"{user_message=}")
212
+
213
+ with about_time() as atime: # type: ignore
214
+ flag = 1
215
+ prefix = ""
216
+ then = time.time()
217
+
218
+ logger.debug("about to generate")
219
+
220
+ config = GenerationConfig(reset=True)
221
+ for elm in generate(user_message, config=config):
222
+ if flag == 1:
223
+ logger.debug("in the loop")
224
+ prefix = f"({time.time() - then:.2f}s) "
225
+ flag = 0
226
+ print(prefix, end="", flush=True)
227
+ logger.debug(f"{prefix=}")
228
+ print(elm, end="", flush=True)
229
+ # logger.debug(f"{elm}")
230
+
231
+ response.append(elm)
232
+ history[-1][1] = prefix + "".join(response)
233
+ yield history
234
+
235
+ _ = (
236
+ f"(time elapsed: {atime.duration_human}, " # type: ignore
237
+ f"{atime.duration/len(''.join(response)):.2f}s/char)" # type: ignore
238
+ )
239
+
240
+ history[-1][1] = "".join(response) + f"\n{_}"
241
+ yield history
242
+
243
+
244
+ def predict_api(prompt):
245
+ logger.debug(f"{prompt=}")
246
+ try:
247
+ # user_prompt = prompt
248
+ config = GenerationConfig(
249
+ temperature=0.2,
250
+ top_k=10,
251
+ top_p=0.9,
252
+ repetition_penalty=1.0,
253
+ max_new_tokens=512, # adjust as needed
254
+ seed=42,
255
+ reset=True, # reset history (cache)
256
+ stream=False,
257
+ # threads=cpu_count,
258
+ # stop=prompt_prefix[1:2],
259
+ )
260
+
261
+ response = generate(
262
+ prompt,
263
+ config=config,
264
+ )
265
+
266
+ logger.debug(f"api: {response=}")
267
+ except Exception as exc:
268
+ logger.error(exc)
269
+ response = f"{exc=}"
270
+ # bot = {"inputs": [response]}
271
+ # bot = [(prompt, response)]
272
+
273
+ return response
274
+
275
+
276
+ css = """
277
+ .importantButton {
278
+ background: linear-gradient(45deg, #7e0570,#5d1c99, #6e00ff) !important;
279
+ border: none !important;
280
+ }
281
+ .importantButton:hover {
282
+ background: linear-gradient(45deg, #ff00e0,#8500ff, #6e00ff) !important;
283
+ border: none !important;
284
+ }
285
+ .disclaimer {font-variant-caps: all-small-caps; font-size: xx-small;}
286
+ .xsmall {font-size: x-small;}
287
+ """
288
+ etext = """In America, where cars are an important part of the national psyche, a decade ago people had suddenly started to drive less, which had not happened since the oil shocks of the 1970s. """
289
+ examples_list = [
290
+ ["What NFL team won the Super Bowl in the year Justin Bieber was born?"],
291
+ [
292
+ "What NFL team won the Super Bowl in the year Justin Bieber was born? Think step by step."
293
+ ],
294
+ ["How to pick a lock? Provide detailed steps."],
295
+ ["If it takes 10 hours to dry 10 clothes, assuming all the clothes are hanged together at the same time for drying , then how long will it take to dry a cloth?"],
296
+ ["is infinity + 1 bigger than infinity?"],
297
+ ["Explain the plot of Cinderella in a sentence."],
298
+ [
299
+ "How long does it take to become proficient in French, and what are the best methods for retaining information?"
300
+ ],
301
+ ["What are some common mistakes to avoid when writing code?"],
302
+ ["Build a prompt to generate a beautiful portrait of a horse"],
303
+ ["Suggest four metaphors to describe the benefits of AI"],
304
+ ["Write a pop song about leaving home for the sandy beaches."],
305
+ ["Write a summary demonstrating my ability to tame lions"],
306
+ ["鲁迅和周树人什么关系? 说中文。"],
307
+ ["鲁迅和周树人什么关系?"],
308
+ ["鲁迅和周树人什么关系? 用英文回答。"],
309
+ ["从前有一头牛,这头牛后面有什么?"],
310
+ ["正无穷大加一大于正无穷大吗?"],
311
+ ["正无穷大加正无穷大大于正无穷大吗?"],
312
+ ["-2的平方根等于什么?"],
313
+ ["树上有5只鸟,猎人开枪打死了一只。树上还有几只鸟?"],
314
+ ["树上有11只鸟,猎人开枪打死了一只。树上还有几只鸟?提示:需考虑鸟可能受惊吓飞走。"],
315
+ ["以红楼梦的行文风格写一张委婉的请假条。不少于320字。"],
316
+ [f"{etext} 翻成中文,列出3个版本。"],
317
+ [f"{etext} \n 翻成中文,保留原意,但使用文学性的语言。不要写解释。列出3个版本。"],
318
+ ["假定 1 + 2 = 4, 试求 7 + 8。"],
319
+ ["给出判断一个数是不是质数的 javascript 码。"],
320
+ ["给出实现python 里 range(10)的 javascript 码。"],
321
+ ["给出实现python 里 [*(range(10)]的 javascript 码。"],
322
+ ["Erkläre die Handlung von Cinderella in einem Satz."],
323
+ ["Erkläre die Handlung von Cinderella in einem Satz. Auf Deutsch."],
324
+ ]
325
+
326
+ logger.info("start block")
327
+
328
+ with gr.Blocks(
329
+ title=f"{Path(model_loc).name}",
330
+ theme=gr.themes.Soft(text_size="sm", spacing_size="sm"),
331
+ css=css,
332
+ ) as block:
333
+ # buff_var = gr.State("")
334
+ with gr.Accordion("🎈 Info", open=False):
335
+ # gr.HTML(
336
+ # """<center><a href="https://huggingface.co/spaces/mikeee/mpt-30b-chat?duplicate=true"><img src="https://bit.ly/3gLdBN6" alt="Duplicate"></a> and spin a CPU UPGRADE to avoid the queue</center>"""
337
+ # )
338
+ gr.Markdown(
339
+ f"""<h5><center>{Path(model_loc).name}</center></h4>
340
+ Most examples are meant for another model.
341
+ You probably should try to test
342
+ some related prompts.""",
343
+ elem_classes="xsmall",
344
+ )
345
+
346
+ # chatbot = gr.Chatbot().style(height=700) # 500
347
+ chatbot = gr.Chatbot(height=500)
348
+
349
+ # buff = gr.Textbox(show_label=False, visible=True)
350
+
351
+ with gr.Row():
352
+ with gr.Column(scale=5):
353
+ msg = gr.Textbox(
354
+ label="Chat Message Box",
355
+ placeholder="Ask me anything (press Shift+Enter or click Submit to send)",
356
+ show_label=False,
357
+ # container=False,
358
+ lines=6,
359
+ max_lines=30,
360
+ show_copy_button=True,
361
+ # ).style(container=False)
362
+ )
363
+ with gr.Column(scale=1, min_width=50):
364
+ with gr.Row():
365
+ submit = gr.Button("Submit", elem_classes="xsmall")
366
+ stop = gr.Button("Stop", visible=True)
367
+ clear = gr.Button("Clear History", visible=True)
368
+ with gr.Row(visible=False):
369
+ with gr.Accordion("Advanced Options:", open=False):
370
+ with gr.Row():
371
+ with gr.Column(scale=2):
372
+ system = gr.Textbox(
373
+ label="System Prompt",
374
+ value=prompt_template,
375
+ show_label=False,
376
+ container=False,
377
+ # ).style(container=False)
378
+ )
379
+ with gr.Column():
380
+ with gr.Row():
381
+ change = gr.Button("Change System Prompt")
382
+ reset = gr.Button("Reset System Prompt")
383
+
384
+ with gr.Accordion("Example Inputs", open=True):
385
+ examples = gr.Examples(
386
+ examples=examples_list,
387
+ inputs=[msg],
388
+ examples_per_page=40,
389
+ )
390
+
391
+ # with gr.Row():
392
+ with gr.Accordion("Disclaimer", open=False):
393
+ _ = Path(model_loc).name
394
+ gr.Markdown(
395
+ f"Disclaimer: {_} can produce factually incorrect output, and should not be relied on to produce "
396
+ "factually accurate information. {_} was trained on various public datasets; while great efforts "
397
+ "have been taken to clean the pretraining data, it is possible that this model could generate lewd, "
398
+ "biased, or otherwise offensive outputs.",
399
+ elem_classes=["disclaimer"],
400
+ )
401
+
402
+ msg_submit_event = msg.submit(
403
+ # fn=conversation.user_turn,
404
+ fn=user,
405
+ inputs=[msg, chatbot],
406
+ outputs=[msg, chatbot],
407
+ queue=True,
408
+ show_progress="full",
409
+ # api_name=None,
410
+ ).then(bot, chatbot, chatbot, queue=True)
411
+ submit_click_event = submit.click(
412
+ # fn=lambda x, y: ("",) + user(x, y)[1:], # clear msg
413
+ fn=user1, # clear msg
414
+ inputs=[msg, chatbot],
415
+ outputs=[msg, chatbot],
416
+ queue=True,
417
+ # queue=False,
418
+ show_progress="full",
419
+ # api_name=None,
420
+ ).then(bot, chatbot, chatbot, queue=True)
421
+ stop.click(
422
+ fn=None,
423
+ inputs=None,
424
+ outputs=None,
425
+ cancels=[msg_submit_event, submit_click_event],
426
+ queue=False,
427
+ )
428
+ clear.click(lambda: None, None, chatbot, queue=False)
429
+
430
+ with gr.Accordion("For Chat/Translation API", open=False, visible=False):
431
+ input_text = gr.Text()
432
+ api_btn = gr.Button("Go", variant="primary")
433
+ out_text = gr.Text()
434
+
435
+ api_btn.click(
436
+ predict_api,
437
+ input_text,
438
+ out_text,
439
+ api_name="api",
440
+ )
441
+
442
+ # block.load(update_buff, [], buff, every=1)
443
+ # block.load(update_buff, [buff_var], [buff_var, buff], every=1)
444
+
445
+ # concurrency_count=5, max_size=20
446
+ # max_size=36, concurrency_count=14
447
+ # CPU cpu_count=2 16G, model 7G
448
+ # CPU UPGRADE cpu_count=8 32G, model 7G
449
+
450
+ # does not work
451
+ _ = """
452
+ # _ = int(psutil.virtual_memory().total / 10**9 // file_size - 1)
453
+ # concurrency_count = max(_, 1)
454
+ if psutil.cpu_count(logical=False) >= 8:
455
+ # concurrency_count = max(int(32 / file_size) - 1, 1)
456
+ else:
457
+ # concurrency_count = max(int(16 / file_size) - 1, 1)
458
+ # """
459
+
460
+ concurrency_count = 1
461
+ logger.info(f"{concurrency_count=}")
462
+
463
+ block.queue(concurrency_count=concurrency_count, max_size=5).launch(debug=True)
requirements.txt ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ ctransformers # ==0.2.10 0.2.13
2
+ transformers # ==4.30.2
3
+ # huggingface_hub
4
+ gradio
5
+ loguru
6
+ about-time
7
+ psutil
8
+ dl-hf-model
run-app.sh ADDED
@@ -0,0 +1 @@
 
 
1
+ nodemon -w app.py -x python app.py