Commit
•
60d3fa3
0
Parent(s):
Duplicate from OedoSoldier/chatglm_int4_demo
Browse filesCo-authored-by: OedoSoldier <OedoSoldier@users.noreply.huggingface.co>
- .gitattributes +34 -0
- LICENSE +28 -0
- README.md +14 -0
- app.py +193 -0
- config.json +6 -0
- requirements.txt +6 -0
.gitattributes
ADDED
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
28 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
29 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
30 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
31 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
32 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
33 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
34 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
LICENSE
ADDED
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
BSD 3-Clause License
|
2 |
+
|
3 |
+
Copyright (c) 2023, OedoSoldier
|
4 |
+
|
5 |
+
Redistribution and use in source and binary forms, with or without
|
6 |
+
modification, are permitted provided that the following conditions are met:
|
7 |
+
|
8 |
+
1. Redistributions of source code must retain the above copyright notice, this
|
9 |
+
list of conditions and the following disclaimer.
|
10 |
+
|
11 |
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
12 |
+
this list of conditions and the following disclaimer in the documentation
|
13 |
+
and/or other materials provided with the distribution.
|
14 |
+
|
15 |
+
3. Neither the name of the copyright holder nor the names of its
|
16 |
+
contributors may be used to endorse or promote products derived from
|
17 |
+
this software without specific prior written permission.
|
18 |
+
|
19 |
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
20 |
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
21 |
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
22 |
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
23 |
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
24 |
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
25 |
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
26 |
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
27 |
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
28 |
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
README.md
ADDED
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
---
|
2 |
+
title: ChatGLM (int-4) Demo
|
3 |
+
emoji: 😋
|
4 |
+
colorFrom: red
|
5 |
+
colorTo: yellow
|
6 |
+
sdk: gradio
|
7 |
+
sdk_version: 3.21.0
|
8 |
+
app_file: app.py
|
9 |
+
models:
|
10 |
+
- THUDM/chatglm-6b
|
11 |
+
- THUDM/chatglm-6b-int4
|
12 |
+
pinned: true
|
13 |
+
duplicated_from: OedoSoldier/chatglm_int4_demo
|
14 |
+
---
|
app.py
ADDED
@@ -0,0 +1,193 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import os
|
2 |
+
import json
|
3 |
+
import datetime
|
4 |
+
import gradio as gr
|
5 |
+
import torch
|
6 |
+
from transformers import AutoTokenizer, AutoModel
|
7 |
+
|
8 |
+
|
9 |
+
if torch.cuda.is_available():
|
10 |
+
device = 'cuda'
|
11 |
+
else:
|
12 |
+
device = 'cpu'
|
13 |
+
|
14 |
+
tokenizer = AutoTokenizer.from_pretrained('THUDM/chatglm-6b-int4', trust_remote_code=True)
|
15 |
+
|
16 |
+
if device == 'cuda':
|
17 |
+
model = AutoModel.from_pretrained('THUDM/chatglm-6b-int4', trust_remote_code=True).half().cuda()
|
18 |
+
else:
|
19 |
+
model = AutoModel.from_pretrained('THUDM/chatglm-6b-int4', trust_remote_code=True).float()
|
20 |
+
|
21 |
+
model = model.eval()
|
22 |
+
|
23 |
+
|
24 |
+
def parse_text(text):
|
25 |
+
lines = text.split('\n')
|
26 |
+
for i, line in enumerate(lines):
|
27 |
+
if '```' in line:
|
28 |
+
item = line.split('`')[-1]
|
29 |
+
if item:
|
30 |
+
lines[i] = f'<pre><code class="{item}">'
|
31 |
+
else:
|
32 |
+
lines[i] = '</code></pre>'
|
33 |
+
else:
|
34 |
+
if i > 0:
|
35 |
+
line = line.replace('<', '<').replace('>', '>')
|
36 |
+
lines[i] = f'<br/>{line}'
|
37 |
+
return ''.join(lines)
|
38 |
+
|
39 |
+
|
40 |
+
def chat_wrapper(query, styled_history, history, max_length, top_p, temperature, memory_limit):
|
41 |
+
if query == '':
|
42 |
+
return [], [], '', *gr_hide()
|
43 |
+
if memory_limit == 0:
|
44 |
+
history = []
|
45 |
+
styled_history = []
|
46 |
+
elif memory_limit > 0:
|
47 |
+
history = history[-memory_limit:]
|
48 |
+
styled_history = styled_history[-memory_limit:]
|
49 |
+
flag = True
|
50 |
+
styled_history_pos = 0
|
51 |
+
for message, history in model.stream_chat(tokenizer, query, history=history,
|
52 |
+
max_length=max_length, top_p=top_p, temperature=temperature):
|
53 |
+
if flag:
|
54 |
+
styled_history.append((parse_text(query), parse_text(message)))
|
55 |
+
styled_history_pos = len(styled_history) - 1
|
56 |
+
flag = False
|
57 |
+
else:
|
58 |
+
styled_history[styled_history_pos] = (parse_text(query), parse_text(message))
|
59 |
+
yield styled_history, history, '', *gr_hide()
|
60 |
+
|
61 |
+
|
62 |
+
def regenerate_wrapper(styled_history, history, max_length, top_p, temperature, memory_limit):
|
63 |
+
if not history:
|
64 |
+
return [], [], '', *gr_hide()
|
65 |
+
|
66 |
+
styled_history, history, query, _, _, _ = edit_wrapper(styled_history, history)
|
67 |
+
for ret in chat_wrapper(query, styled_history, history, max_length, top_p, temperature, memory_limit):
|
68 |
+
yield ret
|
69 |
+
|
70 |
+
|
71 |
+
def edit_wrapper(styled_history, history):
|
72 |
+
if len(history) == 0:
|
73 |
+
return [], [], ''
|
74 |
+
query = history[-1][0]
|
75 |
+
history = history[:-1]
|
76 |
+
styled_history = styled_history[:-1]
|
77 |
+
return styled_history, history, query, *gr_hide()
|
78 |
+
|
79 |
+
|
80 |
+
def reset_history():
|
81 |
+
return [], [], '', *gr_hide()
|
82 |
+
|
83 |
+
|
84 |
+
def save_history(history):
|
85 |
+
os.makedirs('log', exist_ok=True)
|
86 |
+
dict_list = [{'input': q, 'output': a} for q, a in history]
|
87 |
+
with open(f'log/{datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}.json', 'w', encoding='utf-8') as f:
|
88 |
+
json.dump(dict_list, f, ensure_ascii=False, indent=2)
|
89 |
+
|
90 |
+
|
91 |
+
def save_config(max_length=2048, top_p=0.7, temperature=0.95, memory_limit=-1.0):
|
92 |
+
with open('config.json', 'w') as f:
|
93 |
+
json.dump({'max_length': max_length, 'top_p': top_p, 'temperature': temperature, 'memory_limit': memory_limit}, f, indent=2)
|
94 |
+
|
95 |
+
|
96 |
+
def load_history(file, styled_history, history):
|
97 |
+
current_styled_history, current_history = styled_history.copy(), history.copy()
|
98 |
+
try:
|
99 |
+
with open(file.name, 'r', encoding='utf-8') as f:
|
100 |
+
dict_list = json.load(f)
|
101 |
+
history = [(item['input'], item['output']) for item in dict_list]
|
102 |
+
styled_history = [(parse_text(item['input']), parse_text(item['output'])) for item in dict_list]
|
103 |
+
except BaseException:
|
104 |
+
return current_styled_history, current_history, ''
|
105 |
+
return styled_history, history, '', *gr_hide()
|
106 |
+
|
107 |
+
|
108 |
+
def gr_show_and_load(history, evt: gr.SelectData):
|
109 |
+
if evt.index[1] == 0:
|
110 |
+
label = f'Editing Q{evt.index[0]}:'
|
111 |
+
else:
|
112 |
+
label = f'Editing Q{evt.index[0]}:'
|
113 |
+
return {'visible': True, '__type__': 'update'}, {'value': history[evt.index[0]][evt.index[1]], 'label': label, '__type__': 'update'}, evt.index
|
114 |
+
|
115 |
+
|
116 |
+
def update_history(styled_history, history, log, idx):
|
117 |
+
if log == '':
|
118 |
+
return styled_history, history, {'visible': True, '__type__': 'update'}, {'value': history[idx[0]][idx[1]], '__type__': 'update'}, idx
|
119 |
+
|
120 |
+
def swap_value(lst, idx, value):
|
121 |
+
lst[idx[0]] = tuple(value if j == idx[1] else elem for j, elem in enumerate(lst[idx[0]]))
|
122 |
+
return lst
|
123 |
+
styled_history = swap_value(styled_history, idx, parse_text(log))
|
124 |
+
history = swap_value(history, idx, log)
|
125 |
+
return styled_history, history, *gr_hide()
|
126 |
+
|
127 |
+
|
128 |
+
def gr_hide():
|
129 |
+
return {'visible': False, '__type__': 'update'}, {'value': '', 'label': '', '__type__': 'update'}, []
|
130 |
+
|
131 |
+
|
132 |
+
with gr.Blocks() as demo:
|
133 |
+
if not os.path.isfile('config.json'):
|
134 |
+
save_config()
|
135 |
+
|
136 |
+
with open('config.json', 'r', encoding='utf-8') as f:
|
137 |
+
configs = json.loads(f.read())
|
138 |
+
|
139 |
+
gr.Markdown('''<h1><center>ChatGLM Demo</center></h1>''')
|
140 |
+
|
141 |
+
with gr.Row():
|
142 |
+
max_length = gr.Slider(minimum=4.0, maximum=4096.0, step=4.0, label='Max Length', value=configs['max_length'])
|
143 |
+
top_p = gr.Slider(minimum=0.01, maximum=1.0, step=0.01, label='Top P', value=configs['top_p'])
|
144 |
+
temperature = gr.Slider(minimum=0.01, maximum=2.0, step=0.01, label='Temperature', value=configs['temperature'])
|
145 |
+
memory_limit = gr.Slider(minimum=-1.0, maximum=20.0, step=1.0, label='Memory Limit', value=configs['memory_limit'])
|
146 |
+
save_conf = gr.Button('保存设置', visible=False)
|
147 |
+
|
148 |
+
gr.Markdown('''<h2>Hint: click on a chat bubble to edit chat history</h2>''')
|
149 |
+
|
150 |
+
state = gr.State([])
|
151 |
+
chatbot = gr.Chatbot(elem_id='chatbot', show_label=False)
|
152 |
+
with gr.Row(visible=False) as edit_log:
|
153 |
+
with gr.Column():
|
154 |
+
log = gr.Textbox()
|
155 |
+
with gr.Row():
|
156 |
+
submit_log = gr.Button('保存')
|
157 |
+
cancel_log = gr.Button('取消')
|
158 |
+
log_idx = gr.State([])
|
159 |
+
|
160 |
+
message = gr.Textbox(placeholder='Input your message', label='Q:')
|
161 |
+
|
162 |
+
with gr.Row():
|
163 |
+
submit = gr.Button('Submit')
|
164 |
+
edit = gr.Button('Edit last question')
|
165 |
+
regen = gr.Button('Re-generate')
|
166 |
+
|
167 |
+
delete = gr.Button('Reset chat')
|
168 |
+
|
169 |
+
with gr.Row(visible=False):
|
170 |
+
save = gr.Button('保存对话(在 `log` 文件夹下)')
|
171 |
+
load = gr.UploadButton('读取对话', file_types=['file'], file_count='single')
|
172 |
+
|
173 |
+
input_list = [message, chatbot, state, max_length, top_p, temperature, memory_limit]
|
174 |
+
output_list = [chatbot, state, message]
|
175 |
+
edit_list = [edit_log, log, log_idx]
|
176 |
+
|
177 |
+
save_conf.click(save_config, inputs=input_list[3:])
|
178 |
+
load.upload(load_history, inputs=[load, chatbot, state], outputs=output_list + edit_list)
|
179 |
+
save.click(save_history, inputs=[state])
|
180 |
+
message.submit(chat_wrapper, inputs=input_list, outputs=output_list + edit_list)
|
181 |
+
submit.click(chat_wrapper, inputs=input_list, outputs=output_list + edit_list)
|
182 |
+
edit.click(edit_wrapper, inputs=input_list[1:3], outputs=output_list + edit_list)
|
183 |
+
regen.click(regenerate_wrapper, inputs=input_list[1:], outputs=output_list + edit_list)
|
184 |
+
delete.click(reset_history, outputs=output_list + edit_list)
|
185 |
+
chatbot.select(gr_show_and_load, inputs=[state], outputs=edit_list)
|
186 |
+
edit_kwargs = {'inputs': [chatbot, state, log, log_idx], 'outputs': [chatbot, state] + edit_list}
|
187 |
+
log.submit(update_history, **edit_kwargs)
|
188 |
+
submit_log.click(update_history, **edit_kwargs)
|
189 |
+
cancel_log.click(gr_hide, outputs=edit_list)
|
190 |
+
|
191 |
+
|
192 |
+
if __name__ == '__main__':
|
193 |
+
demo.queue(concurrency_count=5, max_size=20).launch(debug=True)
|
config.json
ADDED
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"max_length": 2048,
|
3 |
+
"top_p": 0.7,
|
4 |
+
"temperature": 0.95,
|
5 |
+
"memory_limit": -1
|
6 |
+
}
|
requirements.txt
ADDED
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
protobuf>=3.19.5,<3.20.1
|
2 |
+
transformers>=4.26.1
|
3 |
+
icetk
|
4 |
+
cpm_kernels
|
5 |
+
torch>=1.10
|
6 |
+
gradio>=3.21.0
|