kevinwang676 OedoSoldier commited on
Commit
60d3fa3
0 Parent(s):

Duplicate from OedoSoldier/chatglm_int4_demo

Browse files

Co-authored-by: OedoSoldier <OedoSoldier@users.noreply.huggingface.co>

Files changed (6) hide show
  1. .gitattributes +34 -0
  2. LICENSE +28 -0
  3. README.md +14 -0
  4. app.py +193 -0
  5. config.json +6 -0
  6. requirements.txt +6 -0
.gitattributes ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tflite filter=lfs diff=lfs merge=lfs -text
29
+ *.tgz filter=lfs diff=lfs merge=lfs -text
30
+ *.wasm filter=lfs diff=lfs merge=lfs -text
31
+ *.xz filter=lfs diff=lfs merge=lfs -text
32
+ *.zip filter=lfs diff=lfs merge=lfs -text
33
+ *.zst filter=lfs diff=lfs merge=lfs -text
34
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
LICENSE ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ BSD 3-Clause License
2
+
3
+ Copyright (c) 2023, OedoSoldier
4
+
5
+ Redistribution and use in source and binary forms, with or without
6
+ modification, are permitted provided that the following conditions are met:
7
+
8
+ 1. Redistributions of source code must retain the above copyright notice, this
9
+ list of conditions and the following disclaimer.
10
+
11
+ 2. Redistributions in binary form must reproduce the above copyright notice,
12
+ this list of conditions and the following disclaimer in the documentation
13
+ and/or other materials provided with the distribution.
14
+
15
+ 3. Neither the name of the copyright holder nor the names of its
16
+ contributors may be used to endorse or promote products derived from
17
+ this software without specific prior written permission.
18
+
19
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
23
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
26
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
README.md ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: ChatGLM (int-4) Demo
3
+ emoji: 😋
4
+ colorFrom: red
5
+ colorTo: yellow
6
+ sdk: gradio
7
+ sdk_version: 3.21.0
8
+ app_file: app.py
9
+ models:
10
+ - THUDM/chatglm-6b
11
+ - THUDM/chatglm-6b-int4
12
+ pinned: true
13
+ duplicated_from: OedoSoldier/chatglm_int4_demo
14
+ ---
app.py ADDED
@@ -0,0 +1,193 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import json
3
+ import datetime
4
+ import gradio as gr
5
+ import torch
6
+ from transformers import AutoTokenizer, AutoModel
7
+
8
+
9
+ if torch.cuda.is_available():
10
+ device = 'cuda'
11
+ else:
12
+ device = 'cpu'
13
+
14
+ tokenizer = AutoTokenizer.from_pretrained('THUDM/chatglm-6b-int4', trust_remote_code=True)
15
+
16
+ if device == 'cuda':
17
+ model = AutoModel.from_pretrained('THUDM/chatglm-6b-int4', trust_remote_code=True).half().cuda()
18
+ else:
19
+ model = AutoModel.from_pretrained('THUDM/chatglm-6b-int4', trust_remote_code=True).float()
20
+
21
+ model = model.eval()
22
+
23
+
24
+ def parse_text(text):
25
+ lines = text.split('\n')
26
+ for i, line in enumerate(lines):
27
+ if '```' in line:
28
+ item = line.split('`')[-1]
29
+ if item:
30
+ lines[i] = f'<pre><code class="{item}">'
31
+ else:
32
+ lines[i] = '</code></pre>'
33
+ else:
34
+ if i > 0:
35
+ line = line.replace('<', '&lt;').replace('>', '&gt;')
36
+ lines[i] = f'<br/>{line}'
37
+ return ''.join(lines)
38
+
39
+
40
+ def chat_wrapper(query, styled_history, history, max_length, top_p, temperature, memory_limit):
41
+ if query == '':
42
+ return [], [], '', *gr_hide()
43
+ if memory_limit == 0:
44
+ history = []
45
+ styled_history = []
46
+ elif memory_limit > 0:
47
+ history = history[-memory_limit:]
48
+ styled_history = styled_history[-memory_limit:]
49
+ flag = True
50
+ styled_history_pos = 0
51
+ for message, history in model.stream_chat(tokenizer, query, history=history,
52
+ max_length=max_length, top_p=top_p, temperature=temperature):
53
+ if flag:
54
+ styled_history.append((parse_text(query), parse_text(message)))
55
+ styled_history_pos = len(styled_history) - 1
56
+ flag = False
57
+ else:
58
+ styled_history[styled_history_pos] = (parse_text(query), parse_text(message))
59
+ yield styled_history, history, '', *gr_hide()
60
+
61
+
62
+ def regenerate_wrapper(styled_history, history, max_length, top_p, temperature, memory_limit):
63
+ if not history:
64
+ return [], [], '', *gr_hide()
65
+
66
+ styled_history, history, query, _, _, _ = edit_wrapper(styled_history, history)
67
+ for ret in chat_wrapper(query, styled_history, history, max_length, top_p, temperature, memory_limit):
68
+ yield ret
69
+
70
+
71
+ def edit_wrapper(styled_history, history):
72
+ if len(history) == 0:
73
+ return [], [], ''
74
+ query = history[-1][0]
75
+ history = history[:-1]
76
+ styled_history = styled_history[:-1]
77
+ return styled_history, history, query, *gr_hide()
78
+
79
+
80
+ def reset_history():
81
+ return [], [], '', *gr_hide()
82
+
83
+
84
+ def save_history(history):
85
+ os.makedirs('log', exist_ok=True)
86
+ dict_list = [{'input': q, 'output': a} for q, a in history]
87
+ with open(f'log/{datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}.json', 'w', encoding='utf-8') as f:
88
+ json.dump(dict_list, f, ensure_ascii=False, indent=2)
89
+
90
+
91
+ def save_config(max_length=2048, top_p=0.7, temperature=0.95, memory_limit=-1.0):
92
+ with open('config.json', 'w') as f:
93
+ json.dump({'max_length': max_length, 'top_p': top_p, 'temperature': temperature, 'memory_limit': memory_limit}, f, indent=2)
94
+
95
+
96
+ def load_history(file, styled_history, history):
97
+ current_styled_history, current_history = styled_history.copy(), history.copy()
98
+ try:
99
+ with open(file.name, 'r', encoding='utf-8') as f:
100
+ dict_list = json.load(f)
101
+ history = [(item['input'], item['output']) for item in dict_list]
102
+ styled_history = [(parse_text(item['input']), parse_text(item['output'])) for item in dict_list]
103
+ except BaseException:
104
+ return current_styled_history, current_history, ''
105
+ return styled_history, history, '', *gr_hide()
106
+
107
+
108
+ def gr_show_and_load(history, evt: gr.SelectData):
109
+ if evt.index[1] == 0:
110
+ label = f'Editing Q{evt.index[0]}:'
111
+ else:
112
+ label = f'Editing Q{evt.index[0]}:'
113
+ return {'visible': True, '__type__': 'update'}, {'value': history[evt.index[0]][evt.index[1]], 'label': label, '__type__': 'update'}, evt.index
114
+
115
+
116
+ def update_history(styled_history, history, log, idx):
117
+ if log == '':
118
+ return styled_history, history, {'visible': True, '__type__': 'update'}, {'value': history[idx[0]][idx[1]], '__type__': 'update'}, idx
119
+
120
+ def swap_value(lst, idx, value):
121
+ lst[idx[0]] = tuple(value if j == idx[1] else elem for j, elem in enumerate(lst[idx[0]]))
122
+ return lst
123
+ styled_history = swap_value(styled_history, idx, parse_text(log))
124
+ history = swap_value(history, idx, log)
125
+ return styled_history, history, *gr_hide()
126
+
127
+
128
+ def gr_hide():
129
+ return {'visible': False, '__type__': 'update'}, {'value': '', 'label': '', '__type__': 'update'}, []
130
+
131
+
132
+ with gr.Blocks() as demo:
133
+ if not os.path.isfile('config.json'):
134
+ save_config()
135
+
136
+ with open('config.json', 'r', encoding='utf-8') as f:
137
+ configs = json.loads(f.read())
138
+
139
+ gr.Markdown('''<h1><center>ChatGLM Demo</center></h1>''')
140
+
141
+ with gr.Row():
142
+ max_length = gr.Slider(minimum=4.0, maximum=4096.0, step=4.0, label='Max Length', value=configs['max_length'])
143
+ top_p = gr.Slider(minimum=0.01, maximum=1.0, step=0.01, label='Top P', value=configs['top_p'])
144
+ temperature = gr.Slider(minimum=0.01, maximum=2.0, step=0.01, label='Temperature', value=configs['temperature'])
145
+ memory_limit = gr.Slider(minimum=-1.0, maximum=20.0, step=1.0, label='Memory Limit', value=configs['memory_limit'])
146
+ save_conf = gr.Button('保存设置', visible=False)
147
+
148
+ gr.Markdown('''<h2>Hint: click on a chat bubble to edit chat history</h2>''')
149
+
150
+ state = gr.State([])
151
+ chatbot = gr.Chatbot(elem_id='chatbot', show_label=False)
152
+ with gr.Row(visible=False) as edit_log:
153
+ with gr.Column():
154
+ log = gr.Textbox()
155
+ with gr.Row():
156
+ submit_log = gr.Button('保存')
157
+ cancel_log = gr.Button('取消')
158
+ log_idx = gr.State([])
159
+
160
+ message = gr.Textbox(placeholder='Input your message', label='Q:')
161
+
162
+ with gr.Row():
163
+ submit = gr.Button('Submit')
164
+ edit = gr.Button('Edit last question')
165
+ regen = gr.Button('Re-generate')
166
+
167
+ delete = gr.Button('Reset chat')
168
+
169
+ with gr.Row(visible=False):
170
+ save = gr.Button('保存对话(在 `log` 文件夹下)')
171
+ load = gr.UploadButton('读取对话', file_types=['file'], file_count='single')
172
+
173
+ input_list = [message, chatbot, state, max_length, top_p, temperature, memory_limit]
174
+ output_list = [chatbot, state, message]
175
+ edit_list = [edit_log, log, log_idx]
176
+
177
+ save_conf.click(save_config, inputs=input_list[3:])
178
+ load.upload(load_history, inputs=[load, chatbot, state], outputs=output_list + edit_list)
179
+ save.click(save_history, inputs=[state])
180
+ message.submit(chat_wrapper, inputs=input_list, outputs=output_list + edit_list)
181
+ submit.click(chat_wrapper, inputs=input_list, outputs=output_list + edit_list)
182
+ edit.click(edit_wrapper, inputs=input_list[1:3], outputs=output_list + edit_list)
183
+ regen.click(regenerate_wrapper, inputs=input_list[1:], outputs=output_list + edit_list)
184
+ delete.click(reset_history, outputs=output_list + edit_list)
185
+ chatbot.select(gr_show_and_load, inputs=[state], outputs=edit_list)
186
+ edit_kwargs = {'inputs': [chatbot, state, log, log_idx], 'outputs': [chatbot, state] + edit_list}
187
+ log.submit(update_history, **edit_kwargs)
188
+ submit_log.click(update_history, **edit_kwargs)
189
+ cancel_log.click(gr_hide, outputs=edit_list)
190
+
191
+
192
+ if __name__ == '__main__':
193
+ demo.queue(concurrency_count=5, max_size=20).launch(debug=True)
config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "max_length": 2048,
3
+ "top_p": 0.7,
4
+ "temperature": 0.95,
5
+ "memory_limit": -1
6
+ }
requirements.txt ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ protobuf>=3.19.5,<3.20.1
2
+ transformers>=4.26.1
3
+ icetk
4
+ cpm_kernels
5
+ torch>=1.10
6
+ gradio>=3.21.0