paralym commited on
Commit
8d0adf7
1 Parent(s): 85b5788

Upload 70 files

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +7 -0
  2. README.md +8 -10
  3. app.py +535 -0
  4. examples/.DS_Store +0 -0
  5. examples/172197131626056_P7966202.png +0 -0
  6. examples/A-17-processors-1024x576.jpg +0 -0
  7. examples/Iphone-15-Usb-c-charger-1024x576.jpg +0 -0
  8. examples/Iphone-15-specs-1024x576.jpg +0 -0
  9. examples/africa.jpg +0 -0
  10. examples/ballon.jpg +0 -0
  11. examples/bar.jpg +0 -0
  12. examples/bigcompany.png +3 -0
  13. examples/bijiasuo2.jpeg +0 -0
  14. examples/book.jpg +0 -0
  15. examples/camera.jpg +0 -0
  16. examples/changed_bench.jpeg +0 -0
  17. examples/code.mp4 +0 -0
  18. examples/code1.jpeg +0 -0
  19. examples/code2.jpeg +0 -0
  20. examples/dog.jpg +0 -0
  21. examples/dog1.jpg +0 -0
  22. examples/dog6.jpeg +0 -0
  23. examples/dog9.jpeg +0 -0
  24. examples/dog_to_monkey1.png +3 -0
  25. examples/dog_to_monkey2.png +3 -0
  26. examples/dynamic-island-1024x576.jpg +0 -0
  27. examples/eagles.jpg +0 -0
  28. examples/ecg_example1.jpg +0 -0
  29. examples/ecg_example2.png +0 -0
  30. examples/examples_image12.jpg +0 -0
  31. examples/examples_image13.jpg +0 -0
  32. examples/examples_image14.jpg +0 -0
  33. examples/fangao1.jpeg +0 -0
  34. examples/fangao2.jpeg +0 -0
  35. examples/fangao3.jpeg +0 -0
  36. examples/food.jpg +0 -0
  37. examples/girl.jpg +0 -0
  38. examples/hanzi.jpg +0 -0
  39. examples/hot_ballon.jpg +0 -0
  40. examples/ice_cream.jpg +0 -0
  41. examples/image-00007.jpeg +0 -0
  42. examples/image-00053.jpeg +0 -0
  43. examples/iphone-15-colors-1024x576.jpg +0 -0
  44. examples/iphone-15-price-1024x576.jpg +0 -0
  45. examples/iphone-15-pricing-1024x576.jpg +0 -0
  46. examples/line chart.jpg +0 -0
  47. examples/norway.jpg +0 -0
  48. examples/oprah-winfrey-resume.png +0 -0
  49. examples/orange.png +0 -0
  50. examples/original_bench.jpeg +0 -0
.gitattributes CHANGED
@@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ examples/bigcompany.png filter=lfs diff=lfs merge=lfs -text
37
+ examples/dog_to_monkey1.png filter=lfs diff=lfs merge=lfs -text
38
+ examples/dog_to_monkey2.png filter=lfs diff=lfs merge=lfs -text
39
+ examples/twitter2.jpeg filter=lfs diff=lfs merge=lfs -text
40
+ examples/twitter3.jpeg filter=lfs diff=lfs merge=lfs -text
41
+ examples/twitter4.jpeg filter=lfs diff=lfs merge=lfs -text
42
+ examples/user_example_07.jpg filter=lfs diff=lfs merge=lfs -text
README.md CHANGED
@@ -1,14 +1,12 @@
1
  ---
2
- title: PULSE Debug
3
- emoji: 📈
4
- colorFrom: gray
5
- colorTo: blue
6
  sdk: gradio
7
- sdk_version: 5.4.0
8
  app_file: app.py
9
- pinned: false
10
- license: apache-2.0
11
- short_description: ECG
12
  ---
13
-
14
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
1
  ---
2
+ title: Pangea
3
+ emoji: 🚀
4
+ colorFrom: green
5
+ colorTo: red
6
  sdk: gradio
7
+ sdk_version: 4.37.2
8
  app_file: app.py
9
+ pinned: true
10
+ short_description: A Fully Open Multilingual Multimodal LLM for 39 Languages
 
11
  ---
12
+ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
app.py ADDED
@@ -0,0 +1,535 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # from .demo_modelpart import InferenceDemo
2
+ import gradio as gr
3
+ import os
4
+ from threading import Thread
5
+
6
+ # import time
7
+ import cv2
8
+
9
+ import datetime
10
+ # import copy
11
+ import torch
12
+
13
+ import spaces
14
+ import numpy as np
15
+
16
+ from llava import conversation as conversation_lib
17
+ from llava.constants import DEFAULT_IMAGE_TOKEN
18
+
19
+
20
+ from llava.constants import (
21
+ IMAGE_TOKEN_INDEX,
22
+ DEFAULT_IMAGE_TOKEN,
23
+ DEFAULT_IM_START_TOKEN,
24
+ DEFAULT_IM_END_TOKEN,
25
+ )
26
+ from llava.conversation import conv_templates, SeparatorStyle
27
+ from llava.model.builder import load_pretrained_model
28
+ from llava.utils import disable_torch_init
29
+ from llava.mm_utils import (
30
+ tokenizer_image_token,
31
+ process_images,
32
+ get_model_name_from_path,
33
+ KeywordsStoppingCriteria,
34
+ )
35
+
36
+ from serve_constants import html_header, bibtext, learn_more_markdown, tos_markdown
37
+
38
+ import requests
39
+ from PIL import Image
40
+ from io import BytesIO
41
+ from transformers import TextStreamer, TextIteratorStreamer
42
+
43
+ import hashlib
44
+ import PIL
45
+ import base64
46
+ import json
47
+
48
+ import datetime
49
+ import gradio as gr
50
+ import gradio_client
51
+ import subprocess
52
+ import sys
53
+
54
+ from huggingface_hub import HfApi
55
+ from huggingface_hub import login
56
+ from huggingface_hub import revision_exists
57
+
58
+ login(token=os.environ["HF_TOKEN"],
59
+ write_permission=True)
60
+
61
+ api = HfApi()
62
+ repo_name = os.environ["LOG_REPO"]
63
+
64
+ external_log_dir = "./logs"
65
+ LOGDIR = external_log_dir
66
+
67
+
68
+ def install_gradio_4_35_0():
69
+ current_version = gr.__version__
70
+ if current_version != "4.35.0":
71
+ print(f"Current Gradio version: {current_version}")
72
+ print("Installing Gradio 4.35.0...")
73
+ subprocess.check_call([sys.executable, "-m", "pip", "install", "gradio==4.35.0", "--force-reinstall"])
74
+ print("Gradio 4.35.0 installed successfully.")
75
+ else:
76
+ print("Gradio 4.35.0 is already installed.")
77
+
78
+ # Call the function to install Gradio 4.35.0 if needed
79
+ install_gradio_4_35_0()
80
+
81
+ import gradio as gr
82
+ import gradio_client
83
+ print(f"Gradio version: {gr.__version__}")
84
+ print(f"Gradio-client version: {gradio_client.__version__}")
85
+
86
+ def get_conv_log_filename():
87
+ t = datetime.datetime.now()
88
+ name = os.path.join(LOGDIR, f"{t.year}-{t.month:02d}-{t.day:02d}-user_conv.json")
89
+ return name
90
+
91
+ class InferenceDemo(object):
92
+ def __init__(
93
+ self, args, model_path, tokenizer, model, image_processor, context_len
94
+ ) -> None:
95
+ disable_torch_init()
96
+
97
+ self.tokenizer, self.model, self.image_processor, self.context_len = (
98
+ tokenizer,
99
+ model,
100
+ image_processor,
101
+ context_len,
102
+ )
103
+
104
+ if "llama-2" in model_name.lower():
105
+ conv_mode = "llava_llama_2"
106
+ elif "v1" in model_name.lower() or "pulse" in model_name.lower():
107
+ conv_mode = "llava_v1"
108
+ elif "mpt" in model_name.lower():
109
+ conv_mode = "mpt"
110
+ elif "qwen" in model_name.lower():
111
+ conv_mode = "qwen_1_5"
112
+ else:
113
+ conv_mode = "llava_v0"
114
+
115
+ if args.conv_mode is not None and conv_mode != args.conv_mode:
116
+ print(
117
+ "[WARNING] the auto inferred conversation mode is {}, while `--conv-mode` is {}, using {}".format(
118
+ conv_mode, args.conv_mode, args.conv_mode
119
+ )
120
+ )
121
+ else:
122
+ args.conv_mode = conv_mode
123
+ self.conv_mode = conv_mode
124
+ self.conversation = conv_templates[args.conv_mode].copy()
125
+ self.num_frames = args.num_frames
126
+
127
+
128
+ def is_valid_video_filename(name):
129
+ video_extensions = ["avi", "mp4", "mov", "mkv", "flv", "wmv", "mjpeg"]
130
+
131
+ ext = name.split(".")[-1].lower()
132
+
133
+ if ext in video_extensions:
134
+ return True
135
+ else:
136
+ return False
137
+
138
+ def is_valid_image_filename(name):
139
+ image_extensions = ["jpg", "jpeg", "png", "bmp", "gif", "tiff", "webp", "heic", "heif", "jfif", "svg", "eps", "raw"]
140
+
141
+ ext = name.split(".")[-1].lower()
142
+
143
+ if ext in image_extensions:
144
+ return True
145
+ else:
146
+ return False
147
+
148
+
149
+ def sample_frames(video_file, num_frames):
150
+ video = cv2.VideoCapture(video_file)
151
+ total_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
152
+ interval = total_frames // num_frames
153
+ frames = []
154
+ for i in range(total_frames):
155
+ ret, frame = video.read()
156
+ pil_img = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
157
+ if not ret:
158
+ continue
159
+ if i % interval == 0:
160
+ frames.append(pil_img)
161
+ video.release()
162
+ return frames
163
+
164
+
165
+ def load_image(image_file):
166
+ if image_file.startswith("http") or image_file.startswith("https"):
167
+ response = requests.get(image_file)
168
+ if response.status_code == 200:
169
+ image = Image.open(BytesIO(response.content)).convert("RGB")
170
+ else:
171
+ print("failed to load the image")
172
+ else:
173
+ print("Load image from local file")
174
+ print(image_file)
175
+ image = Image.open(image_file).convert("RGB")
176
+
177
+ return image
178
+
179
+
180
+ def clear_history(history):
181
+
182
+ our_chatbot.conversation = conv_templates[our_chatbot.conv_mode].copy()
183
+
184
+ return None
185
+
186
+
187
+ def clear_response(history):
188
+ for index_conv in range(1, len(history)):
189
+ # loop until get a text response from our model.
190
+ conv = history[-index_conv]
191
+ if not (conv[0] is None):
192
+ break
193
+ question = history[-index_conv][0]
194
+ history = history[:-index_conv]
195
+ return history, question
196
+
197
+
198
+ # def print_like_dislike(x: gr.LikeData):
199
+ # print(x.index, x.value, x.liked)
200
+
201
+
202
+ def add_message(history, message):
203
+ # history=[]
204
+ global our_chatbot
205
+ if len(history) == 0:
206
+ our_chatbot = InferenceDemo(
207
+ args, model_path, tokenizer, model, image_processor, context_len
208
+ )
209
+
210
+ for x in message["files"]:
211
+ history.append(((x,), None))
212
+ if message["text"] is not None:
213
+ history.append((message["text"], None))
214
+ return history, gr.MultimodalTextbox(value=None, interactive=False)
215
+
216
+
217
+ @spaces.GPU
218
+ def bot(history, temperature, top_p, max_output_tokens):
219
+ print("### turn start history",history)
220
+ print("### turn start conv",our_chatbot.conversation)
221
+ text = history[-1][0]
222
+ images_this_term = []
223
+ text_this_term = ""
224
+ # import pdb;pdb.set_trace()
225
+ num_new_images = 0
226
+ for i, message in enumerate(history[:-1]):
227
+ if type(message[0]) is tuple:
228
+ images_this_term.append(message[0][0])
229
+ if is_valid_video_filename(message[0][0]):
230
+ # 不接受视频
231
+ raise ValueError("Video is not supported")
232
+ num_new_images += our_chatbot.num_frames
233
+ elif is_valid_image_filename(message[0][0]):
234
+ print("#### Load image from local file",message[0][0])
235
+ num_new_images += 1
236
+ else:
237
+ raise ValueError("Invalid image file")
238
+ else:
239
+ num_new_images = 0
240
+
241
+ # for message in history[-i-1:]:
242
+ # images_this_term.append(message[0][0])
243
+
244
+ assert len(images_this_term) > 0, "must have an image"
245
+ # image_files = (args.image_file).split(',')
246
+ # image = [load_image(f) for f in images_this_term if f]
247
+
248
+ all_image_hash = []
249
+ all_image_path = []
250
+ for image_path in images_this_term:
251
+ with open(image_path, "rb") as image_file:
252
+ image_data = image_file.read()
253
+ image_hash = hashlib.md5(image_data).hexdigest()
254
+ all_image_hash.append(image_hash)
255
+ image = PIL.Image.open(image_path).convert("RGB")
256
+ t = datetime.datetime.now()
257
+ filename = os.path.join(
258
+ LOGDIR,
259
+ "serve_images",
260
+ f"{t.year}-{t.month:02d}-{t.day:02d}",
261
+ f"{image_hash}.jpg",
262
+ )
263
+ all_image_path.append(filename)
264
+ if not os.path.isfile(filename):
265
+ os.makedirs(os.path.dirname(filename), exist_ok=True)
266
+ print("image save to",filename)
267
+ image.save(filename)
268
+
269
+ image_list = []
270
+ for f in images_this_term:
271
+ if is_valid_video_filename(f):
272
+ image_list += sample_frames(f, our_chatbot.num_frames)
273
+ elif is_valid_image_filename(f):
274
+ image_list.append(load_image(f))
275
+ else:
276
+ raise ValueError("Invalid image file")
277
+
278
+ image_tensor = [
279
+ process_images([f], our_chatbot.image_processor, our_chatbot.model.config)[0]
280
+ .to(our_chatbot.model.device)
281
+ for f in image_list
282
+ ]
283
+
284
+
285
+ image_tensor = torch.stack(image_tensor)
286
+ image_token = DEFAULT_IMAGE_TOKEN * num_new_images
287
+ # if our_chatbot.model.config.mm_use_im_start_end:
288
+ # inp = DEFAULT_IM_START_TOKEN + image_token + DEFAULT_IM_END_TOKEN + "\n" + inp
289
+ # else:
290
+ inp = text
291
+ inp = image_token + "\n" + inp
292
+ our_chatbot.conversation.append_message(our_chatbot.conversation.roles[0], inp)
293
+ # image = None
294
+ our_chatbot.conversation.append_message(our_chatbot.conversation.roles[1], None)
295
+ prompt = our_chatbot.conversation.get_prompt()
296
+
297
+ # input_ids = (
298
+ # tokenizer_image_token(
299
+ # prompt, our_chatbot.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt"
300
+ # )
301
+ # .unsqueeze(0)
302
+ # .to(our_chatbot.model.device)
303
+ # )
304
+ input_ids = tokenizer_image_token(
305
+ prompt, our_chatbot.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt"
306
+ ).unsqueeze(0).to(our_chatbot.model.device)
307
+ # print("### input_id",input_ids)
308
+ stop_str = (
309
+ our_chatbot.conversation.sep
310
+ if our_chatbot.conversation.sep_style != SeparatorStyle.TWO
311
+ else our_chatbot.conversation.sep2
312
+ )
313
+ keywords = [stop_str]
314
+ stopping_criteria = KeywordsStoppingCriteria(
315
+ keywords, our_chatbot.tokenizer, input_ids
316
+ )
317
+ # streamer = TextStreamer(
318
+ # our_chatbot.tokenizer, skip_prompt=True, skip_special_tokens=True
319
+ # )
320
+ streamer = TextIteratorStreamer(
321
+ our_chatbot.tokenizer, skip_prompt=True, skip_special_tokens=True
322
+ )
323
+ print(our_chatbot.model.device)
324
+ print(input_ids.device)
325
+ print(image_tensor.device)
326
+
327
+ # with torch.inference_mode():
328
+ # output_ids = our_chatbot.model.generate(
329
+ # input_ids,
330
+ # images=image_tensor,
331
+ # do_sample=True,
332
+ # temperature=0.7,
333
+ # top_p=1.0,
334
+ # max_new_tokens=4096,
335
+ # streamer=streamer,
336
+ # use_cache=False,
337
+ # stopping_criteria=[stopping_criteria],
338
+ # )
339
+
340
+ # outputs = our_chatbot.tokenizer.decode(output_ids[0]).strip()
341
+ # if outputs.endswith(stop_str):
342
+ # outputs = outputs[: -len(stop_str)]
343
+ # our_chatbot.conversation.messages[-1][-1] = outputs
344
+
345
+ # history[-1] = [text, outputs]
346
+
347
+ # return history
348
+ generate_kwargs = dict(
349
+ inputs=input_ids,
350
+ streamer=streamer,
351
+ images=image_tensor,
352
+ do_sample=True,
353
+ temperature=temperature,
354
+ top_p=top_p,
355
+ max_new_tokens=max_output_tokens,
356
+ use_cache=False,
357
+ stopping_criteria=[stopping_criteria],
358
+ )
359
+
360
+ t = Thread(target=our_chatbot.model.generate, kwargs=generate_kwargs)
361
+ t.start()
362
+
363
+ outputs = []
364
+ for stream_token in streamer:
365
+ outputs.append(stream_token)
366
+ # print("### stream_token",stream_token)
367
+ # our_chatbot.conversation.messages[-1][-1] = "".join(outputs)
368
+ history[-1] = [text, "".join(outputs)]
369
+ yield history
370
+ our_chatbot.conversation.messages[-1][-1] = "".join(outputs)
371
+ print("### turn end history", history)
372
+ print("### turn end conv",our_chatbot.conversation)
373
+
374
+ with open(get_conv_log_filename(), "a") as fout:
375
+ data = {
376
+ "type": "chat",
377
+ "model": "PULSE-7b",
378
+ "state": history,
379
+ "images": all_image_hash,
380
+ "images_path": all_image_path
381
+ }
382
+ print("#### conv log",data)
383
+ fout.write(json.dumps(data) + "\n")
384
+ for upload_img in all_image_path:
385
+ api.upload_file(
386
+ path_or_fileobj=upload_img,
387
+ path_in_repo=upload_img.replace("./logs/", ""),
388
+ repo_id=repo_name,
389
+ repo_type="dataset",
390
+ # revision=revision,
391
+ # ignore_patterns=["data*"]
392
+ )
393
+ # upload json
394
+ api.upload_file(
395
+ path_or_fileobj=get_conv_log_filename(),
396
+ path_in_repo=get_conv_log_filename().replace("./logs/", ""),
397
+ repo_id=repo_name,
398
+ repo_type="dataset")
399
+
400
+
401
+
402
+ txt = gr.Textbox(
403
+ scale=4,
404
+ show_label=False,
405
+ placeholder="Enter text and press enter.",
406
+ container=False,
407
+ )
408
+
409
+ with gr.Blocks(
410
+ css=".message-wrap.svelte-1lcyrx4>div.svelte-1lcyrx4 img {min-width: 40px}",
411
+ ) as demo:
412
+
413
+ cur_dir = os.path.dirname(os.path.abspath(__file__))
414
+ # gr.Markdown(title_markdown)
415
+ gr.HTML(html_header)
416
+
417
+ with gr.Column():
418
+ with gr.Accordion("Parameters", open=False) as parameter_row:
419
+ temperature = gr.Slider(
420
+ minimum=0.0,
421
+ maximum=1.0,
422
+ value=0.0,
423
+ step=0.1,
424
+ interactive=True,
425
+ label="Temperature",
426
+ )
427
+ top_p = gr.Slider(
428
+ minimum=0.0,
429
+ maximum=1.0,
430
+ value=1,
431
+ step=0.1,
432
+ interactive=True,
433
+ label="Top P",
434
+ )
435
+ max_output_tokens = gr.Slider(
436
+ minimum=0,
437
+ maximum=8192,
438
+ value=4096,
439
+ step=256,
440
+ interactive=True,
441
+ label="Max output tokens",
442
+ )
443
+ with gr.Row():
444
+ chatbot = gr.Chatbot([], elem_id="PULSE", bubble_full_width=False, height=750)
445
+
446
+ with gr.Row():
447
+ upvote_btn = gr.Button(value="👍 Upvote", interactive=True)
448
+ downvote_btn = gr.Button(value="👎 Downvote", interactive=True)
449
+ flag_btn = gr.Button(value="⚠️ Flag", interactive=True)
450
+ # stop_btn = gr.Button(value="⏹️ Stop Generation", interactive=True)
451
+ regenerate_btn = gr.Button(value="🔄 Regenerate", interactive=True)
452
+ clear_btn = gr.Button(value="🗑️ Clear history", interactive=True)
453
+
454
+
455
+ chat_input = gr.MultimodalTextbox(
456
+ interactive=True,
457
+ file_types=["image"],
458
+ placeholder="Enter message or upload file...",
459
+ show_label=False,
460
+ submit_btn="🚀"
461
+ )
462
+
463
+ print(cur_dir)
464
+ gr.Examples(
465
+ examples_per_page=5,
466
+ examples=[
467
+ [
468
+ {
469
+ "files": [
470
+ f"{cur_dir}/examples/ecg_example2.png",
471
+ ],
472
+ "text": "What are the main features in this ECG image?",
473
+ },
474
+ ],
475
+ [
476
+ {
477
+ "files": [
478
+ f"{cur_dir}/examples/ecg_example1.jpg",
479
+ ],
480
+ "text": "What can be inferred from the pattern of the qR complexes and rS complexes in the leads of this ECG image?",
481
+ },
482
+ ]
483
+ ],
484
+ inputs=[chat_input],
485
+ label="Image",
486
+ )
487
+
488
+ gr.Markdown(tos_markdown)
489
+ gr.Markdown(learn_more_markdown)
490
+ gr.Markdown(bibtext)
491
+
492
+ chat_msg = chat_input.submit(
493
+ add_message, [chatbot, chat_input], [chatbot, chat_input]
494
+ )
495
+ bot_msg = chat_msg.then(bot, [chatbot,temperature, top_p, max_output_tokens], chatbot, api_name="bot_response")
496
+ bot_msg.then(lambda: gr.MultimodalTextbox(interactive=True), None, [chat_input])
497
+
498
+ # chatbot.like(print_like_dislike, None, None)
499
+ clear_btn.click(
500
+ fn=clear_history, inputs=[chatbot], outputs=[chatbot], api_name="clear_all"
501
+ )
502
+
503
+
504
+ demo.queue()
505
+
506
+ if __name__ == "__main__":
507
+ import argparse
508
+
509
+ argparser = argparse.ArgumentParser()
510
+ argparser.add_argument("--server_name", default="0.0.0.0", type=str)
511
+ argparser.add_argument("--port", default="6123", type=str)
512
+ argparser.add_argument(
513
+ "--model_path", default="PULSE-ECG/PULSE-7B", type=str
514
+ )
515
+ # argparser.add_argument("--model-path", type=str, default="facebook/opt-350m")
516
+ argparser.add_argument("--model-base", type=str, default=None)
517
+ argparser.add_argument("--num-gpus", type=int, default=1)
518
+ argparser.add_argument("--conv-mode", type=str, default=None)
519
+ argparser.add_argument("--temperature", type=float, default=0.0)
520
+ argparser.add_argument("--max-new-tokens", type=int, default=1024)
521
+ argparser.add_argument("--num_frames", type=int, default=16)
522
+ argparser.add_argument("--load-8bit", action="store_true")
523
+ argparser.add_argument("--load-4bit", action="store_true")
524
+ argparser.add_argument("--debug", action="store_true")
525
+
526
+ args = argparser.parse_args()
527
+
528
+ model_path = args.model_path
529
+ filt_invalid = "cut"
530
+ model_name = get_model_name_from_path(args.model_path)
531
+ tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name, args.load_8bit, args.load_4bit)
532
+ print("### image_processor",image_processor)
533
+ model=model.to(torch.device('cuda'))
534
+ our_chatbot = None
535
+ demo.launch()
examples/.DS_Store ADDED
Binary file (6.15 kB). View file
 
examples/172197131626056_P7966202.png ADDED
examples/A-17-processors-1024x576.jpg ADDED
examples/Iphone-15-Usb-c-charger-1024x576.jpg ADDED
examples/Iphone-15-specs-1024x576.jpg ADDED
examples/africa.jpg ADDED
examples/ballon.jpg ADDED
examples/bar.jpg ADDED
examples/bigcompany.png ADDED

Git LFS Details

  • SHA256: 32e5066f3ced74d5d28fbfd3696232d728b90e9572335fc4ccb80ac1863aa6ff
  • Pointer size: 132 Bytes
  • Size of remote file: 2.01 MB
examples/bijiasuo2.jpeg ADDED
examples/book.jpg ADDED
examples/camera.jpg ADDED
examples/changed_bench.jpeg ADDED
examples/code.mp4 ADDED
Binary file (91.1 kB). View file
 
examples/code1.jpeg ADDED
examples/code2.jpeg ADDED
examples/dog.jpg ADDED
examples/dog1.jpg ADDED
examples/dog6.jpeg ADDED
examples/dog9.jpeg ADDED
examples/dog_to_monkey1.png ADDED

Git LFS Details

  • SHA256: 03b21f907e6e0614972102936fbbc6c5d45c93ddb2780c26e04770f860c986d7
  • Pointer size: 132 Bytes
  • Size of remote file: 2.23 MB
examples/dog_to_monkey2.png ADDED

Git LFS Details

  • SHA256: 28724dace4f330df6fb2ade40b7113d3c77c5e7b5fd4aeff8e1bd39407246fb1
  • Pointer size: 132 Bytes
  • Size of remote file: 2.14 MB
examples/dynamic-island-1024x576.jpg ADDED
examples/eagles.jpg ADDED
examples/ecg_example1.jpg ADDED
examples/ecg_example2.png ADDED
examples/examples_image12.jpg ADDED
examples/examples_image13.jpg ADDED
examples/examples_image14.jpg ADDED
examples/fangao1.jpeg ADDED
examples/fangao2.jpeg ADDED
examples/fangao3.jpeg ADDED
examples/food.jpg ADDED
examples/girl.jpg ADDED
examples/hanzi.jpg ADDED
examples/hot_ballon.jpg ADDED
examples/ice_cream.jpg ADDED
examples/image-00007.jpeg ADDED
examples/image-00053.jpeg ADDED
examples/iphone-15-colors-1024x576.jpg ADDED
examples/iphone-15-price-1024x576.jpg ADDED
examples/iphone-15-pricing-1024x576.jpg ADDED
examples/line chart.jpg ADDED
examples/norway.jpg ADDED
examples/oprah-winfrey-resume.png ADDED
examples/orange.png ADDED
examples/original_bench.jpeg ADDED