Spaces:
Running
on
Zero
Running
on
Zero
File size: 4,349 Bytes
b6fa3b6 d02b0d1 b6fa3b6 b0db85d b6fa3b6 b0db85d b6fa3b6 d02b0d1 91c00b7 7377d18 b0db85d 7377d18 b0db85d 7377d18 91c00b7 b0db85d 7377d18 b0db85d d02b0d1 b0db85d b6fa3b6 2d6886d b0db85d 6c67d55 d02b0d1 6c67d55 f04732f b6fa3b6 f04732f d5fb61d b6fa3b6 d5fb61d b6fa3b6 1117f0e 8eae1e0 b0db85d 1117f0e 70f2766 b0db85d b6fa3b6 2d6886d 8db3ef1 2d6886d 70f2766 6530eb5 b6fa3b6 099dbb9 d5fb61d 70f2766 b6fa3b6 70f2766 b6fa3b6 70f2766 3a84945 58cf028 b6fa3b6 58cf028 f04732f 4bf4adf b0db85d 7377d18 b0db85d 7377d18 50def22 d5fb61d b6fa3b6 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 |
import time
from threading import Thread
import gradio as gr
import spaces
import torch
from PIL import Image
from transformers import AutoProcessor, AutoModelForCausalLM
from transformers import TextIteratorStreamer
import subprocess
subprocess.run('pip install flash-attn --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True)
PLACEHOLDER = """
<div style="padding: 30px; text-align: center; display: flex; flex-direction: column; align-items: center;">
<h1 style="font-size: 28px; margin-bottom: 2px; opacity: 0.55;">microsoft/Phi-3-vision-128k-instruct</h1>
</div>
"""
user_prompt = '<|user|>\n'
assistant_prompt = '<|assistant|>\n'
prompt_suffix = "<|end|>\n"
model_id = "microsoft/Phi-3-vision-128k-instruct"
processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(
model_id,
torch_dtype="auto",
trust_remote_code=True,
)
model.to("cuda:0")
@spaces.GPU
def bot_streaming(message, history):
print(message)
if message["files"]:
# message["files"][-1] is a Dict or just a string
if type(message["files"][-1]) == dict:
image = message["files"][-1]["path"]
else:
image = message["files"][-1]
else:
# if there's no image uploaded for this turn, look for images in the past turns
# kept inside tuples, take the last one
for hist in history:
if type(hist[0]) == tuple:
image = hist[0][0]
try:
if image is None:
# Handle the case where image is None
gr.Error("You need to upload an image for Phi-3-vision to work.")
except NameError:
# Handle the case where 'image' is not defined at all
gr.Error("You need to upload an image for Phi-3-vision to work.")
# prompt = f"{message['text']}<|image_1|>\nCan you convert the table to markdown format?{prompt_suffix}{assistant_prompt}"
chat = [
{"role": "user", "content": f"<|image_1|>\n{message['text']}"},
]
prompt = processor.tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)
# need to remove last <|endoftext|> if it is there, which is used for training, not inference. For training, make sure to add <|endoftext|> in the end.
if prompt.endswith("<|endoftext|>"):
prompt = prompt.rstrip("<|endoftext|>")
print(f">>> Prompt\n{prompt})")
image = Image.open(image)
inputs = processor(prompt, [image], return_tensors='pt').to("cuda:0")
streamer = TextIteratorStreamer(processor, **{"skip_special_tokens": False, "skip_prompt": True})
generation_kwargs = dict(
inputs,
streamer=streamer,
max_new_tokens=1024,
do_sample=False,
temperature=0.0,
eos_token_id=processor.tokenizer.eos_token_id
)
thread = Thread(target=model.generate, kwargs=generation_kwargs)
thread.start()
buffer = ""
time.sleep(0.5)
for new_text in streamer:
# if "<|endoftext|>" in new_text:
# break
buffer += new_text
generated_text_without_prompt = buffer
# print(generated_text_without_prompt)
time.sleep(0.06)
# print(f"new_text: {generated_text_without_prompt}")
yield generated_text_without_prompt
chatbot = gr.Chatbot(placeholder=PLACEHOLDER, scale=1, height=550)
chat_input = gr.MultimodalTextbox(interactive=True, file_types=["image"], placeholder="Enter message or upload file...",
show_label=False)
with gr.Blocks(fill_height=True, ) as demo:
gr.ChatInterface(
fn=bot_streaming,
title="Phi-3 Vision 128k Instruct",
examples=[{"text": "What is on the flower?", "files": ["./bee.jpg"]},
{"text": "How to make this pastry?", "files": ["./baklava.png"]}],
description="Try [microsoft/Phi-3-vision-128k-instruct](https://huggingface.co/microsoft/Phi-3-vision-128k-instruct). Upload an image and start chatting about it, or simply try one of the examples below. If you don't upload an image, you will receive an error.",
stop_btn="Stop Generation",
multimodal=True,
textbox=chat_input,
chatbot=chatbot,
)
demo.queue(api_open=False)
demo.launch(show_api=False, share=False)
|