Spaces:
Runtime error
Runtime error
File size: 4,337 Bytes
09b15be f160eaf 09b15be f160eaf 09b15be 3fbe084 f160eaf 09b15be f160eaf 09b15be 5cc174c 09b15be f160eaf 09b15be f160eaf 09b15be |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 |
import gradio as gr
import torch
from transformers import FuyuForCausalLM, AutoTokenizer
from transformers.models.fuyu.processing_fuyu import FuyuProcessor
from transformers.models.fuyu.image_processing_fuyu import FuyuImageProcessor
from PIL import Image
model_id = "adept/fuyu-8b"
dtype = torch.bfloat16
device = "cuda"
tokenizer = AutoTokenizer.from_pretrained(model_id)
model = FuyuForCausalLM.from_pretrained(model_id, device_map="auto", torch_dtype=dtype)
processor = FuyuProcessor(image_processor=FuyuImageProcessor(), tokenizer=tokenizer)
caption_prompt = "Generate a coco-style caption.\\n"
def resize_to_max(image, max_width=1080, max_height=1080):
width, height = image.size
if width <= max_width and height <= max_height:
return image
scale = min(max_width/width, max_height/height)
width = int(width*scale)
height = int(height*scale)
return image.resize((width, height), Image.LANCZOS)
def predict(image, prompt):
# image = image.convert('RGB')
image = resize_to_max(image)
model_inputs = processor(text=prompt, images=[image])
model_inputs = {k: v.to(dtype=dtype if torch.is_floating_point(v) else v.dtype, device=device) for k,v in model_inputs.items()}
generation_output = model.generate(**model_inputs, max_new_tokens=40)
prompt_len = model_inputs["input_ids"].shape[-1]
return tokenizer.decode(generation_output[0][prompt_len:], skip_special_tokens=True)
def caption(image):
return predict(image, caption_prompt)
def set_example_image(example: list) -> dict:
return gr.Image.update(value=example[0])
css = """
#mkd {
height: 500px;
overflow: auto;
border: 1px solid #ccc;
}
"""
with gr.Blocks(css=css) as demo:
gr.HTML(
"""
<h1 id="title">Fuyu Multimodal Demo</h1>
<h3><a href="https://hf.co/adept/fuyu-8b">Fuyu-8B</a> is a multimodal model that supports a variety of tasks combining text and image prompts.</h3>
For example, you can use it for captioning by asking it to describe an image. You can also ask it questions about an image, a task known as Visual Question Answering, or VQA. This demo lets you explore captioning and VQA, with more tasks coming soon :)
Learn more about the model in <a href="https://www.adept.ai/blog/fuyu-8b">our blog post</a>.
<br>
<br>
<strong>Note: This is a raw model release. We have not added further instruction-tuning, postprocessing or sampling strategies to control for undesirable outputs. The model may hallucinate, and you should expect to have to fine-tune the model for your use-case!</strong>
<h3>Play with Fuyu-8B in this demo! 💬</h3>
"""
)
with gr.Tab("Visual Question Answering"):
with gr.Row():
with gr.Column():
image_input = gr.Image(label="Upload your Image", type="pil")
text_input = gr.Textbox(label="Ask a Question")
vqa_output = gr.Textbox(label="Output")
vqa_btn = gr.Button("Answer Visual Question")
gr.Examples(
[["assets/vqa_example_1.png", "How is this made?"], ["assets/vqa_example_2.png", "What is this flower and where is it's origin?"]],
inputs = [image_input, text_input],
outputs = [vqa_output],
fn=predict,
cache_examples=True,
label='Click on any Examples below to get VQA results quickly 👇'
)
with gr.Tab("Image Captioning"):
with gr.Row():
captioning_input = gr.Image(label="Upload your Image", type="pil")
captioning_output = gr.Textbox(label="Output")
captioning_btn = gr.Button("Generate Caption")
gr.Examples(
[["assets/captioning_example_1.png"], ["assets/captioning_example_2.png"]],
inputs = [captioning_input],
outputs = [captioning_output],
fn=caption,
cache_examples=True,
label='Click on any Examples below to get captioning results quickly 👇'
)
captioning_btn.click(fn=caption, inputs=captioning_input, outputs=captioning_output)
vqa_btn.click(fn=predict, inputs=[image_input, text_input], outputs=vqa_output)
demo.launch(server_name="0.0.0.0") |