Spaces:
Runtime error
Runtime error
| import gradio as gr | |
| import requests | |
| from PIL import Image | |
| import torch | |
| from transformers import AutoModelForCausalLM | |
| model = AutoModelForCausalLM.from_pretrained("q-future/co-instruct-preview", | |
| trust_remote_code=True, | |
| torch_dtype=torch.float16, | |
| attn_implementation="eager", | |
| device_map={"":"cuda:0"}) | |
| def chat(message, history, image_1, image_2, image_3, image_4): | |
| print(history) | |
| if history: | |
| if image_1 is not None and image_2 is None: | |
| past_message = "USER: The image: <|image|> " + history[0][0] + " ASSISTANT:" + history[0][1] | |
| for i in range((len(history) - 1)): | |
| past_message += "USER:" +history[i][0] + " ASSISTANT:" + history[i][1] + "</s>" | |
| message = past_message + "USER:" + message + " ASSISTANT:" | |
| images = [image_1] | |
| if image_1 is not None and image_2 is not None: | |
| if image_3 is None: | |
| past_message = "USER: The first image: <|image|>\nThe second image: <|image|>" + history[0][0] + " ASSISTANT:" + history[0][1] + "</s>" | |
| for i in range((len(history) - 1)): | |
| past_message += "USER:" + history[i][0] + " ASSISTANT:" + history[i][1] + "</s>" | |
| message = past_message + "USER:" + message + " ASSISTANT:" | |
| images = [image_1, image_2] | |
| else: | |
| if image_4 is None: | |
| past_message = "USER: The first image: <|image|>\nThe second image: <|image|>\nThe third image:<|image|>" + history[0][0] + " ASSISTANT:" + history[0][1] + "</s>" | |
| for i in range((len(history) - 1)): | |
| past_message += "USER:" + history[i][0] + " ASSISTANT:" + history[i][1] + "</s>" | |
| message = past_message + "USER:" + message + " ASSISTANT:" | |
| images = [image_1, image_2, image_3] | |
| else: | |
| past_message = "USER: The first image: <|image|>\nThe second image: <|image|>\nThe third image:<|image|>\nThe fourth image:<|image|>" + history[0][0] + " ASSISTANT:" + history[0][1] + "</s>" | |
| for i in range((len(history) - 1)): | |
| past_message += "USER:" + history[i][0] + " ASSISTANT:" + history[i][1] + "</s>" | |
| message = past_message + "USER:" + message + " ASSISTANT:" | |
| images = [image_1, image_2, image_3, image_4] | |
| else: | |
| if image_1 is not None and image_2 is None: | |
| message = "USER: The image: <|image|> " + message + " ASSISTANT:" | |
| images = [image_1] | |
| if image_1 is not None and image_2 is not None: | |
| if image_3 is None: | |
| message = "USER: The first image: <|image|>\nThe second image: <|image|>" + message + " ASSISTANT:" | |
| images = [image_1, image_2] | |
| else: | |
| if image_4 is None: | |
| message = "USER: The first image: <|image|>\nThe second image: <|image|>\nThe third image:<|image|>" + message + " ASSISTANT:" | |
| images = [image_1, image_2, image_3] | |
| else: | |
| message = "USER: The first image: <|image|>\nThe second image: <|image|>\nThe third image:<|image|>\nThe fourth image:<|image|>" + message + " ASSISTANT:" | |
| images = [image_1, image_2, image_3, image_4] | |
| print(message) | |
| return model.tokenizer.batch_decode(model.chat(message, images, max_new_tokens=300).clamp(0, 100000))[0].split("ASSISTANT:")[-1] | |
| with gr.Blocks(title="img") as demo: | |
| title_markdown = (""" | |
| <h3 align="center">*Super Version of Q-Instruct with Multi-image (up to 4, same as GPT-4V) Support!*</h3> | |
| <h1 align="center"><a href="https://github.com/Q-Future/Q-Instruct"><img src="https://github.com/Q-Future/Q-Instruct/blob/main/q_instruct_logo.png?raw=true", alt="Q-Instruct (mPLUG-Owl-2)" border="0" style="margin: 0 auto; height: 85px;" /></a> </h1> | |
| <h2 align="center">Q-Instruct: Improving Low-level Visual Abilities for Multi-modality Foundation Models</h2> | |
| <h5 align="center"> Please find our more accurate visual scoring demo on <a href='https://huggingface.co/spaces/teowu/OneScorer'>[OneScorer]</a>!</h2> | |
| <div align="center"> | |
| <div style="display:flex; gap: 0.25rem;" align="center"> | |
| <a href='https://github.com/Q-Future/Q-Instruct'><img src='https://img.shields.io/badge/Github-Code-blue'></a> | |
| <a href="https://Q-Instruct.github.io/Q-Instruct/fig/Q_Instruct_v0_1_preview.pdf"><img src="https://img.shields.io/badge/Technical-Report-red"></a> | |
| <a href='https://github.com/Q-Future/Q-Instruct/stargazers'><img src='https://img.shields.io/github/stars/Q-Future/Q-Instruct.svg?style=social'></a> | |
| </div> | |
| </div> | |
| """) | |
| gr.Markdown(title_markdown) | |
| with gr.Row(): | |
| input_img_1 = gr.Image(type='pil', label="Image 1 (First image)") | |
| input_img_2 = gr.Image(type='pil', label="Image 2 (Second image)") | |
| input_img_3 = gr.Image(type='pil', label="Image 3 (Third image)") | |
| input_img_4 = gr.Image(type='pil', label="Image 4 (Third image)") | |
| gr.ChatInterface(fn = chat, additional_inputs=[input_img_1, input_img_2, input_img_3, input_img_4]) | |
| demo.launch(share=True) |