import string import gradio as gr import requests import torch from models.VLE import VLEForVQA, VLEProcessor, VLEForVQAPipeline from PIL import Image model_name="hfl/vle-base-for-vqa" model = VLEForVQA.from_pretrained(model_name) vle_processor = VLEProcessor.from_pretrained(model_name) vqa_pipeline = VLEForVQAPipeline(model=model, device='cpu', vle_processor=vle_processor) from transformers import BlipForQuestionAnswering, BlipProcessor device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-capfilt-large") model_vqa = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-capfilt-large").to(device) from transformers import BlipProcessor, BlipForConditionalGeneration cap_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-large") cap_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large") def caption(input_image): inputs = cap_processor(input_image, return_tensors="pt") # inputs["num_beams"] = 1 # inputs['num_return_sequences'] =1 out = cap_model.generate(**inputs) return "\n".join(cap_processor.batch_decode(out, skip_special_tokens=True)) import openai import os openai.api_key= os.getenv('openai_appkey') def gpt3_short(question,vqa_answer,caption): vqa_answer,vqa_score=vqa_answer prompt="This is the caption of a picture: "+caption+". Question: "+question+" VQA model predicts:"+"A: "+vqa_answer[0]+", socre:"+str(vqa_score[0])+\ "; B: "+vqa_answer[1]+", score:"+str(vqa_score[1])+"; C: "+vqa_answer[2]+", score:"+str(vqa_score[2])+\ "; D: "+vqa_answer[3]+', score:'+str(vqa_score[3])+\ ". Choose A if it is not in conflict with the description of the picture and A's score is bigger than 0.8; otherwise choose the B, C or D based on the description." # prompt=caption+"\n"+question+"\n"+vqa_answer+"\n Tell me the right answer." response = openai.Completion.create( engine="text-davinci-003", prompt=prompt, max_tokens=10, n=1, stop=None, temperature=0.7, ) answer = response.choices[0].text.strip() llm_ans=answer choice=set(["A","B","C","D"]) llm_ans=llm_ans.replace("\n"," ").replace(":"," ").replace("."," " ).replace(","," ") sllm_ans=llm_ans.split(" ") for cho in sllm_ans: if cho in choice: llm_ans=cho break if llm_ans not in choice: llm_ans="A" llm_ans=vqa_answer[ord(llm_ans)-ord("A")] answer=llm_ans return answer def gpt3_long(question,vqa_answer,caption): vqa_answer,vqa_score=vqa_answer # prompt="prompt: This is the caption of a picture: "+caption+". Question: "+question+" VQA model predicts:"+"A: "+vqa_answer[0]+"socre:"+str(vqa_score[0])+\ # " B: "+vqa_answer[1]+" score:"+str(vqa_score[1])+" C: "+vqa_answer[2]+" score:"+str(vqa_score[2])+\ # " D: "+vqa_answer[3]+'score:'+str(vqa_score[3])+\ # "Tell me the right answer with a long sentence." prompt="This is the caption of a picture: "+caption+". Question: "+question+" VQA model predicts:"+" "+vqa_answer[0]+", socre:"+str(vqa_score[0])+\ "; "+vqa_answer[1]+", score:"+str(vqa_score[1])+"; "+vqa_answer[2]+", score:"+str(vqa_score[2])+\ "; "+vqa_answer[3]+', score:'+str(vqa_score[3])+\ ". Question: "+question+" Tell me the right answer with a sentence." # prompt="prompt: This is the caption of a picture: "+caption+". Question: "+question+" VQA model predicts:"+" "+vqa_answer[0]+" socre:"+str(vqa_score[0])+\ # " "+vqa_answer[1]+" score:"+str(vqa_score[1])+" "+vqa_answer[2]+" score:"+str(vqa_score[2])+\ # " "+vqa_answer[3]+'score:'+str(vqa_score[3])+\ # "Tell me the right answer with a long sentence." # prompt=caption+"\n"+question+"\n"+vqa_answer+"\n Tell me the right answer." response = openai.Completion.create( engine="text-davinci-003", prompt=prompt, max_tokens=30, n=1, stop=None, temperature=0.7, ) answer = response.choices[0].text.strip() return answer def gpt3(question,vqa_answer,caption): prompt=caption+"\n"+question+"\n"+vqa_answer+"\n Tell me the right answer." response = openai.Completion.create( engine="text-davinci-003", prompt=prompt, max_tokens=30, n=1, stop=None, temperature=0.7, ) answer = response.choices[0].text.strip() # return "input_text:\n"+prompt+"\n\n output_answer:\n"+answer return answer def vle(input_image,input_text): vqa_answers = vqa_pipeline({"image":input_image, "question":input_text}, top_k=4) # return [" ".join([str(value) for key,value in vqa.items()] )for vqa in vqa_answers] return [vqa['answer'] for vqa in vqa_answers],[vqa['score'] for vqa in vqa_answers] def inference_chat(input_image,input_text): cap=caption(input_image) print(cap) # inputs = processor(images=input_image, text=input_text,return_tensors="pt") # inputs["max_length"] = 10 # inputs["num_beams"] = 5 # inputs['num_return_sequences'] =4 # out = model_vqa.generate(**inputs) # out=processor.batch_decode(out, skip_special_tokens=True) out=vle(input_image,input_text) # vqa="\n".join(out[0]) # gpt3_out=gpt3(input_text,vqa,cap) gpt3_out=gpt3_long(input_text,out,cap) gpt3_out1=gpt3_short(input_text,out,cap) return out[0][0], gpt3_out,gpt3_out1 title = """# VQA with VLE and LLM""" description = """**VLE** (Visual-Language Encoder) is an image-text multimodal understanding model built on the pre-trained text and image encoders. See https://github.com/iflytek/VLE for more details. We demonstrate visual question answering systems built with VLE and LLM.""" description1 = """**VQA**: The image and the question are fed to a VQA model (VLEForVQA) and the model predicts the answer. **VQA+LLM**: We feed the caption, question, and answers predicted by the VQA model to the LLM and ask the LLM to generate the final answer. The outptus from VQA+LLM may vary due to the decoding strategy of the LLM.""" with gr.Blocks( css=""" .message.svelte-w6rprc.svelte-w6rprc.svelte-w6rprc {font-size: 20px; margin-top: 20px} #component-21 > div.wrap.svelte-w6rprc {height: 600px;} """ ) as iface: state = gr.State([]) #caption_output = None gr.Markdown(title) gr.Markdown(description) #gr.Markdown(article) with gr.Row(): with gr.Column(scale=1): image_input = gr.Image(type="pil",label="VQA Image Input") with gr.Row(): with gr.Column(scale=1): chat_input = gr.Textbox(lines=1, label="VQA Question Input") with gr.Row(): clear_button = gr.Button(value="Clear", interactive=True,width=30) submit_button = gr.Button( value="Submit", interactive=True, variant="primary" ) ''' cap_submit_button = gr.Button( value="Submit_CAP", interactive=True, variant="primary" ) gpt3_submit_button = gr.Button( value="Submit_GPT3", interactive=True, variant="primary" ) ''' with gr.Column(): gr.Markdown(description1) caption_output = gr.Textbox(lines=0, label="VQA") caption_output_v1 = gr.Textbox(lines=0, label="VQA + LLM (short answer)") gpt3_output_v1 = gr.Textbox(lines=0, label="VQA+LLM (long answer)") # image_input.change( # lambda: ("", [],"","",""), # [], # [ caption_output, state,caption_output,gpt3_output_v1,caption_output_v1], # queue=False, # ) chat_input.submit( inference_chat, [ image_input, chat_input, ], [ caption_output,gpt3_output_v1,caption_output_v1], ) clear_button.click( lambda: ("", [],"","",""), [], [chat_input, state,caption_output,gpt3_output_v1,caption_output_v1], queue=False, ) submit_button.click( inference_chat, [ image_input, chat_input, ], [caption_output,gpt3_output_v1,caption_output_v1], ) ''' cap_submit_button.click( caption, [ image_input, ], [caption_output_v1], ) gpt3_submit_button.click( gpt3, [ chat_input, caption_output , caption_output_v1, ], [gpt3_output_v1], ) ''' examples=[['bird.jpeg',"How many birds are there in the tree?","2","2","2"], ['qa9.jpg',"What type of vehicle is being pulled by the horses ?",'carriage','sled','Sled'], ['upload4.jpg',"What is this old man doing?","fishing","fishing","Fishing"]] examples = gr.Examples( examples=examples,inputs=[image_input, chat_input,caption_output,caption_output_v1,gpt3_output_v1], ) iface.queue(concurrency_count=1, api_open=False, max_size=10) iface.launch(enable_queue=True)