Spaces:
Runtime error
Runtime error
| import string | |
| import gradio as gr | |
| import requests | |
| import torch | |
| from models.VLE import VLEForVQA, VLEProcessor, VLEForVQAPipeline | |
| from PIL import Image | |
| # model_name="hfl/vle-base-for-vqa" | |
| # model = VLEForVQA.from_pretrained(model_name) | |
| # vle_processor = VLEProcessor.from_pretrained(model_name) | |
| # vqa_pipeline = VLEForVQAPipeline(model=model, device='cpu', vle_processor=vle_processor) | |
| from transformers import BlipForQuestionAnswering, BlipProcessor | |
| # device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') | |
| # processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-capfilt-large") | |
| # model_vqa = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-capfilt-large").to(device) | |
| from transformers import BlipProcessor, BlipForConditionalGeneration | |
| # cap_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-large") | |
| # cap_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large") | |
| def caption(input_image): | |
| inputs = cap_processor(input_image, return_tensors="pt") | |
| # inputs["num_beams"] = 1 | |
| # inputs['num_return_sequences'] =1 | |
| out = cap_model.generate(**inputs) | |
| return "\n".join(cap_processor.batch_decode(out, skip_special_tokens=True)) | |
| import openai | |
| import os | |
| openai.api_key= os.getenv('openai_appkey') | |
| def gpt3_short(question,vqa_answer,caption): | |
| vqa_answer,vqa_score=vqa_answer | |
| prompt="This is the caption of a picture: "+caption+". Question: "+question+" VQA model predicts:"+"A: "+vqa_answer[0]+", socre:"+str(vqa_score[0])+\ | |
| "; B: "+vqa_answer[1]+", score:"+str(vqa_score[1])+"; C: "+vqa_answer[2]+", score:"+str(vqa_score[2])+\ | |
| "; D: "+vqa_answer[3]+', score:'+str(vqa_score[3])+\ | |
| ". Choose A if it is not in conflict with the description of the picture and A's score is bigger than 0.8; otherwise choose the B, C or D based on the description." | |
| # prompt=caption+"\n"+question+"\n"+vqa_answer+"\n Tell me the right answer." | |
| response = openai.Completion.create( | |
| engine="text-davinci-003", | |
| prompt=prompt, | |
| max_tokens=10, | |
| n=1, | |
| stop=None, | |
| temperature=0.7, | |
| ) | |
| answer = response.choices[0].text.strip() | |
| llm_ans=answer | |
| choice=set(["A","B","C","D"]) | |
| llm_ans=llm_ans.replace("\n"," ").replace(":"," ").replace("."," " ).replace(","," ") | |
| sllm_ans=llm_ans.split(" ") | |
| for cho in sllm_ans: | |
| if cho in choice: | |
| llm_ans=cho | |
| break | |
| if llm_ans not in choice: | |
| llm_ans="A" | |
| llm_ans=vqa_answer[ord(llm_ans)-ord("A")] | |
| answer=llm_ans | |
| return answer | |
| def gpt3_long(question,vqa_answer,caption): | |
| vqa_answer,vqa_score=vqa_answer | |
| # prompt="prompt: This is the caption of a picture: "+caption+". Question: "+question+" VQA model predicts:"+"A: "+vqa_answer[0]+"socre:"+str(vqa_score[0])+\ | |
| # " B: "+vqa_answer[1]+" score:"+str(vqa_score[1])+" C: "+vqa_answer[2]+" score:"+str(vqa_score[2])+\ | |
| # " D: "+vqa_answer[3]+'score:'+str(vqa_score[3])+\ | |
| # "Tell me the right answer with a long sentence." | |
| prompt="This is the caption of a picture: "+caption+". Question: "+question+" VQA model predicts:"+" "+vqa_answer[0]+", socre:"+str(vqa_score[0])+\ | |
| "; "+vqa_answer[1]+", score:"+str(vqa_score[1])+"; "+vqa_answer[2]+", score:"+str(vqa_score[2])+\ | |
| "; "+vqa_answer[3]+', score:'+str(vqa_score[3])+\ | |
| ". Question: "+question+" Tell me the right answer with a sentence." | |
| # prompt="prompt: This is the caption of a picture: "+caption+". Question: "+question+" VQA model predicts:"+" "+vqa_answer[0]+" socre:"+str(vqa_score[0])+\ | |
| # " "+vqa_answer[1]+" score:"+str(vqa_score[1])+" "+vqa_answer[2]+" score:"+str(vqa_score[2])+\ | |
| # " "+vqa_answer[3]+'score:'+str(vqa_score[3])+\ | |
| # "Tell me the right answer with a long sentence." | |
| # prompt=caption+"\n"+question+"\n"+vqa_answer+"\n Tell me the right answer." | |
| response = openai.Completion.create( | |
| engine="text-davinci-003", | |
| prompt=prompt, | |
| max_tokens=30, | |
| n=1, | |
| stop=None, | |
| temperature=0.7, | |
| ) | |
| answer = response.choices[0].text.strip() | |
| return answer | |
| def gpt3(question,vqa_answer,caption): | |
| prompt=caption+"\n"+question+"\n"+vqa_answer+"\n Tell me the right answer." | |
| response = openai.Completion.create( | |
| engine="text-davinci-003", | |
| prompt=prompt, | |
| max_tokens=30, | |
| n=1, | |
| stop=None, | |
| temperature=0.7, | |
| ) | |
| answer = response.choices[0].text.strip() | |
| # return "input_text:\n"+prompt+"\n\n output_answer:\n"+answer | |
| return answer | |
| def vle(input_image,input_text): | |
| vqa_answers = vqa_pipeline({"image":input_image, "question":input_text}, top_k=4) | |
| # return [" ".join([str(value) for key,value in vqa.items()] )for vqa in vqa_answers] | |
| return [vqa['answer'] for vqa in vqa_answers],[vqa['score'] for vqa in vqa_answers] | |
| def inference_chat(input_image,input_text): | |
| cap=caption(input_image) | |
| print(cap) | |
| # inputs = processor(images=input_image, text=input_text,return_tensors="pt") | |
| # inputs["max_length"] = 10 | |
| # inputs["num_beams"] = 5 | |
| # inputs['num_return_sequences'] =4 | |
| # out = model_vqa.generate(**inputs) | |
| # out=processor.batch_decode(out, skip_special_tokens=True) | |
| out=vle(input_image,input_text) | |
| # vqa="\n".join(out[0]) | |
| # gpt3_out=gpt3(input_text,vqa,cap) | |
| gpt3_out=gpt3_long(input_text,out,cap) | |
| gpt3_out1=gpt3_short(input_text,out,cap) | |
| return out[0][0], gpt3_out,gpt3_out1 | |
| title = """# VQA with VLE and LLM""" | |
| description = """We demonstrate three visual question answering systems built with VLE and LLM: | |
| * VQA + LLM (short answer): The captioning model generates a caption of the image. We feed the caption, the question, and the answer candidates predicted by the VQA model to the LLM, and ask the LLM to select the most reasonable answer from the candidates. | |
| * VQA + LLM (long answer): The pipeline is the same as VQA + LLM (short answer), except that the answer is freely generated by the LLM and not limited to VQA candidates. | |
| For more details about VLE and the VQA pipeline, see [http://vle.hfl-rc.com](http://vle.hfl-rc.com)""" | |
| with gr.Blocks( | |
| css=""" | |
| .message.svelte-w6rprc.svelte-w6rprc.svelte-w6rprc {font-size: 20px; margin-top: 20px} | |
| #component-21 > div.wrap.svelte-w6rprc {height: 600px;} | |
| """ | |
| ) as iface: | |
| state = gr.State([]) | |
| #caption_output = None | |
| gr.Markdown(title) | |
| #gr.Markdown(article) | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| image_input = gr.Image(type="pil",label="VQA Image Input") | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| chat_input = gr.Textbox(lines=1, label="VQA Question Input") | |
| with gr.Row(): | |
| clear_button = gr.Button(value="Clear", interactive=True,width=30) | |
| submit_button = gr.Button( | |
| value="Submit", interactive=True, variant="primary" | |
| ) | |
| ''' | |
| cap_submit_button = gr.Button( | |
| value="Submit_CAP", interactive=True, variant="primary" | |
| ) | |
| gpt3_submit_button = gr.Button( | |
| value="Submit_GPT3", interactive=True, variant="primary" | |
| ) | |
| ''' | |
| with gr.Column(): | |
| caption_output = gr.Textbox(lines=0, label="* VQA + LLM (short answer):\n The captioning model generates a caption \n of the image. We feed the caption") | |
| caption_output_v1 = gr.Textbox(lines=0, label="VQA + LLM (short answer)") | |
| gpt3_output_v1 = gr.Textbox(lines=0, label="VQA+LLM (long answer)") | |
| gr.Markdown(description) | |
| # image_input.change( | |
| # lambda: ("", [],"","",""), | |
| # [], | |
| # [ caption_output, state,caption_output,gpt3_output_v1,caption_output_v1], | |
| # queue=False, | |
| # ) | |
| chat_input.submit( | |
| inference_chat, | |
| [ | |
| image_input, | |
| chat_input, | |
| ], | |
| [ caption_output,gpt3_output_v1,caption_output_v1], | |
| ) | |
| clear_button.click( | |
| lambda: ("", [],"","",""), | |
| [], | |
| [chat_input, state,caption_output,gpt3_output_v1,caption_output_v1], | |
| queue=False, | |
| ) | |
| submit_button.click( | |
| inference_chat, | |
| [ | |
| image_input, | |
| chat_input, | |
| ], | |
| [caption_output,gpt3_output_v1,caption_output_v1], | |
| ) | |
| ''' | |
| cap_submit_button.click( | |
| caption, | |
| [ | |
| image_input, | |
| ], | |
| [caption_output_v1], | |
| ) | |
| gpt3_submit_button.click( | |
| gpt3, | |
| [ | |
| chat_input, | |
| caption_output , | |
| caption_output_v1, | |
| ], | |
| [gpt3_output_v1], | |
| ) | |
| ''' | |
| examples=[['bird.jpeg',"How many birds are there in the tree?","2","2","2"], | |
| ['qa9.jpg',"What type of vehicle is being pulled by the horses ?",'carriage','sled','Sled'], | |
| ['upload4.jpg',"What is this old man doing?","fishing","fishing","Fishing"]] | |
| examples = gr.Examples( | |
| examples=examples,inputs=[image_input, chat_input,caption_output,caption_output_v1,gpt3_output_v1], | |
| ) | |
| iface.queue(concurrency_count=1, api_open=False, max_size=10) | |
| iface.launch(enable_queue=True) |