Update app.py

app.py

@@ -2,6 +2,13 @@ import string
 import gradio as gr
 import requests
 import torch
+from models.VLE import VLEForVQA, VLEProcessor, VLEForVQAPipeline
+from PIL import Image
+
+model_name="hfl/vle-base-for-vqa"
+model = VLEForVQA.from_pretrained(model_name)
+vle_processor = VLEProcessor.from_pretrained(model_name)
+vqa_pipeline = VLEForVQAPipeline(model=model, device='cpu', vle_processor=vle_processor)
 
 
 from transformers import BlipForQuestionAnswering, BlipProcessor
@@ -41,15 +48,19 @@ def gpt3(question,vqa_answer,caption):
     # return "input_text:\n"+prompt+"\n\n output_answer:\n"+answer
     return answer
 
-    
+def vle(input_image,input_text):
+    vqa_answers = vqa_pipeline(image=input_image, question=input_image, top_k=4)
+    return vqa_answers
 def inference_chat(input_image,input_text):
     cap=caption(input_image)
-    inputs = processor(images=input_image, text=input_text,return_tensors="pt")
-    inputs["max_length"] = 10
-    inputs["num_beams"] = 5
-    inputs['num_return_sequences'] =4
-    out = model_vqa.generate(**inputs)
-    out=processor.batch_decode(out, skip_special_tokens=True)
+    # inputs = processor(images=input_image, text=input_text,return_tensors="pt")
+    # inputs["max_length"] = 10
+    # inputs["num_beams"] = 5
+    # inputs['num_return_sequences'] =4
+    # out = model_vqa.generate(**inputs)
+    # out=processor.batch_decode(out, skip_special_tokens=True)
+
+    out=vle(input_image,input_text)
     vqa="\n".join(out)
     gpt3_out=gpt3(input_text,vqa,cap)
     gpt3_out1=gpt3(input_text,'',cap)