Update app.py

app.py

@@ -8,10 +8,10 @@ from PIL import Image
 
 
 class _MLPVectorProjector(nn.Module):
-    def init(
+    def __init__(
         self, input_hidden_size: int, lm_hidden_size: int, num_layers: int, width: int
     ):
-        super(_MLPVectorProjector, self).init()
+        super(_MLPVectorProjector, self).__init__()
         self.mlps = nn.ModuleList()
         for _ in range(width):
             mlp = [nn.Linear(input_hidden_size, lm_hidden_size, bias=False)]
@@ -92,8 +92,26 @@ def textMode(text, count):
 
 def imageMode(image, question):
     image_embedding = encode_image(image)
+    print('-------Image embedding from clip obtained-----------')
     imgToTextEmb = img_proj_head(image_embedding)
-    return "In progress"
+    print('-------text embedding from projection obtained-----------')
+    question = "Question: " + question + "Answer: "
+    Qtokens = tokenizer_text.encode(question, add_special_tokens=True)
+    Qtoken_embeddings = phi2_finetuned.get_submodule('model.embed_tokens')(Qtokens)
+    print('-------question embedding from phi2 obtained-----------')
+    inputs = torch.concat((imgToTextEmb, Qtoken_embeddings), axis=-2)
+    
+    prediction = tokenizer.batch_decode(
+        phi2.generate(
+            inputs_embeds=inputs, 
+            max_new_tokens=50,
+            bos_token_id=tokenizer.bos_token_id, 
+            eos_token_id=tokenizer.eos_token_id,
+            pad_token_id=tokenizer.pad_token_id
+        )
+    )
+    text_pred = prediction[0].rstrip('<|endoftext|>').rstrip("\n")
+    return text_pred
 
 def audioMode(audio):
     if audio is None: