Add new examples

app.py

@@ -1,9 +1,6 @@
-from urllib.request import urlopen
-import argparse
 import clip
 from PIL import Image
 import pandas as pd
-import time
 import torch
 from dataloader.extract_features_dataloader import transform_resize, question_preprocess
 from model.vqa_model import NetVQA
@@ -30,7 +27,7 @@ class InferenceConfig:
                    5: "color",
                    6: "other"}
     folds = 10   
-    tta = False
+    
     # Data
     n_classes: int =  5726
     
@@ -38,7 +35,8 @@ class InferenceConfig:
     class_mapping: str = "./data/annotations/class_mapping.csv"
 
     device = "cuda" if torch.cuda.is_available() else "cpu"
-         
+    
+           
 config = InferenceConfig()
 
 # load class mapping
@@ -48,7 +46,7 @@ for i in range(len(cm)):
     row = cm.iloc[i]
     classid_to_answer[row["class_id"]] = row["answer"]
 
-clip_model, preprocess = clip.load(config.model, download_root=config.checkpoint_root_clip)
+clip_model, preprocess = clip.load(config.model, download_root=config.checkpoint_root_clip, device=config.device)
 
 model = NetVQA(config).to(config.device)
 
@@ -58,8 +56,8 @@ config.checkpoint_head = "{}/{}.pt".format(config.checkpoint_root_head, config.m
 model_state_dict = torch.load(config.checkpoint_head)  
 model.load_state_dict(model_state_dict, strict=True)
 
+model.eval()
 
-#%%
 # Select Preprocessing
 image_transforms = transform_resize(clip_model.visual.input_resolution)
 
@@ -69,30 +67,21 @@ else:
     question_transforms = None
 
 clip_model.eval()
-model.eval()
 
 
 def predict(img, text):
     img = Image.fromarray(img)
-    if config.tta:   
-        image_augmentations = []
-        for transform in image_transforms:
-            image_augmentations.append(transform(img))
-        img = torch.stack(image_augmentations, dim=0)
+    img = image_transforms(img)
+    img = img.unsqueeze(dim=0)
+    
+    if question_transforms is not None:
+        question = question_transforms(text)
     else:
-        img = image_transforms(img)
-        img = img.unsqueeze(dim=0)
-
-    question = question_transforms(text)
+        question = text
     question_tokens = clip.tokenize(question, truncate=True)
     with torch.no_grad():
         img = img.to(config.device)
         img_feature = clip_model.encode_image(img)
-        if config.tta:
-            weights = torch.tensor(config.features_selection).reshape((len(config.features_selection),1))    
-            img_feature =  img_feature *  weights.to(config.device)
-            img_feature = img_feature.sum(0)
-            img_feature = img_feature.unsqueeze(0)
         
         question_tokens = question_tokens.to(config.device)
         question_feature = clip_model.encode_text(question_tokens)
@@ -116,6 +105,6 @@ def predict(img, text):
 gr.Interface(fn=predict, 
              inputs=[gr.Image(label='Image'), gr.Textbox(label='Question')],
              outputs=[gr.outputs.Label(label='Answer', num_top_classes=5), gr.outputs.Label(label='Answer Category', num_top_classes=7)],
-             examples=[['examples/VizWiz_train_00004056.jpg', 'Is that a beer or a coke?'], ['examples/VizWiz_train_00017146.jpg', 'Can you tell me what\'s on this envelope please?'], ['examples/VizWiz_val_00003077.jpg', 'What is this?']]
+             examples=[['examples/Augustiner.jpg', 'What is this?'],['examples/VizWiz_test_00006968.jpg', 'Can you tell me the color of the dog?'], ['examples/VizWiz_test_00005604.jpg', 'What drink is this?'], ['examples/VizWiz_test_00006246.jpg', 'Can you please tell me what kind of tea this is?'], ['examples/VizWiz_train_00004056.jpg', 'Is that a beer or a coke?'], ['examples/VizWiz_train_00017146.jpg', 'Can you tell me what\'s on this envelope please?'], ['examples/VizWiz_val_00003077.jpg', 'What is this?']]
              ).launch()