add sample images and rewrite app.py

app.py

@@ -3,6 +3,7 @@ sys.path.append(".")
 
 import streamlit as st
 import pandas as pd
+from PIL import Image
 
 from model_loader import *
 from datasets import load_dataset
@@ -10,7 +11,8 @@ from datasets import load_dataset
 
 # load dataset
 #ds = load_dataset("test")
-ds = load_dataset("HuggingFaceM4/VQAv2", split="validation", cache_dir="cache", streaming=False)
+# ds = load_dataset("HuggingFaceM4/VQAv2", split="validation", cache_dir="cache", streaming=False)
+df = pd.read_json('vqa_samples.json', orient="columns")
 
 # define selector
 model_name = st.sidebar.selectbox(
@@ -20,17 +22,23 @@ model_name = st.sidebar.selectbox(
 
 image_selector_unspecific = st.number_input(
     "Select an image id: ",
-    0, len(ds)
+    0, len(df)
 )
 
 # select and display
-sample = ds[image_selector_unspecific]
-image = sample['image']
-image
+#sample = ds[image_selector_unspecific]
+sample = df.iloc[image_selector_unspecific]
+img_path = sample['img_path']
+image = Image.open(f'images/{img_path}.jpg')
+
+st.image(image, channels="RGB")
+question = sample['ques']
+label = sample['label']
 
 # inference
 question = st.text_input(f"Ask the model a question related to the image: \n"
                                f"(e.g. \"{sample['question']}\")")
 args = load_model(model_name) # TODO: cache
 answer = get_answer(args, image, question, model_name)
-st.write("answer")
+st.text(f"Answer by {model_name}: {answer}")
+st.text(f"Ground truth: {label}")

extract_100_samples.py

@@ -0,0 +1,23 @@
+from PIL import Image
+import torch
+from datasets import load_dataset, get_dataset_split_names
+import pandas as pd
+dataset = load_dataset("HuggingFaceM4/VQAv2", split="validation", cache_dir="cache", streaming=False)
+
+index_range = 1000
+
+## can see index here
+df = pd.DataFrame(columns=['ques', 'label', 'q_id', 'img_path', 'question_type'])
+
+for idx in range(index_range):
+    sample = dataset[idx]
+    ques = sample['question']
+    img = sample['image']
+    img_id = sample['image_id']
+    img.save(f'images/{img_id}.jpg')
+    label = sample['multiple_choice_answer']
+    q_id = sample['question_id']
+    q_type = sample['question_type']
+    df.loc[len(df.index)] = [ques, label, q_id, img_id, q_type]
+
+df.to_json('vqa_samples.json', orient='columns')