diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..b9ea89a3c95026900c2e0057ecccba045588f9c1
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,7 @@
+temp*
+
+
+# python temp files
+__pycache__
+*.pyc
+.vscode
diff --git a/README.md b/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..32897cd3e640101ba184f8c4ccd896981de3804a
--- /dev/null
+++ b/README.md
@@ -0,0 +1,3 @@
+---
+license: mit
+---
diff --git a/app.py b/app.py
new file mode 100644
index 0000000000000000000000000000000000000000..0a06ed00d40e74f6caf0ef6d784454f11673f0aa
--- /dev/null
+++ b/app.py
@@ -0,0 +1,400 @@
+import io
+import os
+
+import json
+import base64
+import random
+import numpy as np
+import pandas as pd
+import gradio as gr
+from pathlib import Path
+from PIL import Image
+
+from plots import get_pre_define_colors
+from utils.load_model import load_xclip
+from utils.predict import xclip_pred
+
+
+DEVICE = "cpu"
+XCLIP, OWLVIT_PRECESSOR = load_xclip(DEVICE)
+XCLIP_DESC_PATH = "data/jsons/bs_cub_desc.json"
+XCLIP_DESC = json.load(open(XCLIP_DESC_PATH, "r"))
+PREPROCESS = lambda x: OWLVIT_PRECESSOR(images=x, return_tensors='pt')
+IMAGES_FOLDER = "data/images"
+# XCLIP_RESULTS = json.load(open("data/jsons/xclip_org.json", "r"))
+# correct_predictions = [k for k, v in XCLIP_RESULTS.items() if v['prediction']]
+
+# get the intersection of sachit and xclip (revised)
+# INTERSECTION = []
+# IMAGE_RES = 400 * 400 # minimum resolution
+# TOTAL_SAMPLES = 20
+# for file_name in XCLIP_RESULTS:
+# image = Image.open(os.path.join(IMAGES_FOLDER, 'org', file_name)).convert('RGB')
+# w, h = image.size
+# if w * h < IMAGE_RES:
+# continue
+# else:
+# INTERSECTION.append(file_name)
+
+# IMAGE_FILE_LIST = random.sample(INTERSECTION, TOTAL_SAMPLES)
+IMAGE_FILE_LIST = json.load(open("data/jsons/file_list.json", "r"))
+# IMAGE_FILE_LIST = IMAGE_FILE_LIST[:19]
+# IMAGE_FILE_LIST.append('Eastern_Bluebird.jpg')
+IMAGE_GALLERY = [Image.open(os.path.join(IMAGES_FOLDER, 'org', file_name)).convert('RGB') for file_name in IMAGE_FILE_LIST]
+
+ORG_PART_ORDER = ['back', 'beak', 'belly', 'breast', 'crown', 'forehead', 'eyes', 'legs', 'wings', 'nape', 'tail', 'throat']
+ORDERED_PARTS = ['crown', 'forehead', 'nape', 'eyes', 'beak', 'throat', 'breast', 'belly', 'back', 'wings', 'legs', 'tail']
+COLORS = get_pre_define_colors(12, cmap_set=['Set2', 'tab10'])
+SACHIT_COLOR = "#ADD8E6"
+# CUB_BOXES = json.load(open("data/jsons/cub_boxes_owlvit_large.json", "r"))
+VISIBILITY_DICT = json.load(open("data/jsons/cub_vis_dict_binary.json", 'r'))
+VISIBILITY_DICT['Eastern_Bluebird.jpg'] = dict(zip(ORDERED_PARTS, [True]*12))
+
+# --- Image related functions ---
+def img_to_base64(img):
+ img_pil = Image.fromarray(img) if isinstance(img, np.ndarray) else img
+ buffered = io.BytesIO()
+ img_pil.save(buffered, format="JPEG")
+ img_str = base64.b64encode(buffered.getvalue())
+ return img_str.decode()
+
+def create_blank_image(width=500, height=500, color=(255, 255, 255)):
+ """Create a blank image of the given size and color."""
+ return np.array(Image.new("RGB", (width, height), color))
+
+# Convert RGB colors to hex
+def rgb_to_hex(rgb):
+ return f"#{''.join(f'{x:02x}' for x in rgb)}"
+
+def load_part_images(file_name: str) -> dict:
+ part_images = {}
+ # start_time = time.time()
+ for part_name in ORDERED_PARTS:
+ base_name = Path(file_name).stem
+ part_image_path = os.path.join(IMAGES_FOLDER, "boxes", f"{base_name}_{part_name}.jpg")
+ if not Path(part_image_path).exists():
+ continue
+ image = np.array(Image.open(part_image_path))
+ part_images[part_name] = img_to_base64(image)
+ # print(f"Time cost to load 12 images: {time.time() - start_time}")
+ # This takes less than 0.01 seconds. So the loading time is not the bottleneck.
+ return part_images
+
+def generate_xclip_explanations(result_dict:dict, visibility: dict, part_mask: dict = dict(zip(ORDERED_PARTS, [1]*12))):
+ """
+ The result_dict needs three keys: 'descriptions', 'pred_scores', 'file_name'
+ descriptions: {part_name1: desc_1, part_name2: desc_2, ...}
+ pred_scores: {part_name1: score_1, part_name2: score_2, ...}
+ file_name: str
+ """
+
+ descriptions = result_dict['descriptions']
+ image_name = result_dict['file_name']
+ part_images = PART_IMAGES_DICT[image_name]
+ MAX_LENGTH = 50
+ exp_length = 400
+ fontsize = 15
+
+ # Start the SVG inside a div
+ svg_parts = [f'',
+ "", "
"))
+ # Join everything into a single string
+ html = "".join(svg_parts)
+
+
+ return html
+
+
+
+def generate_sachit_explanations(result_dict:dict):
+ descriptions = result_dict['descriptions']
+ scores = result_dict['scores']
+ MAX_LENGTH = 50
+ exp_length = 400
+ fontsize = 15
+
+ descriptions = zip(scores, descriptions)
+ descriptions = sorted(descriptions, key=lambda x: x[0], reverse=True)
+
+ # Start the SVG inside a div
+ svg_parts = [f'',
+ "", "
"))
+ # Join everything into a single string
+ html = "".join(svg_parts)
+
+
+ return html
+
+# --- Constants created by the functions above ---
+BLANK_OVERLAY = img_to_base64(create_blank_image())
+PART_COLORS = {part: rgb_to_hex(COLORS[i]) for i, part in enumerate(ORDERED_PARTS)}
+blank_image = np.array(Image.open('data/images/final.png').convert('RGB'))
+PART_IMAGES_DICT = {file_name: load_part_images(file_name) for file_name in IMAGE_FILE_LIST}
+
+# --- Gradio Functions ---
+def update_selected_image(event: gr.SelectData):
+ image_height = 400
+ index = event.index
+
+ image_name = IMAGE_FILE_LIST[index]
+ current_image.state = image_name
+ org_image = Image.open(os.path.join(IMAGES_FOLDER, 'org', image_name)).convert('RGB')
+ img_base64 = f"""
+
+
})
+
+