initail commit

.gitignore

@@ -0,0 +1,13 @@
+outputs/
+datasets/sidechef/images
+datasets/sidechef/sample_images
+datasets/sidechef/my_tags.json
+datasets/sidechef/tag_categories.json
+datasets/sidechef/tags.json
+launching
+annotation/
+.vscode/
+bert-base-uncased/
+delete*
+__pycache__/
+env/

app.py

@@ -0,0 +1,208 @@
+import pandas as pd
+import json
+from PIL import Image
+import numpy as np
+
+import os
+from pathlib import Path
+
+import torch
+import torch.nn.functional as F
+
+# from src.data.embs import ImageDataset
+from src.model.blip_embs import blip_embs
+from src.data.transforms import transform_test
+
+from transformers import StoppingCriteria, StoppingCriteriaList, TextIteratorStreamer
+import gradio as gr
+
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_groq import ChatGroq
+
+
+# GROQ_API_KEY = os.getenv("GROQ_API_KEY")
+GROQ_API_KEY = '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'
+os.environ["GROQ_API_KEY"] = GROQ_API_KEY
+
+# Initialize LLM
+llm = ChatGroq(model="llama-3.1-70b-versatile", temperature=0, max_tokens=1024, max_retries=2)
+
+# QA system prompt and chain
+qa_system_prompt = """
+Prompt:
+You are a highly intelligent assistant. Use the following context to answer user questions. Analyze the data carefully and generate a clear, concise, and informative response to the user's question based on this data.
+
+Response Guidelines:
+    - Use only the information provided in the data to answer the question.
+    - Ensure the answer is accurate and directly related to the question.
+    - If the data is insufficient to answer the question, politey apologise and tell the user that there is insufficient data available to answer their question.
+    - Provide the response in a conversational yet professional tone.
+
+Context:
+{context}
+"""
+qa_prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", qa_system_prompt),
+        ("human", "{input}")
+    ]
+)
+
+question_answer_chain = qa_prompt | llm | StrOutputParser()
+
+
+class StoppingCriteriaSub(StoppingCriteria):
+
+    def __init__(self, stops=[], encounters=1):
+        super().__init__()
+        self.stops = stops
+
+    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor):
+        for stop in self.stops:
+            if torch.all(input_ids[:, -len(stop):] == stop).item():
+                return True
+
+        return False
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+def get_blip_config(model="base"):
+    config = dict()
+    if model == "base":
+        config[
+            "pretrained"
+        ] = "https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_capfilt_large.pth "
+        config["vit"] = "base"
+        config["batch_size_train"] = 32
+        config["batch_size_test"] = 16
+        config["vit_grad_ckpt"] = True
+        config["vit_ckpt_layer"] = 4
+        config["init_lr"] = 1e-5
+    elif model == "large":
+        config[
+            "pretrained"
+        ] = "https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_large_retrieval_coco.pth"
+        config["vit"] = "large"
+        config["batch_size_train"] = 16
+        config["batch_size_test"] = 32
+        config["vit_grad_ckpt"] = True
+        config["vit_ckpt_layer"] = 12
+        config["init_lr"] = 5e-6
+
+    config["image_size"] = 384
+    config["queue_size"] = 57600
+    config["alpha"] = 0.4
+    config["k_test"] = 256
+    config["negative_all_rank"] = True
+
+    return config
+
+
+print("Creating model")
+config = get_blip_config("large")
+
+model = blip_embs(
+        pretrained=config["pretrained"],
+        image_size=config["image_size"],
+        vit=config["vit"],
+        vit_grad_ckpt=config["vit_grad_ckpt"],
+        vit_ckpt_layer=config["vit_ckpt_layer"],
+        queue_size=config["queue_size"],
+        negative_all_rank=config["negative_all_rank"],
+    )
+
+model = model.to(device)
+model.eval()
+print("Model Loaded !")
+print("="*50)
+
+transform = transform_test(384)
+
+print("Loading Data")
+df = pd.read_json("datasets/sidechef/my_recipes.json")
+
+print("Loading Target Embedding")
+tar_img_feats = []
+for _id in df["id_"].tolist():     
+    tar_img_feats.append(torch.load("datasets/sidechef/blip-embs-large/{:07d}.pth".format(_id)).unsqueeze(0))
+
+tar_img_feats = torch.cat(tar_img_feats, dim=0)
+
+
+class Chat:
+
+    def __init__(self, model, transform, dataframe, tar_img_feats, device='cuda:0', stopping_criteria=None):
+        self.device = device
+        self.model = model
+        self.transform = transform
+        self.df = dataframe
+        self.tar_img_feats = tar_img_feats
+        self.img_feats = None
+        self.target_recipe = None
+        self.messages = []
+
+        if stopping_criteria is not None:
+            self.stopping_criteria = stopping_criteria
+        else:
+            stop_words_ids = [torch.tensor([2]).to(self.device)]
+            self.stopping_criteria = StoppingCriteriaList([StoppingCriteriaSub(stops=stop_words_ids)])
+
+    def encode_image(self, image_path):
+        img = Image.fromarray(image_path).convert("RGB")
+        img = self.transform(img).unsqueeze(0)
+        img = img.to(self.device)
+        img_embs = model.visual_encoder(img)
+        img_feats = F.normalize(model.vision_proj(img_embs[:, 0, :]), dim=-1).cpu()
+
+        self.img_feats = img_feats 
+
+        self.get_target(self.img_feats, self.tar_img_feats)
+
+    def get_target(self, img_feats, tar_img_feats) : 
+        score = (img_feats @ tar_img_feats.t()).squeeze(0).cpu().detach().numpy()
+        index = np.argsort(score)[::-1][0] + 1
+        self.target_recipe = df.iloc[index]
+
+    def ask(self):
+        return json.dumps(self.target_recipe.to_json())
+
+
+
+chat = Chat(model,transform,df,tar_img_feats)
+print("Chat Initialized !")
+
+
+custom_css = """
+.primary{
+    background-color: #4CAF50; /* Green */
+}
+"""
+
+
+def respond_to_user(image, message):
+    # Process the image and message here
+    # For demonstration, I'll just return a simple text response
+    chat = Chat(model,transform,df,tar_img_feats)
+    chat.encode_image(image)
+    data = chat.ask()
+    formated_input = {
+        'input': message,
+        'context': data
+    }
+    try:
+        response = question_answer_chain.invoke(formated_input)
+    except Exception as e:
+        response = {'content':"An error occurred while processing your request."}
+    return response
+
+iface = gr.Interface(
+    fn=respond_to_user,
+    inputs=[gr.Image(), gr.Textbox(label="Ask Query")],
+    outputs=gr.Textbox(label="Nutrition-GPT"),
+    title="Nutrition-GPT Demo",
+    description="Upload an food image and ask queries!",
+    css=".component-12 {background-color: red}",
+)
+
+iface.launch()

configs/data/cirr.yaml

@@ -0,0 +1,22 @@
+dataname: cirr
+_target_: src.data.cirr.CIRRDataModule
+
+# Paths
+dataset_dir: ${paths.datasets_dir}/CIRR
+
+batch_size: ${machine.batch_size}
+num_workers: ${machine.num_workers}
+
+annotation:
+  train: ${paths.work_dir}/annotation/cirr/cap.rc2.train.json
+  val: ${paths.work_dir}/annotation/cirr/cap.rc2.val.json
+
+img_dirs:
+  train: ${data.dataset_dir}/images/train
+  val: ${data.dataset_dir}/images/dev
+
+emb_dirs:
+  train: ${data.dataset_dir}/blip-embs-large/train
+  val: ${data.dataset_dir}/blip-embs-large/dev
+
+image_size: 384

configs/data/fashioniq-base.yaml

@@ -0,0 +1,28 @@
+dataname: fashioniq-${data.category}
+_target_: src.data.fashioniq.FashionIQDataModule
+
+# Paths
+dataset_dir: ${paths.datasets_dir}/fashion-iq
+
+batch_size: ${machine.batch_size}
+num_workers: ${machine.num_workers}
+
+annotation:
+  train: ${paths.work_dir}/annotation/fashion-iq/cap.${data.category}.train.json
+  val: ${paths.work_dir}/annotation/fashion-iq/cap.${data.category}.val.json
+
+targets:
+  train: ${paths.work_dir}/annotation/fashion-iq/split.${data.category}.train.json
+  val: ${paths.work_dir}/annotation/fashion-iq/split.${data.category}.val.json
+
+img_dirs:
+  train: ${data.dataset_dir}/images/
+  val: ${data.dataset_dir}/images/
+
+emb_dirs:
+  train: ${data.dataset_dir}/blip-embs-large/
+  val: ${data.dataset_dir}/blip-embs-large/
+
+image_size: 384
+
+category: ???

configs/data/fashioniq-dress.yaml

@@ -0,0 +1,4 @@
+defaults:
+  - fashioniq-base.yaml
+
+category: dress

configs/data/fashioniq-shirt.yaml

@@ -0,0 +1,4 @@
+defaults:
+  - fashioniq-base.yaml
+
+category: shirt

configs/data/fashioniq-toptee.yaml

@@ -0,0 +1,4 @@
+defaults:
+  - fashioniq-base.yaml
+
+category: toptee

configs/data/webvid-covr.yaml

@@ -0,0 +1,26 @@
+dataname: webvid-covr
+_target_: src.data.webvid_covr.WebVidCoVRDataModule
+
+image_size: 384
+iterate: "pth2"
+vid_query_method: middle
+vid_frames: 1
+emb_pool: query
+
+# Paths
+dataset_dir: ${paths.datasets_dir}/WebVid
+
+batch_size: ${machine.batch_size}
+num_workers: ${machine.num_workers}
+
+annotation:
+  train: ${paths.work_dir}/annotation/webvid-covr/webvid2m-covr_train.csv
+  val: ${paths.work_dir}/annotation/webvid-covr/webvid8m-covr_val.csv
+
+vid_dirs:
+  train: ${data.dataset_dir}/2M/train
+  val: ${data.dataset_dir}/8M/train
+
+emb_dirs:
+  train: ${data.dataset_dir}/2M/blip-vid-embs-${model.model.vit}-all
+  val: ${data.dataset_dir}/8M/blip-vid-embs-${model.model.vit}-all

configs/data/webvid-covr_rule-based.yaml

@@ -0,0 +1,26 @@
+dataname: webvid-covr-rule-based
+_target_: src.data.webvid_covr_rulebased.WebVidCoVRDataModuleRuleBased
+
+image_size: 384
+iterate: "pth2"
+vid_query_method: middle
+vid_frames: 1
+emb_pool: query
+
+# Paths
+dataset_dir: ${paths.datasets_dir}/WebVid
+
+batch_size: ${machine.batch_size}
+num_workers: ${machine.num_workers}
+
+annotation:
+  train: ${paths.work_dir}/annotation/webvid-covr/webvid2m-covr_train.csv
+  val: ${paths.work_dir}/annotation/webvid-covr/webvid8m-covr_val.csv
+
+vid_dirs:
+  train: ${data.dataset_dir}/2M/train
+  val: ${data.dataset_dir}/8M/train
+
+emb_dirs:
+  train: ${data.dataset_dir}/2M/blip-vid-embs-${model.model.vit}-all
+  val: ${data.dataset_dir}/8M/blip-vid-embs-${model.model.vit}-all

configs/experiment/cirr.yaml

@@ -0,0 +1,13 @@
+# @package _global_
+
+defaults:
+  - override /data: cirr.yaml
+  - override /test: cirr.yaml
+  # - override /model/ckpt: webvid-covr.yaml
+
+model:
+  optimizer:
+    lr: 1e-4
+
+trainer:
+  max_epochs: 6

configs/experiment/covr_hard-negatives.yaml

@@ -0,0 +1,6 @@
+# @package _global_
+
+defaults:
+  - override /data: webvid-covr.yaml
+  - override /test: main.yaml
+  - override /model/loss: cross_entropy

configs/experiment/covr_iterate-triplets.yaml

@@ -0,0 +1,14 @@
+# @package _global_
+
+defaults:
+  - override /data: webvid-covr.yaml
+  - override /test: main.yaml
+
+run_name: "iterate-triplets"
+
+data:
+  iterate: "triplets"
+
+trainer:
+  max_epochs: 2
+  print_interval: 1

configs/experiment/covr_late-fusion.yaml

@@ -0,0 +1,12 @@
+# @package _global_
+
+defaults:
+  - override /data: webvid-covr.yaml
+  - override /model: blip-large_late-fusion.yaml
+  - override /test: webvid-covr_late-fusion.yaml
+
+val: False
+
+model:
+  optimizer:
+    lr: 1e-4

configs/experiment/covr_middle-emb.yaml

@@ -0,0 +1,14 @@
+# @package _global_
+
+run_name: "middle_emb"
+
+defaults:
+  - override /data: webvid-covr.yaml
+  - override /test: webvid-covr.yaml
+
+data:
+  emb_pool: "middle"
+
+test:
+  webvid_covr:
+    emb_pool: "middle"

configs/experiment/covr_only-text.yaml

@@ -0,0 +1,8 @@
+# @package _global_
+
+defaults:
+  - override /data: webvid-covr.yaml
+  - override /test: webvid-covr_text.yaml
+  - override /model: blip-large_text.yaml
+
+val: False

configs/experiment/covr_only-visual.yaml

@@ -0,0 +1,20 @@
+# @package _global_
+
+defaults:
+  - override /data: webvid-covr.yaml
+  - override /test: webvid-covr_visual.yaml
+  - override /model: blip-large_visual.yaml
+
+val: False
+
+run_name: only-visual
+
+machine:
+  batch_size: 64 # We have to reduce the learning rate because we are training the ViT
+
+model:
+  optimizer:
+    lr: 0.125e-4 # We have to reduce the learning rate because we are reducing the batch size
+
+data:
+  emb_pool: mean

configs/experiment/covr_random-frame.yaml

@@ -0,0 +1,10 @@
+# @package _global_
+
+run_name: "random-frame"
+
+defaults:
+  - override /data: webvid-covr.yaml
+  - override /test: webvid-covr.yaml
+
+data:
+  vid_query_method: "random"

configs/experiment/covr_rule-based.yaml

@@ -0,0 +1,8 @@
+# @package _global_
+
+defaults:
+  - override /data: webvid-covr_rule-based.yaml
+  - override /test: main.yaml
+
+trainer:
+  print_interval: 2

configs/experiment/fiq-dress.yaml

@@ -0,0 +1,17 @@
+# @package _global_
+
+defaults:
+  - override /data: fashioniq-dress.yaml
+  - override /test: fashioniq-dress.yaml
+  - override /model/ckpt: webvid-covr.yaml
+
+model:
+  optimizer:
+    lr: 1e-4
+
+machine:
+  batch_size: 256
+
+trainer:
+  max_epochs: 6
+  print_interval: 2

configs/experiment/fiq-shirt.yaml

@@ -0,0 +1,17 @@
+# @package _global_
+
+defaults:
+  - override /data: fashioniq-shirt.yaml
+  - override /test: fashioniq-shirt.yaml
+  - override /model/ckpt: webvid-covr.yaml
+
+model:
+  optimizer:
+    lr: 1e-4
+
+machine:
+  batch_size: 256
+
+trainer:
+  max_epochs: 6
+  print_interval: 2

configs/experiment/fiq-toptee.yaml

@@ -0,0 +1,17 @@
+# @package _global_
+
+defaults:
+  - override /data: fashioniq-toptee.yaml
+  - override /test: fashioniq-toptee.yaml
+  - override /model/ckpt: webvid-covr.yaml
+
+model:
+  optimizer:
+    lr: 1e-4
+
+machine:
+  batch_size: 256
+
+trainer:
+  max_epochs: 6
+  print_interval: 2

configs/machine/default.yaml

@@ -0,0 +1,16 @@
+# path to root directory
+root_dir: "."
+
+# path to working directory
+work_dir: ${hydra:runtime.cwd}
+
+# path to output directory, created dynamically by hydra
+# path generation pattern is specified in `configs/hydra/default.yaml`
+# use it to store all files generated during the run, like ckpts and metrics
+output_dir: ${hydra:runtime.output_dir}
+
+# path to dataset directory
+datasets_dir: ${hydra:runtime.cwd}/datasets/
+
+# path to logging directory
+log_dir: ${paths.root_dir}/logs/

configs/machine/server.yaml

@@ -0,0 +1,8 @@
+name: server
+
+# specific attributes to this machine
+batch_size: 512
+num_workers: 8
+
+defaults:
+  - default@paths

configs/med_config.json

@@ -0,0 +1,21 @@
+{
+  "architectures": [
+    "BertModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "type_vocab_size": 2,
+  "vocab_size": 30524,
+  "encoder_width": 768,
+  "add_cross_attention": true   
+}

configs/model/blip-large.yaml

@@ -0,0 +1,15 @@
+modelname: blip-large
+_target_: src.model.blip_cir.blip_cir
+
+ckpt_path: ${model.ckpt.path}
+
+model:
+  _target_: src.model.blip_cir.BLIPCir
+  med_config: ${paths.work_dir}/configs/med_config.json
+  image_size: ${data.image_size}
+  vit: "large"
+  vit_grad_ckpt: True
+  vit_ckpt_layer: 12
+  embed_dim: 256
+  train_vit: False
+  loss: ${model.loss}

configs/model/blip-large_text.yaml

@@ -0,0 +1,15 @@
+modelname: blip-large-text
+_target_: src.model.blip_cir_text.blip_cir_text
+
+ckpt_path: ${model.ckpt.path}
+
+model:
+  _target_: src.model.blip_cir_text.BLIPCirTextOnly
+  med_config: ${paths.work_dir}/configs/med_config.json
+  image_size: ${data.image_size}
+  vit: "large"
+  vit_grad_ckpt: True
+  vit_ckpt_layer: 12
+  embed_dim: 256
+  train_vit: False
+  loss: ${model.loss}

configs/model/blip-large_visual.yaml

@@ -0,0 +1,15 @@
+modelname: blip-large-visual
+_target_: src.model.blip_cir_visual.blip_cir_visual
+
+ckpt_path: ${model.ckpt.path}
+
+model:
+  _target_: src.model.blip_cir_visual.BLIPCirVisualOnly
+  med_config: ${paths.work_dir}/configs/med_config.json
+  image_size: ${data.image_size}
+  vit: "large"
+  vit_grad_ckpt: True
+  vit_ckpt_layer: 12
+  embed_dim: 256
+  train_vit: True
+  loss: ${model.loss}

configs/model/ckpt/blip-l-coco.yaml

@@ -0,0 +1,3 @@
+name: blip-l-coco
+
+path: "https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_large_retrieval_coco.pth"

configs/model/ckpt/cirr-gt.yaml

@@ -0,0 +1,3 @@
+name: cirr-gt
+
+path: ${paths.work_dir}/outputs/cirr/blip-large/blip-l-coco/tv-False_loss-hnnce_lr-1e-05/base/ckpt_4.ckpt

configs/model/ckpt/cirr_ft-covr+gt.yaml

@@ -0,0 +1,3 @@
+name: cirr_ft-covr+gt
+
+path: ${paths.work_dir}/outputs/cirr/blip-large/webvid-covr/tv-False_loss-hnnce_lr-0.0001/base/ckpt_5.ckpt

configs/model/ckpt/webvid-covr.yaml

@@ -0,0 +1,3 @@
+name: webvid-covr
+
+path: ${paths.work_dir}/outputs/webvid-covr/blip-large/blip-l-coco/tv-False_loss-hnnce_lr-1e-05/good/ckpt_4.ckpt

configs/model/loss/cross_entropy.yaml

@@ -0,0 +1,2 @@
+_target_: src.model.loss.CrossEntropyLoss
+name: ce

configs/model/loss/hn_nce.yaml

@@ -0,0 +1,5 @@
+_target_: src.model.loss.HardNegativeNCE
+name: hnnce
+
+alpha: 1
+beta: 0.5

configs/model/optimizer/adamw.yaml

@@ -0,0 +1,5 @@
+_target_: torch.optim.AdamW
+_partial_: true
+
+lr: 1e-05
+weight_decay: 0.05

configs/model/scheduler/cosine.yaml

@@ -0,0 +1,6 @@
+_target_: src.tools.scheduler.CosineSchedule
+
+init_lr: ${model.optimizer.lr}
+min_lr: 0
+decay_rate: ${model.optimizer.weight_decay}
+max_epochs: ${trainer.max_epochs}

configs/model/scheduler/step.yaml

@@ -0,0 +1,5 @@
+_target_: src.tools.scheduler.StepSchedule
+
+init_lr: ${model.optimizer.lr}
+decay_rate: ${model.optimizer.weight_decay}
+min_lr: 0

configs/test.yaml

@@ -0,0 +1,27 @@
+hydra:
+  run:
+    dir: outputs/test/${model.modelname}/${model.ckpt.name}/${run_name}
+  job: # automatically go to the job folder (needed for hydra > 1.2 with new behavior)
+    chdir: true
+
+# Global configurations shared between different modules
+run_name: base
+
+seed: 1234
+logger_level: INFO
+
+# Composing nested config with default
+defaults:
+  - _self_
+  - data: cirr
+  - test: all
+  - machine: server
+  - trainer: gpu
+  - model: blip-large
+  - model/ckpt: blip-l-coco
+  - model/loss: hn_nce
+  - trainer/logger: none
+
+  - experiment: null
+
+paths: ${machine.paths}

configs/test/all.yaml

@@ -0,0 +1,6 @@
+defaults:
+  - cirr.yaml
+  - webvid-covr.yaml
+  - fashioniq-dress.yaml
+  - fashioniq-shirt.yaml
+  - fashioniq-toptee.yaml

configs/test/cirr.yaml

@@ -0,0 +1,15 @@
+cirr:
+  dataname: cirr
+  _target_: src.data.cirr.CIRRTestDataModule
+
+  test:
+    _target_: src.test.cirr.TestCirr
+
+  batch_size: ${machine.batch_size}
+  num_workers: ${machine.num_workers}
+
+  annotation: ${paths.work_dir}/annotation/cirr/cap.rc2.test1.json
+  img_dirs: ${paths.datasets_dir}/CIRR/images/test1
+  emb_dirs: ${paths.datasets_dir}/CIRR/blip-embs-large/test1
+
+  image_size: 384

configs/test/fashioniq-dress.yaml

@@ -0,0 +1,18 @@
+fashioniq-dress:
+  dataname: fashioniq-dress
+  _target_: src.data.fashioniq.FashionIQTestDataModule
+
+  batch_size: ${machine.batch_size}
+  num_workers: ${machine.num_workers}
+
+  annotation: ${paths.work_dir}/annotation/fashion-iq/cap.dress.val.json
+  targets: ${paths.work_dir}/annotation/fashion-iq/split.dress.val.json
+
+  img_dirs: ${paths.datasets_dir}/fashion-iq/images/
+  emb_dirs: ${paths.datasets_dir}/fashion-iq/blip-embs-large/
+
+  image_size: 384
+
+  test:
+    _target_: src.test.fashioniq.TestFashionIQ
+    category: dress

configs/test/fashioniq-shirt.yaml

@@ -0,0 +1,18 @@
+fashioniq-shirt:
+  dataname: fashioniq-shirt
+  _target_: src.data.fashioniq.FashionIQTestDataModule
+
+  batch_size: ${machine.batch_size}
+  num_workers: ${machine.num_workers}
+
+  annotation: ${paths.work_dir}/annotation/fashion-iq/cap.shirt.val.json
+  targets: ${paths.work_dir}/annotation/fashion-iq/split.shirt.val.json
+
+  img_dirs: ${paths.datasets_dir}/fashion-iq/images/
+  emb_dirs: ${paths.datasets_dir}/fashion-iq/blip-embs-large/
+
+  image_size: 384
+
+  test:
+    _target_: src.test.fashioniq.TestFashionIQ
+    category: shirt

configs/test/fashioniq-toptee.yaml

@@ -0,0 +1,18 @@
+fashioniq-toptee:
+  dataname: fashioniq-toptee
+  _target_: src.data.fashioniq.FashionIQTestDataModule
+
+  batch_size: ${machine.batch_size}
+  num_workers: ${machine.num_workers}
+
+  annotation: ${paths.work_dir}/annotation/fashion-iq/cap.toptee.val.json
+  targets: ${paths.work_dir}/annotation/fashion-iq/split.toptee.val.json
+
+  img_dirs: ${paths.datasets_dir}/fashion-iq/images/
+  emb_dirs: ${paths.datasets_dir}/fashion-iq/blip-embs-large/
+
+  image_size: 384
+
+  test:
+    _target_: src.test.fashioniq.TestFashionIQ
+    category: toptee

configs/test/fashioniq.yaml

@@ -0,0 +1,4 @@
+defaults:
+  - fashioniq-dress.yaml
+  - fashioniq-shirt.yaml
+  - fashioniq-toptee.yaml

configs/test/main.yaml

@@ -0,0 +1,3 @@
+defaults:
+  - cirr.yaml
+  - webvid-covr.yaml

configs/test/webvid-covr.yaml

@@ -0,0 +1,20 @@
+webvid_covr:
+  dataname: webvid-covr
+  _target_: src.data.webvid_covr.WebVidCoVRTestDataModule
+
+  image_size: 384
+
+  vid_query_method: middle
+  vid_frames: 1
+  emb_pool: query
+
+  batch_size: ${machine.batch_size}
+  num_workers: ${machine.num_workers}
+
+  # Paths
+  annotation: ${paths.work_dir}/annotation/webvid-covr/webvid8m-covr_test.csv
+  vid_dirs: ${paths.datasets_dir}/WebVid/8M/train
+  emb_dirs: ${paths.datasets_dir}/WebVid/8M/blip-vid-embs-${model.model.vit}-all
+
+  test:
+    _target_: src.test.webvid_covr.TestWebVidCoVR

configs/test/webvid-covr_text.yaml

@@ -0,0 +1,20 @@
+webvid_covr_text:
+  dataname: webvid-covr_text
+  _target_: src.data.webvid_covr.WebVidCoVRTestDataModule
+
+  image_size: 384
+
+  vid_query_method: middle
+  vid_frames: 1
+  emb_pool: query
+
+  batch_size: ${machine.batch_size}
+  num_workers: ${machine.num_workers}
+
+  # Paths
+  annotation: ${paths.work_dir}/annotation/webvid-covr/webvid8m-covr_test.csv
+  vid_dirs: ${paths.datasets_dir}/WebVid/8M/train
+  emb_dirs: ${paths.datasets_dir}/WebVid/8M/blip-vid-embs-${model.model.vit}-all
+
+  test:
+    _target_: src.test.webvid_covr_exp.TestWebVidCoVRTextOnly

configs/test/webvid-covr_visual.yaml

@@ -0,0 +1,20 @@
+webvid_covr_visual:
+  dataname: webvid-covr_visual
+  _target_: src.data.webvid_covr.WebVidCoVRTestDataModule
+
+  image_size: 384
+
+  vid_query_method: middle
+  vid_frames: 1
+  emb_pool: mean
+
+  batch_size: ${machine.batch_size}
+  num_workers: ${machine.num_workers}
+
+  # Paths
+  annotation: ${paths.work_dir}/annotation/webvid-covr/webvid8m-covr_test.csv
+  vid_dirs: ${paths.datasets_dir}/WebVid/8M/train
+  emb_dirs: ${paths.datasets_dir}/WebVid/8M/blip-vid-embs-${model.model.vit}-all
+
+  test:
+    _target_: src.test.webvid_covr_exp.TestWebVidCoVRVisualOnly

configs/train.yaml

@@ -0,0 +1,33 @@
+hydra:
+  run:
+    dir: outputs/${data.dataname}/${model.modelname}/${model.ckpt.name}/${experiment}/${run_name}
+  job: # automatically go to the job folder (needed for hydra > 1.2 with new behavior)
+    chdir: true
+
+# Global configurations shared between different modules
+experiment: tv-${model.model.train_vit}_loss-${model.model.loss.name}_lr-${model.optimizer.lr}
+run_name: base
+
+seed: 1234
+logger_level: INFO
+
+# Composing nested config with default
+defaults:
+  - _self_
+  - data: webvid-covr
+  - machine: server
+  - trainer: gpu
+  - test: all
+  - trainer/logger: csv
+  - model: blip-large
+  - model/optimizer: adamw
+  - model/scheduler: cosine
+  - model/loss: hn_nce
+  - model/ckpt: blip-l-coco
+
+  - experiment: null
+
+paths: ${machine.paths}
+
+# Flag to validate at the end of every epoch
+val: True

configs/trainer/cpu.yaml

@@ -0,0 +1,5 @@
+defaults:
+  - default.yaml
+
+accelerator: cpu
+devices: 1

configs/trainer/ddp.yaml

@@ -0,0 +1,12 @@
+defaults:
+  - default.yaml
+
+strategy: ddp
+
+accelerator: gpu
+devices: 4
+num_nodes: 1
+
+fabric:
+  num_nodes: ${trainer.num_nodes}
+  strategy: ${trainer.strategy}