pipeline/clip_wrapper.py

from typing import List

import torch
from PIL import Image
from transformers import CLIPModel, CLIPProcessor

MODEL_DIM = 512


class ClipWrapper:
    def __init__(self):
        self.model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
        self.processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")

    def images2vec(self, images: List[Image.Image]) -> torch.Tensor:
        inputs = self.processor(images=images, return_tensors="pt")
        with torch.no_grad():
            model_inputs = {k: v.to(self.model.device) for k, v in inputs.items()}
            image_embeds = self.model.vision_model(**model_inputs)
            clip_vectors = self.model.visual_projection(image_embeds[1])
        return clip_vectors / clip_vectors.norm(dim=-1, keepdim=True)

    def texts2vec(self, texts: List[str]) -> torch.Tensor:
        inputs = self.processor(text=texts, return_tensors="pt", padding=True)
        with torch.no_grad():
            model_inputs = {k: v.to(self.model.device) for k, v in inputs.items()}
            text_embeds = self.model.text_model(**model_inputs)
            text_vectors = self.model.text_projection(text_embeds[1])
        return text_vectors / text_vectors.norm(dim=-1, keepdim=True)