from transformers import AutoTokenizer, CLIPProcessor, SiglipModel, AutoProcessor import requests from PIL import Image from modeling_nllb_clip import NLLBCLIPModel import torch.nn.functional as F from sentence_transformers import SentenceTransformer, util from PIL import Image, ImageFile import requests import torch import numpy as np import gradio as gr ## NLLB Inference nllb_clip_processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32") nllb_clip_processor = nllb_clip_processor.image_processor nllb_clip_tokenizer = AutoTokenizer.from_pretrained( "facebook/nllb-200-distilled-600M" ) def nllb_clip_inference(image,labels): labels = labels.split(",") image_inputs = nllb_clip_processor(images=image, return_tensors="pt") text_inputs = nllb_clip_tokenizer(labels, padding="longest", return_tensors="pt",) nllb_clip_model = NLLBCLIPModel.from_pretrained("visheratin/nllb-clip-base") outputs = nllb_clip_model(input_ids = text_inputs.input_ids, attention_mask = text_inputs.attention_mask, pixel_values=image_inputs.pixel_values) normalized_tensor = F.softmax(outputs["logits_per_text"], dim=0) normalized_tensor = normalized_tensor.detach().numpy() return {labels[i]: float(np.array(normalized_tensor)[i]) for i in range(len(labels))} # SentenceTransformers CLIP-ViT-B-32 img_model = SentenceTransformer('clip-ViT-B-32') text_model = SentenceTransformer('sentence-transformers/clip-ViT-B-32-multilingual-v1') def infer_st(image, texts): texts = texts.split(",") img_embeddings = img_model.encode(image) text_embeddings = text_model.encode(texts) cos_sim = util.cos_sim(text_embeddings, img_embeddings) return {texts[i]: float(np.array(cos_sim)[i]) for i in range(len(texts))} ### SigLIP Inference siglip_model = SiglipModel.from_pretrained("google/siglip-base-patch16-256-multilingual") siglip_processor = AutoProcessor.from_pretrained("google/siglip-base-patch16-256-multilingual") def postprocess_siglip(output, labels): return {labels[i]: float(np.array(output[0])[i]) for i in range(len(labels))} def siglip_detector(image, texts): inputs = siglip_processor(text=texts, images=image, return_tensors="pt", padding="max_length") with torch.no_grad(): outputs = siglip_model(**inputs) logits_per_image = outputs.logits_per_image probs = torch.sigmoid(logits_per_image) probs = normalize_tensor(probs) return probs def normalize_tensor(tensor): # no other normalization works well for visual purposes sum_tensor = torch.sum(tensor) normalized_tensor = tensor / sum_tensor return normalized_tensor def infer_siglip(image, candidate_labels): candidate_labels = [label.lstrip(" ") for label in candidate_labels.split(",")] siglip_out = siglip_detector(image, candidate_labels) return postprocess_siglip(siglip_out, labels=candidate_labels) def infer(image, labels): st_out = infer_st(image, labels) nllb_out = nllb_clip_inference(image, labels) siglip_out = infer_siglip(image, labels) return st_out, siglip_out, nllb_out with gr.Blocks() as demo: gr.Markdown("# Compare Multilingual Zero-shot Image Classification") gr.Markdown("Compare the performance of SigLIP and other models on zero-shot classification in this Space.") gr.Markdown("Three models are compared: CLIP-ViT, NLLB-CLIP and SigLIP. Note that SigLIP outputs are normalized for visualization purposes.") gr.Markdown("NLLB-CLIP is a multilingual vision-language model that combines [NLLB](https://ai.meta.com/research/no-language-left-behind/) with [CLIP](https://openai.com/research/clip) to extend CLIP to 200+ languages.") gr.Markdown("CLIP-ViT is CLIP model extended to other languages using [multilingual knowledge distillation](https://arxiv.org/abs/2004.09813).") gr.Markdown("Finally, SigLIP is the state-of-the-art vision-language model released by Google. Multilingual checkpoint is pre-trained by Google.") with gr.Row(): with gr.Column(): image_input = gr.Image(type="pil") text_input = gr.Textbox(label="Input a list of labels") run_button = gr.Button("Run", visible=True) with gr.Column(): st_output = gr.Label(label = "CLIP-ViT Multilingual Output", num_top_classes=3) siglip_output = gr.Label(label = "SigLIP Output", num_top_classes=3) nllb_output = gr.Label(label = "NLLB-CLIP Output", num_top_classes=3) examples = [["./cat.jpg", "eine Katze, köpek, un oiseau"]] gr.Examples( examples = examples, inputs=[image_input, text_input], outputs=[st_output, siglip_output, nllb_output], fn=infer, cache_examples=True ) run_button.click(fn=infer, inputs=[image_input, text_input], outputs=[st_output, siglip_output, nllb_output]) demo.launch()