from transformers import CLIPProcessor, CLIPModel model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32") processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32") def inference(input_img, captions): captions_list = captions.split(",") #url = "http://images.cocodataset.org/val2017/000000039769.jpg" #image = Image.open(requests.get(url, stream=True).raw) inputs = processor(text=captions_list, images=input_img, return_tensors="pt", padding=True) outputs = model(**inputs) logits_per_image = outputs.logits_per_image # this is the image-text similarity score probs = logits_per_image.softmax(dim=1) probabilities_percentages = ', '.join(['{:.2f}%'.format(prob.item() * 100) for prob in probs[0]]) return probabilities_percentages title = "TSAI S18 Assignment: Use a pretrained CLIP model and give a demo on its workig" description = "A simple Gradio interface that accepts an image and some captions, and gives a score as to how much the caption describes the image " examples = [["cats.jpg","a photo of a cat, a photo of a dog"] ] demo = gr.Interface( inference, inputs = [gr.Image(shape=(416, 416), label="Input Image"), gr.Textbox(placeholder="Enter different captions for image, separated by comma")], outputs = [gr.Textbox(label="Probability score of captions")], title = title, description = description, examples = examples, ) demo.launch()