nielsr's picture
nielsr HF staff
pad non square image before resize (#1)
from PIL import Image
import requests
import os
from transformers import CLIPSegProcessor, CLIPSegForImageSegmentation
from diffusers import DiffusionPipeline
import torch
from torch import autocast
import gradio as gr
auth_token = os.environ.get("API_TOKEN") or True
url = ""
image =, stream=True).raw)
processor = CLIPSegProcessor.from_pretrained("CIDAS/clipseg-rd64-refined")
model = CLIPSegForImageSegmentation.from_pretrained("CIDAS/clipseg-rd64-refined")
pipe = DiffusionPipeline.from_pretrained(
device = "cuda" if torch.cuda.is_available() else "cpu"
pipe =
def pad_image(image):
w, h = image.size
if w == h:
return image
elif w > h:
new_image =, (w, w), (0, 0, 0))
new_image.paste(image, (0, (w - h) // 2))
return new_image
new_image =, (h, h), (0, 0, 0))
new_image.paste(image, ((h - w) // 2, 0))
return new_image
def process_image(image, text, prompt):
image = pad_image(image)
image = image.resize((512, 512))
with autocast("cuda"):
inpainted_image = pipe(image=image, text=text, prompt=prompt).images[0]
return inpainted_image
title = "Interactive demo: Text-based inpainting with CLIPSeg x Stable Diffusion"
description = "Demo for using CLIPSeg, a CLIP-based model for zero- and one-shot image segmentation. This model can be used to segment things in an image based on text. This way, one can use it to provide a binary mask for Stable Diffusion, which the latter needs to inpaint. To use it, simply upload an image and add a text to mask as well as a text which indicates what to replace, or use one of the examples below and click 'submit'. Results will show up in a few seconds."
article = "<p style='text-align: center'><a href=''>CLIPSeg: Image Segmentation Using Text and Image Prompts</a> | <a href=''>HuggingFace docs</a></p>"
examples = [["example_image.png", "a glass", "a cup"]]
interface = gr.Interface(fn=process_image,
inputs=[gr.Image(type="pil"), gr.Textbox(label="What's the thing you want to replace?"), gr.Textbox(label="What do you want as replacement?")],