Spaces:
Runtime error
Runtime error
use diffusers
Browse files- app.py +30 -12
- requirements.txt +4 -2
app.py
CHANGED
@@ -1,3 +1,4 @@
|
|
|
|
1 |
from transformers import CLIPSegProcessor, CLIPSegForImageSegmentation
|
2 |
import gradio as gr
|
3 |
from PIL import Image
|
@@ -5,32 +6,49 @@ import torch
|
|
5 |
import matplotlib.pyplot as plt
|
6 |
import cv2
|
7 |
|
8 |
-
|
9 |
-
|
|
|
10 |
|
11 |
-
|
12 |
-
|
|
|
13 |
|
14 |
# predict
|
15 |
with torch.no_grad():
|
16 |
-
outputs =
|
17 |
preds = outputs.logits
|
18 |
|
19 |
-
|
20 |
-
plt.imsave(
|
21 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
22 |
|
23 |
|
24 |
|
25 |
-
title = "Interactive demo:
|
26 |
|
27 |
-
description = "Demo for
|
28 |
|
29 |
article = "<p style='text-align: center'><a href='https://arxiv.org/abs/2112.10003'>CLIPSeg: Image Segmentation Using Text and Image Prompts</a> | <a href='https://huggingface.co/docs/transformers/main/en/model_doc/clipseg'>HuggingFace docs</a></p>"
|
30 |
|
31 |
interface = gr.Interface(fn=process_image,
|
32 |
-
inputs=[
|
33 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
34 |
title=title,
|
35 |
description=description,
|
36 |
article=article)
|
|
|
1 |
+
from diffusers import StableDiffusionInpaintPipeline
|
2 |
from transformers import CLIPSegProcessor, CLIPSegForImageSegmentation
|
3 |
import gradio as gr
|
4 |
from PIL import Image
|
|
|
6 |
import matplotlib.pyplot as plt
|
7 |
import cv2
|
8 |
|
9 |
+
clip_seg_processor = CLIPSegProcessor.from_pretrained("CIDAS/clipseg-rd64-refined")
|
10 |
+
clip_seg_model = CLIPSegForImageSegmentation.from_pretrained("CIDAS/clipseg-rd64-refined")
|
11 |
+
sd_inpainting_model = StableDiffusionInpaintPipeline.from_pretrained("stable-diffusion-2-inpainting", revision="fp16", torch_dtype=torch.float16)
|
12 |
|
13 |
+
|
14 |
+
def process_image(image, prompt1, prompt2):
|
15 |
+
inputs = clip_seg_processor(text=prompt, images=image, padding="max_length", return_tensors="pt")
|
16 |
|
17 |
# predict
|
18 |
with torch.no_grad():
|
19 |
+
outputs = clip_seg_model(**inputs)
|
20 |
preds = outputs.logits
|
21 |
|
22 |
+
filename_mask = f"mask.png"
|
23 |
+
plt.imsave(filename_mask, torch.sigmoid(preds))
|
24 |
+
mask_image = Image.open(filename_mask).convert("RGB")
|
25 |
+
|
26 |
+
with torch.no_grad():
|
27 |
+
output_image = sd_inpainting_model(prompt=prompt2, image=image, mask_image=mask_image).images[0]
|
28 |
+
filename_sd = f"sd.png"
|
29 |
+
output_image.save(filename_sd)
|
30 |
+
sd_image = Image.open(filename_sd).convert("RGB")
|
31 |
+
|
32 |
+
return [mask_image, sd_image]
|
33 |
|
34 |
|
35 |
|
36 |
+
title = "Interactive demo: Prompt based inPainting using CLIPSeg x Stable Diffusion"
|
37 |
|
38 |
+
description = "Demo for prompt based inPainting. It uses CLIPSeg, a CLIP-based model for zero- and one-shot image segmentation. Once it identifies the image segment based on a text mask, or use one of the examples below and click 'submit'. Results will show up in a few seconds."
|
39 |
|
40 |
article = "<p style='text-align: center'><a href='https://arxiv.org/abs/2112.10003'>CLIPSeg: Image Segmentation Using Text and Image Prompts</a> | <a href='https://huggingface.co/docs/transformers/main/en/model_doc/clipseg'>HuggingFace docs</a></p>"
|
41 |
|
42 |
interface = gr.Interface(fn=process_image,
|
43 |
+
inputs=[
|
44 |
+
gr.Image(type="pil"),
|
45 |
+
gr.Textbox(label="What to identify"),
|
46 |
+
gr.Textbox(label="What to replace it with"),
|
47 |
+
],
|
48 |
+
outputs=[
|
49 |
+
gr.Image(type="pil"),
|
50 |
+
gr.Image(type="pil"),
|
51 |
+
],
|
52 |
title=title,
|
53 |
description=description,
|
54 |
article=article)
|
requirements.txt
CHANGED
@@ -1,5 +1,7 @@
|
|
1 |
-
git+https://github.com/huggingface/transformers.git
|
2 |
accelerate
|
3 |
-
|
4 |
torch
|
5 |
opencv-python
|
|
|
|
|
|
|
|
|
|
1 |
accelerate
|
2 |
+
diffusers
|
3 |
torch
|
4 |
opencv-python
|
5 |
+
transformers
|
6 |
+
accelerate
|
7 |
+
scipy
|