Duplicate from ArtGAN/Stable-Diffusion-ControlNet-WebUI

Co-authored-by: Kadir Nar <kadirnar@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,16 @@
+---
+title: Stable Diffusion ControlNet WebUI
+emoji: ⚡
+colorFrom: gray
+colorTo: red
+sdk: gradio
+sdk_version: 3.19.0
+app_file: app.py
+pinned: false
+license: apache-2.0
+tags:
+- making-demos
+duplicated_from: ArtGAN/Stable-Diffusion-ControlNet-WebUI
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,33 @@
+import gradio as gr
+
+from diffusion_webui import (
+    StableDiffusionControlNetGenerator,
+    StableDiffusionControlNetInpaintGenerator,
+    StableDiffusionImage2ImageGenerator,
+    StableDiffusionInpaintGenerator,
+    StableDiffusionText2ImageGenerator,
+)
+
+
+def diffusion_app():
+    app = gr.Blocks()
+    with app:
+        with gr.Row():
+            with gr.Column():
+                with gr.Tab(label="Text2Image"):
+                    StableDiffusionText2ImageGenerator.app()
+                with gr.Tab(label="Image2Image"):
+                    StableDiffusionImage2ImageGenerator.app()
+                with gr.Tab(label="Inpaint"):
+                    StableDiffusionInpaintGenerator.app()
+                with gr.Tab(label="Controlnet"):
+                    StableDiffusionControlNetGenerator.app()
+                with gr.Tab(label="Controlnet Inpaint"):
+                    StableDiffusionControlNetInpaintGenerator.app()
+
+    app.queue(concurrency_count=1)
+    app.launch(debug=True, enable_queue=True)
+
+
+if __name__ == "__main__":
+    diffusion_app()

diffusion_webui/__init__.py

@@ -0,0 +1,17 @@
+from diffusion_webui.diffusion_models.controlnet_inpaint_pipeline import (
+    StableDiffusionControlNetInpaintGenerator,
+)
+from diffusion_webui.diffusion_models.controlnet_pipeline import (
+    StableDiffusionControlNetGenerator,
+)
+from diffusion_webui.diffusion_models.img2img_app import (
+    StableDiffusionImage2ImageGenerator,
+)
+from diffusion_webui.diffusion_models.inpaint_app import (
+    StableDiffusionInpaintGenerator,
+)
+from diffusion_webui.diffusion_models.text2img_app import (
+    StableDiffusionText2ImageGenerator,
+)
+
+__version__ = "2.5.0"

diffusion_webui/diffusion_models/base_controlnet_pipeline.py

@@ -0,0 +1,31 @@
+class ControlnetPipeline:
+    def __init__(self):
+        self.pipe = None
+
+    def load_model(self, stable_model_path: str, controlnet_model_path: str):
+        raise NotImplementedError()
+
+    def load_image(self, image_path: str):
+        raise NotImplementedError()
+
+    def controlnet_preprocces(self, read_image: str):
+        raise NotImplementedError()
+
+    def generate_image(
+        self,
+        image_path: str,
+        stable_model_path: str,
+        controlnet_model_path: str,
+        prompt: str,
+        negative_prompt: str,
+        num_images_per_prompt: int,
+        guidance_scale: int,
+        num_inference_step: int,
+        controlnet_conditioning_scale: int,
+        scheduler: str,
+        seed_generator: int,
+    ):
+        raise NotImplementedError()
+
+    def web_interface():
+        raise NotImplementedError()

diffusion_webui/diffusion_models/controlnet_inpaint_pipeline.py

@@ -0,0 +1,258 @@
+import gradio as gr
+import numpy as np
+import torch
+from diffusers import ControlNetModel, StableDiffusionControlNetInpaintPipeline
+from PIL import Image
+
+from diffusion_webui.diffusion_models.base_controlnet_pipeline import (
+    ControlnetPipeline,
+)
+from diffusion_webui.utils.model_list import (
+    controlnet_model_list,
+    stable_model_list,
+)
+from diffusion_webui.utils.preprocces_utils import PREPROCCES_DICT
+from diffusion_webui.utils.scheduler_list import (
+    SCHEDULER_MAPPING,
+    get_scheduler,
+)
+
+
+class StableDiffusionControlNetInpaintGenerator(ControlnetPipeline):
+    def __init__(self):
+        super().__init__()
+
+    def load_model(self, stable_model_path, controlnet_model_path, scheduler):
+        if self.pipe is None or self.pipe.model_name != stable_model_path or self.pipe.scheduler_name != scheduler:
+            controlnet = ControlNetModel.from_pretrained(
+                controlnet_model_path, torch_dtype=torch.float16
+            )
+            self.pipe = (
+                StableDiffusionControlNetInpaintPipeline.from_pretrained(
+                    pretrained_model_name_or_path=stable_model_path,
+                    controlnet=controlnet,
+                    safety_checker=None,
+                    torch_dtype=torch.float16,
+                )
+            )
+            
+            self.pipe.model_name = stable_model_path
+            self.pipe.scheduler_name = scheduler
+            self.pipe = get_scheduler(pipe=self.pipe, scheduler=scheduler)
+            self.pipe.to("cuda")
+            self.pipe.enable_xformers_memory_efficient_attention()
+
+        return self.pipe
+
+    def load_image(self, image):
+        image = np.array(image)
+        image = Image.fromarray(image)
+        return image
+
+    def controlnet_preprocces(
+        self,
+        read_image: str,
+        preprocces_type: str,
+    ):
+        processed_image = PREPROCCES_DICT[preprocces_type](read_image)
+        return processed_image
+
+    def generate_image(
+        self,
+        image_path: str,
+        stable_model_path: str,
+        controlnet_model_path: str,
+        prompt: str,
+        negative_prompt: str,
+        num_images_per_prompt: int,
+        height: int,
+        width: int,
+        strength: int,
+        guess_mode: bool,
+        guidance_scale: int,
+        num_inference_step: int,
+        controlnet_conditioning_scale: int,
+        scheduler: str,
+        seed_generator: int,
+        preprocces_type: str,
+    ):
+        normal_image = image_path["image"].convert("RGB").resize((512, 512))
+        mask_image = image_path["mask"].convert("RGB").resize((512, 512))
+
+        normal_image = self.load_image(image=normal_image)
+        mask_image = self.load_image(image=mask_image)
+
+        control_image = self.controlnet_preprocces(
+            read_image=normal_image, preprocces_type=preprocces_type
+        )
+        pipe = self.load_model(
+            stable_model_path=stable_model_path,
+            controlnet_model_path=controlnet_model_path,
+            scheduler=scheduler,
+        )
+
+        if seed_generator == 0:
+            random_seed = torch.randint(0, 1000000, (1,))
+            generator = torch.manual_seed(random_seed)
+        else:
+            generator = torch.manual_seed(seed_generator)
+
+        output = pipe(
+            prompt=prompt,
+            image=normal_image,
+            height=height,
+            width=width,
+            mask_image=mask_image,
+            strength=strength,
+            guess_mode=guess_mode,
+            control_image=control_image,
+            negative_prompt=negative_prompt,
+            num_images_per_prompt=num_images_per_prompt,
+            num_inference_steps=num_inference_step,
+            guidance_scale=guidance_scale,
+            controlnet_conditioning_scale=float(controlnet_conditioning_scale),
+            generator=generator,
+        ).images
+
+        return output
+
+    def app():
+        with gr.Blocks():
+            with gr.Row():
+                with gr.Column():
+                    controlnet_inpaint_image_path = gr.Image(
+                        source="upload",
+                        tool="sketch",
+                        elem_id="image_upload",
+                        type="pil",
+                        label="Upload",
+                    ).style(height=260)
+
+                    controlnet_inpaint_prompt = gr.Textbox(
+                        lines=1, placeholder="Prompt", show_label=False
+                    )
+                    controlnet_inpaint_negative_prompt = gr.Textbox(
+                        lines=1, placeholder="Negative Prompt", show_label=False
+                    )
+
+                    with gr.Row():
+                        with gr.Column():
+                            controlnet_inpaint_stable_model_path = gr.Dropdown(
+                                choices=stable_model_list,
+                                value=stable_model_list[0],
+                                label="Stable Model Path",
+                            )
+                            controlnet_inpaint_preprocces_type = gr.Dropdown(
+                                choices=list(PREPROCCES_DICT.keys()),
+                                value=list(PREPROCCES_DICT.keys())[0],
+                                label="Preprocess Type",
+                            )
+                            controlnet_inpaint_conditioning_scale = gr.Slider(
+                                minimum=0.0,
+                                maximum=1.0,
+                                step=0.1,
+                                value=1.0,
+                                label="ControlNet Conditioning Scale",
+                            )
+                            controlnet_inpaint_guidance_scale = gr.Slider(
+                                minimum=0.1,
+                                maximum=15,
+                                step=0.1,
+                                value=7.5,
+                                label="Guidance Scale",
+                            )
+                            controlnet_inpaint_height = gr.Slider(
+                                minimum=128,
+                                maximum=1280,
+                                step=32,
+                                value=512,
+                                label="Height",
+                            )
+                            controlnet_inpaint_width = gr.Slider(
+                                minimum=128,
+                                maximum=1280,
+                                step=32,
+                                value=512,
+                                label="Width",
+                            )
+                            controlnet_inpaint_guess_mode = gr.Checkbox(
+                                label="Guess Mode"
+                            )
+
+                        with gr.Column():
+                            controlnet_inpaint_model_path = gr.Dropdown(
+                                choices=controlnet_model_list,
+                                value=controlnet_model_list[0],
+                                label="ControlNet Model Path",
+                            )
+                            controlnet_inpaint_scheduler = gr.Dropdown(
+                                choices=list(SCHEDULER_MAPPING.keys()),
+                                value=list(SCHEDULER_MAPPING.keys())[0],
+                                label="Scheduler",
+                            )
+                            controlnet_inpaint_strength = gr.Slider(
+                                minimum=0.1,
+                                maximum=15,
+                                step=0.1,
+                                value=7.5,
+                                label="Strength",
+                            )
+                            controlnet_inpaint_num_inference_step = gr.Slider(
+                                minimum=1,
+                                maximum=150,
+                                step=1,
+                                value=30,
+                                label="Num Inference Step",
+                            )
+                            controlnet_inpaint_num_images_per_prompt = (
+                                gr.Slider(
+                                    minimum=1,
+                                    maximum=4,
+                                    step=1,
+                                    value=1,
+                                    label="Number Of Images",
+                                )
+                            )
+                            controlnet_inpaint_seed_generator = gr.Slider(
+                                minimum=0,
+                                maximum=1000000,
+                                step=1,
+                                value=0,
+                                label="Seed(0 for random)",
+                            )
+
+                    # Button to generate the image
+                    controlnet_inpaint_predict_button = gr.Button(
+                        value="Generate Image"
+                    )
+
+                with gr.Column():
+                    # Gallery to display the generated images
+                    controlnet_inpaint_output_image = gr.Gallery(
+                        label="Generated images",
+                        show_label=False,
+                        elem_id="gallery",
+                    ).style(grid=(1, 2))
+
+        controlnet_inpaint_predict_button.click(
+            fn=StableDiffusionControlNetInpaintGenerator().generate_image,
+            inputs=[
+                controlnet_inpaint_image_path,
+                controlnet_inpaint_stable_model_path,
+                controlnet_inpaint_model_path,
+                controlnet_inpaint_prompt,
+                controlnet_inpaint_negative_prompt,
+                controlnet_inpaint_num_images_per_prompt,
+                controlnet_inpaint_height,
+                controlnet_inpaint_width,
+                controlnet_inpaint_strength,
+                controlnet_inpaint_guess_mode,
+                controlnet_inpaint_guidance_scale,
+                controlnet_inpaint_num_inference_step,
+                controlnet_inpaint_conditioning_scale,
+                controlnet_inpaint_scheduler,
+                controlnet_inpaint_seed_generator,
+                controlnet_inpaint_preprocces_type,
+            ],
+            outputs=[controlnet_inpaint_output_image],
+        )

diffusion_webui/diffusion_models/controlnet_pipeline.py

@@ -0,0 +1,262 @@
+import gradio as gr
+import torch
+import cv2
+from diffusers import ControlNetModel, StableDiffusionControlNetPipeline
+from PIL import Image
+
+from diffusion_webui.diffusion_models.base_controlnet_pipeline import (
+    ControlnetPipeline,
+)
+from diffusion_webui.utils.model_list import (
+    controlnet_model_list,
+    stable_model_list,
+)
+from diffusion_webui.utils.preprocces_utils import PREPROCCES_DICT
+from diffusion_webui.utils.scheduler_list import (
+    SCHEDULER_MAPPING,
+    get_scheduler,
+)
+
+
+stable_model_list = [
+    "runwayml/stable-diffusion-v1-5",
+    "dreamlike-art/dreamlike-diffusion-1.0",
+    "kadirnar/maturemalemix_v0",
+    "kadirnar/DreamShaper_v6"
+]
+
+stable_inpiant_model_list = [
+    "stabilityai/stable-diffusion-2-inpainting",
+    "runwayml/stable-diffusion-inpainting",
+    "saik0s/realistic_vision_inpainting",
+]
+
+controlnet_model_list = [
+    "lllyasviel/control_v11p_sd15_canny",
+    "lllyasviel/control_v11f1p_sd15_depth",
+    "lllyasviel/control_v11p_sd15_openpose",
+    "lllyasviel/control_v11p_sd15_scribble",
+    "lllyasviel/control_v11p_sd15_mlsd",
+    "lllyasviel/control_v11e_sd15_shuffle",
+    "lllyasviel/control_v11e_sd15_ip2p",
+    "lllyasviel/control_v11p_sd15_lineart",
+    "lllyasviel/control_v11p_sd15s2_lineart_anime",
+    "lllyasviel/control_v11p_sd15_softedge",
+]
+
+class StableDiffusionControlNetGenerator(ControlnetPipeline):
+    def __init__(self):
+        self.pipe = None
+        
+    def load_model(self, stable_model_path, controlnet_model_path, scheduler):
+        if self.pipe is None or self.pipe.model_name != stable_model_path or self.pipe.scheduler_name != scheduler:
+            controlnet = ControlNetModel.from_pretrained(
+                controlnet_model_path, torch_dtype=torch.float16
+            )
+            self.pipe = StableDiffusionControlNetPipeline.from_pretrained(
+                pretrained_model_name_or_path=stable_model_path,
+                controlnet=controlnet,
+                safety_checker=None,
+                torch_dtype=torch.float16,
+            )
+            self.pipe.model_name = stable_model_path
+            self.pipe.scheduler_name = scheduler
+            
+            self.pipe = get_scheduler(pipe=self.pipe, scheduler=scheduler)
+            self.pipe.scheduler_name = scheduler
+            self.pipe.to("cuda")
+            self.pipe.enable_xformers_memory_efficient_attention()
+    
+        return self.pipe
+
+
+    def controlnet_preprocces(
+        self,
+        read_image: str,
+        preprocces_type: str,
+    ):
+        processed_image = PREPROCCES_DICT[preprocces_type](read_image)
+        return processed_image
+
+    def generate_image(
+        self,
+        image_path: str,
+        stable_model_path: str,
+        controlnet_model_path: str,
+        height: int,
+        width: int,
+        guess_mode: bool,
+        controlnet_conditioning_scale: int,
+        prompt: str,
+        negative_prompt: str,
+        num_images_per_prompt: int,
+        guidance_scale: int,
+        num_inference_step: int,
+        scheduler: str,
+        seed_generator: int,
+        preprocces_type: str,
+    ):
+        pipe = self.load_model(
+            stable_model_path=stable_model_path,
+            controlnet_model_path=controlnet_model_path,
+            scheduler=scheduler,
+        )
+        if preprocces_type== "ScribbleXDOG":
+            read_image = cv2.imread(image_path)
+            controlnet_image = self.controlnet_preprocces(read_image=read_image, preprocces_type=preprocces_type)[0]
+            controlnet_image = Image.fromarray(controlnet_image)
+            
+        elif preprocces_type== "None":
+            controlnet_image = self.controlnet_preprocces(read_image=image_path, preprocces_type=preprocces_type)
+        else:
+            read_image = Image.open(image_path)
+            controlnet_image = self.controlnet_preprocces(read_image=read_image, preprocces_type=preprocces_type)
+
+        if seed_generator == 0:
+            random_seed = torch.randint(0, 1000000, (1,))
+            generator = torch.manual_seed(random_seed)
+        else:
+            generator = torch.manual_seed(seed_generator)
+        
+
+        output = pipe(
+            prompt=prompt,
+            height=height,
+            width=width,
+            controlnet_conditioning_scale=float(controlnet_conditioning_scale),
+            guess_mode=guess_mode,
+            image=controlnet_image,
+            negative_prompt=negative_prompt,
+            num_images_per_prompt=num_images_per_prompt,
+            num_inference_steps=num_inference_step,
+            guidance_scale=guidance_scale,
+            generator=generator,
+        ).images
+
+        return output
+
+    def app():
+        with gr.Blocks():
+            with gr.Row():
+                with gr.Column():
+                    controlnet_image_path = gr.Image(
+                        type="filepath", label="Image"
+                    ).style(height=260)
+                    controlnet_prompt = gr.Textbox(
+                        lines=1, placeholder="Prompt", show_label=False
+                    )
+                    controlnet_negative_prompt = gr.Textbox(
+                        lines=1, placeholder="Negative Prompt", show_label=False
+                    )
+
+                    with gr.Row():
+                        with gr.Column():
+                            controlnet_stable_model_path = gr.Dropdown(
+                                choices=stable_model_list,
+                                value=stable_model_list[0],
+                                label="Stable Model Path",
+                            )
+                            controlnet_preprocces_type = gr.Dropdown(
+                                choices=list(PREPROCCES_DICT.keys()),
+                                value=list(PREPROCCES_DICT.keys())[0],
+                                label="Preprocess Type",
+                            )
+                            controlnet_conditioning_scale = gr.Slider(
+                                minimum=0.0,
+                                maximum=1.0,
+                                step=0.1,
+                                value=1.0,
+                                label="ControlNet Conditioning Scale",
+                            )
+                            controlnet_guidance_scale = gr.Slider(
+                                minimum=0.1,
+                                maximum=15,
+                                step=0.1,
+                                value=7.5,
+                                label="Guidance Scale",
+                            )
+                            controlnet_height = gr.Slider(
+                                minimum=128,
+                                maximum=1280,
+                                step=32,
+                                value=512,
+                                label="Height",
+                            )
+                            controlnet_width = gr.Slider(
+                                minimum=128,
+                                maximum=1280,
+                                step=32,
+                                value=512,
+                                label="Width",
+                            )
+
+                        with gr.Row():
+                            with gr.Column():
+                                controlnet_model_path = gr.Dropdown(
+                                    choices=controlnet_model_list,
+                                    value=controlnet_model_list[0],
+                                    label="ControlNet Model Path",
+                                )
+                                controlnet_scheduler = gr.Dropdown(
+                                    choices=list(SCHEDULER_MAPPING.keys()),
+                                    value=list(SCHEDULER_MAPPING.keys())[0],
+                                    label="Scheduler",
+                                )
+                                controlnet_num_inference_step = gr.Slider(
+                                    minimum=1,
+                                    maximum=150,
+                                    step=1,
+                                    value=30,
+                                    label="Num Inference Step",
+                                )
+
+                                controlnet_num_images_per_prompt = gr.Slider(
+                                    minimum=1,
+                                    maximum=4,
+                                    step=1,
+                                    value=1,
+                                    label="Number Of Images",
+                                )
+                                controlnet_seed_generator = gr.Slider(
+                                    minimum=0,
+                                    maximum=1000000,
+                                    step=1,
+                                    value=0,
+                                    label="Seed(0 for random)",
+                                )
+                                controlnet_guess_mode = gr.Checkbox(
+                                    label="Guess Mode"
+                                )
+
+                    # Button to generate the image
+                    predict_button = gr.Button(value="Generate Image")
+
+                with gr.Column():
+                    # Gallery to display the generated images
+                    output_image = gr.Gallery(
+                        label="Generated images",
+                        show_label=False,
+                        elem_id="gallery",
+                    ).style(grid=(1, 2))
+
+        predict_button.click(
+            fn=StableDiffusionControlNetGenerator().generate_image,
+            inputs=[
+                controlnet_image_path,
+                controlnet_stable_model_path,
+                controlnet_model_path,
+                controlnet_height,
+                controlnet_width,
+                controlnet_guess_mode,
+                controlnet_conditioning_scale,
+                controlnet_prompt,
+                controlnet_negative_prompt,
+                controlnet_num_images_per_prompt,
+                controlnet_guidance_scale,
+                controlnet_num_inference_step,
+                controlnet_scheduler,
+                controlnet_seed_generator,
+                controlnet_preprocces_type,
+            ],
+            outputs=[output_image],
+        )

diffusion_webui/diffusion_models/img2img_app.py

@@ -0,0 +1,155 @@
+import gradio as gr
+import torch
+from diffusers import StableDiffusionImg2ImgPipeline
+from PIL import Image
+
+from diffusion_webui.utils.model_list import stable_model_list
+from diffusion_webui.utils.scheduler_list import (
+    SCHEDULER_MAPPING,
+    get_scheduler,
+)
+
+
+class StableDiffusionImage2ImageGenerator:
+    def __init__(self):
+        self.pipe = None
+
+    def load_model(self, stable_model_path, scheduler):
+        if self.pipe is None or self.pipe.model_name != stable_model_path or self.pipe.scheduler_name != scheduler:
+            self.pipe = StableDiffusionImg2ImgPipeline.from_pretrained(
+                stable_model_path, safety_checker=None, torch_dtype=torch.float16
+            )
+            
+            self.pipe.model_name = stable_model_path
+            self.pipe.scheduler_name = scheduler
+            self.pipe = get_scheduler(pipe=self.pipe, scheduler=scheduler)
+            self.pipe.to("cuda")
+            self.pipe.enable_xformers_memory_efficient_attention()
+
+        return self.pipe
+
+    def generate_image(
+        self,
+        image_path: str,
+        stable_model_path: str,
+        prompt: str,
+        negative_prompt: str,
+        num_images_per_prompt: int,
+        scheduler: str,
+        guidance_scale: int,
+        num_inference_step: int,
+        seed_generator=0,
+    ):
+        pipe = self.load_model(
+            stable_model_path=stable_model_path,
+            scheduler=scheduler,
+        )
+
+        if seed_generator == 0:
+            random_seed = torch.randint(0, 1000000, (1,))
+            generator = torch.manual_seed(random_seed)
+        else:
+            generator = torch.manual_seed(seed_generator)
+
+        image = Image.open(image_path)
+        images = pipe(
+            prompt,
+            image=image,
+            negative_prompt=negative_prompt,
+            num_images_per_prompt=num_images_per_prompt,
+            num_inference_steps=num_inference_step,
+            guidance_scale=guidance_scale,
+            generator=generator,
+        ).images
+
+        return images
+
+    def app():
+        with gr.Blocks():
+            with gr.Row():
+                with gr.Column():
+                    image2image_image_file = gr.Image(
+                        type="filepath", label="Image"
+                    ).style(height=260)
+
+                    image2image_prompt = gr.Textbox(
+                        lines=1,
+                        placeholder="Prompt",
+                        show_label=False,
+                    )
+
+                    image2image_negative_prompt = gr.Textbox(
+                        lines=1,
+                        placeholder="Negative Prompt",
+                        show_label=False,
+                    )
+
+                    with gr.Row():
+                        with gr.Column():
+                            image2image_model_path = gr.Dropdown(
+                                choices=stable_model_list,
+                                value=stable_model_list[0],
+                                label="Stable Model Id",
+                            )
+
+                            image2image_guidance_scale = gr.Slider(
+                                minimum=0.1,
+                                maximum=15,
+                                step=0.1,
+                                value=7.5,
+                                label="Guidance Scale",
+                            )
+                            image2image_num_inference_step = gr.Slider(
+                                minimum=1,
+                                maximum=100,
+                                step=1,
+                                value=50,
+                                label="Num Inference Step",
+                            )
+                        with gr.Row():
+                            with gr.Column():
+                                image2image_scheduler = gr.Dropdown(
+                                    choices=list(SCHEDULER_MAPPING.keys()),
+                                    value=list(SCHEDULER_MAPPING.keys())[0],
+                                    label="Scheduler",
+                            )
+                                image2image_num_images_per_prompt = gr.Slider(
+                                    minimum=1,
+                                    maximum=30,
+                                    step=1,
+                                    value=1,
+                                    label="Number Of Images",
+                                )
+
+                                image2image_seed_generator = gr.Slider(
+                                    minimum=0,
+                                    maximum=1000000,
+                                    step=1,
+                                    value=0,
+                                    label="Seed(0 for random)",
+                                )
+
+                    image2image_predict_button = gr.Button(value="Generator")
+
+                with gr.Column():
+                    output_image = gr.Gallery(
+                        label="Generated images",
+                        show_label=False,
+                        elem_id="gallery",
+                    ).style(grid=(1, 2))
+
+        image2image_predict_button.click(
+            fn=StableDiffusionImage2ImageGenerator().generate_image,
+            inputs=[
+                image2image_image_file,
+                image2image_model_path,
+                image2image_prompt,
+                image2image_negative_prompt,
+                image2image_num_images_per_prompt,
+                image2image_scheduler,
+                image2image_guidance_scale,
+                image2image_num_inference_step,
+                image2image_seed_generator,
+            ],
+            outputs=[output_image],
+        )

diffusion_webui/diffusion_models/inpaint_app.py

@@ -0,0 +1,149 @@
+import gradio as gr
+import torch
+from diffusers import DiffusionPipeline
+
+from diffusion_webui.utils.model_list import stable_inpiant_model_list
+
+
+class StableDiffusionInpaintGenerator:
+    def __init__(self):
+        self.pipe = None
+
+    def load_model(self, stable_model_path):
+        if self.pipe is None or self.pipe.model_name != stable_model_path:
+            self.pipe = DiffusionPipeline.from_pretrained(
+                stable_model_path, revision="fp16", torch_dtype=torch.float16
+            )
+            self.pipe.to("cuda")
+            self.pipe.enable_xformers_memory_efficient_attention()
+            self.pipe.model_name = stable_model_path
+            
+            
+        return self.pipe
+
+    def generate_image(
+        self,
+        pil_image: str,
+        stable_model_path: str,
+        prompt: str,
+        negative_prompt: str,
+        num_images_per_prompt: int,
+        guidance_scale: int,
+        num_inference_step: int,
+        seed_generator=0,
+    ):
+        image = pil_image["image"].convert("RGB").resize((512, 512))
+        mask_image = pil_image["mask"].convert("RGB").resize((512, 512))
+        pipe = self.load_model(stable_model_path)
+
+        if seed_generator == 0:
+            random_seed = torch.randint(0, 1000000, (1,))
+            generator = torch.manual_seed(random_seed)
+        else:
+            generator = torch.manual_seed(seed_generator)
+
+        output = pipe(
+            prompt=prompt,
+            image=image,
+            mask_image=mask_image,
+            negative_prompt=negative_prompt,
+            num_images_per_prompt=num_images_per_prompt,
+            num_inference_steps=num_inference_step,
+            guidance_scale=guidance_scale,
+            generator=generator,
+        ).images
+
+        return output
+
+    def app():
+        with gr.Blocks():
+            with gr.Row():
+                with gr.Column():
+                    stable_diffusion_inpaint_image_file = gr.Image(
+                        source="upload",
+                        tool="sketch",
+                        elem_id="image_upload",
+                        type="pil",
+                        label="Upload",
+                    ).style(height=260)
+
+                    stable_diffusion_inpaint_prompt = gr.Textbox(
+                        lines=1,
+                        placeholder="Prompt",
+                        show_label=False,
+                    )
+
+                    stable_diffusion_inpaint_negative_prompt = gr.Textbox(
+                        lines=1,
+                        placeholder="Negative Prompt",
+                        show_label=False,
+                    )
+                    stable_diffusion_inpaint_model_id = gr.Dropdown(
+                        choices=stable_inpiant_model_list,
+                        value=stable_inpiant_model_list[0],
+                        label="Inpaint Model Id",
+                    )
+                    with gr.Row():
+                        with gr.Column():
+                            stable_diffusion_inpaint_guidance_scale = gr.Slider(
+                                minimum=0.1,
+                                maximum=15,
+                                step=0.1,
+                                value=7.5,
+                                label="Guidance Scale",
+                            )
+
+                            stable_diffusion_inpaint_num_inference_step = (
+                                gr.Slider(
+                                    minimum=1,
+                                    maximum=100,
+                                    step=1,
+                                    value=50,
+                                    label="Num Inference Step",
+                                )
+                            )
+
+                        with gr.Row():
+                            with gr.Column():
+                                stable_diffusion_inpiant_num_images_per_prompt = gr.Slider(
+                                    minimum=1,
+                                    maximum=10,
+                                    step=1,
+                                    value=1,
+                                    label="Number Of Images",
+                                )
+                                stable_diffusion_inpaint_seed_generator = (
+                                    gr.Slider(
+                                        minimum=0,
+                                        maximum=1000000,
+                                        step=1,
+                                        value=0,
+                                        label="Seed(0 for random)",
+                                    )
+                                )
+
+                    stable_diffusion_inpaint_predict = gr.Button(
+                        value="Generator"
+                    )
+
+                with gr.Column():
+                    output_image = gr.Gallery(
+                        label="Generated images",
+                        show_label=False,
+                        elem_id="gallery",
+                    ).style(grid=(1, 2))
+
+            stable_diffusion_inpaint_predict.click(
+                fn=StableDiffusionInpaintGenerator().generate_image,
+                inputs=[
+                    stable_diffusion_inpaint_image_file,
+                    stable_diffusion_inpaint_model_id,
+                    stable_diffusion_inpaint_prompt,
+                    stable_diffusion_inpaint_negative_prompt,
+                    stable_diffusion_inpiant_num_images_per_prompt,
+                    stable_diffusion_inpaint_guidance_scale,
+                    stable_diffusion_inpaint_num_inference_step,
+                    stable_diffusion_inpaint_seed_generator,
+                ],
+                outputs=[output_image],
+            )

diffusion_webui/diffusion_models/text2img_app.py

@@ -0,0 +1,168 @@
+import gradio as gr
+import torch
+from diffusers import StableDiffusionPipeline
+
+from diffusion_webui.utils.model_list import stable_model_list
+from diffusion_webui.utils.scheduler_list import (
+    SCHEDULER_MAPPING,
+    get_scheduler,
+)
+
+
+class StableDiffusionText2ImageGenerator:
+    def __init__(self):
+        self.pipe = None
+
+    def load_model(
+        self,
+        stable_model_path,
+        scheduler,
+    ):
+        if self.pipe is None or self.pipe.model_name != stable_model_path or self.pipe.scheduler_name != scheduler:
+            self.pipe = StableDiffusionPipeline.from_pretrained(
+                stable_model_path, safety_checker=None, torch_dtype=torch.float16
+            )
+
+        self.pipe = get_scheduler(pipe=self.pipe, scheduler=scheduler)
+        self.pipe.to("cuda")
+        self.pipe.enable_xformers_memory_efficient_attention()
+        self.pipe.model_name = stable_model_path
+        self.pipe.scheduler_name = scheduler
+
+        return self.pipe
+
+    def generate_image(
+        self,
+        stable_model_path: str,
+        prompt: str,
+        negative_prompt: str,
+        num_images_per_prompt: int,
+        scheduler: str,
+        guidance_scale: int,
+        num_inference_step: int,
+        height: int,
+        width: int,
+        seed_generator=0,
+    ):
+        pipe = self.load_model(
+            stable_model_path=stable_model_path,
+            scheduler=scheduler,
+        )
+        if seed_generator == 0:
+            random_seed = torch.randint(0, 1000000, (1,))
+            generator = torch.manual_seed(random_seed)
+        else:
+            generator = torch.manual_seed(seed_generator)
+
+        images = pipe(
+            prompt=prompt,
+            height=height,
+            width=width,
+            negative_prompt=negative_prompt,
+            num_images_per_prompt=num_images_per_prompt,
+            num_inference_steps=num_inference_step,
+            guidance_scale=guidance_scale,
+            generator=generator,
+        ).images
+
+        return images
+
+    def app():
+        with gr.Blocks():
+            with gr.Row():
+                with gr.Column():
+                    text2image_prompt = gr.Textbox(
+                        lines=1,
+                        placeholder="Prompt",
+                        show_label=False,
+                    )
+
+                    text2image_negative_prompt = gr.Textbox(
+                        lines=1,
+                        placeholder="Negative Prompt",
+                        show_label=False,
+                    )
+                    with gr.Row():
+                        with gr.Column():
+                            text2image_model_path = gr.Dropdown(
+                                choices=stable_model_list,
+                                value=stable_model_list[0],
+                                label="Text-Image Model Id",
+                            )
+
+                            text2image_guidance_scale = gr.Slider(
+                                minimum=0.1,
+                                maximum=15,
+                                step=0.1,
+                                value=7.5,
+                                label="Guidance Scale",
+                            )
+
+                            text2image_num_inference_step = gr.Slider(
+                                minimum=1,
+                                maximum=100,
+                                step=1,
+                                value=50,
+                                label="Num Inference Step",
+                            )
+                            text2image_num_images_per_prompt = gr.Slider(
+                                minimum=1,
+                                maximum=30,
+                                step=1,
+                                value=1,
+                                label="Number Of Images",
+                            )
+                        with gr.Row():
+                            with gr.Column():
+                                text2image_scheduler = gr.Dropdown(
+                                    choices=list(SCHEDULER_MAPPING.keys()),
+                                    value=list(SCHEDULER_MAPPING.keys())[0],
+                                    label="Scheduler",
+                            )
+
+                                text2image_height = gr.Slider(
+                                    minimum=128,
+                                    maximum=1280,
+                                    step=32,
+                                    value=512,
+                                    label="Image Height",
+                                )
+
+                                text2image_width = gr.Slider(
+                                    minimum=128,
+                                    maximum=1280,
+                                    step=32,
+                                    value=512,
+                                    label="Image Width",
+                                )
+                                text2image_seed_generator = gr.Slider(
+                                    label="Seed(0 for random)",
+                                    minimum=0,
+                                    maximum=1000000,
+                                    value=0,
+                                )
+                    text2image_predict = gr.Button(value="Generator")
+
+                with gr.Column():
+                    output_image = gr.Gallery(
+                        label="Generated images",
+                        show_label=False,
+                        elem_id="gallery",
+                    ).style(grid=(1, 2), height=200)
+
+            text2image_predict.click(
+                fn=StableDiffusionText2ImageGenerator().generate_image,
+                inputs=[
+                    text2image_model_path,
+                    text2image_prompt,
+                    text2image_negative_prompt,
+                    text2image_num_images_per_prompt,
+                    text2image_scheduler,
+                    text2image_guidance_scale,
+                    text2image_num_inference_step,
+                    text2image_height,
+                    text2image_width,
+                    text2image_seed_generator,
+                ],
+                outputs=output_image,
+            )

diffusion_webui/utils/data_utils.py

@@ -0,0 +1,12 @@
+from PIL import Image
+
+
+def image_grid(imgs, rows, cols):
+    assert len(imgs) == rows * cols
+
+    w, h = imgs[0].size
+    grid = Image.new("RGB", size=(cols * w, rows * h))
+
+    for i, img in enumerate(imgs):
+        grid.paste(img, box=(i % cols * w, i // cols * h))
+    return grid

diffusion_webui/utils/model_list.py

@@ -0,0 +1,26 @@
+stable_model_list = [
+    "runwayml/stable-diffusion-v1-5",
+    "dreamlike-art/dreamlike-diffusion-1.0",
+    "kadirnar/maturemalemix_v0",
+    "kadirnar/DreamShaper_v6",
+    "stabilityai/stable-diffusion-2-inpainting"
+]
+
+stable_inpiant_model_list = [
+    "stabilityai/stable-diffusion-2-inpainting",
+    "runwayml/stable-diffusion-inpainting",
+    "saik0s/realistic_vision_inpainting",
+]
+
+controlnet_model_list = [
+    "lllyasviel/control_v11p_sd15_canny",
+    "lllyasviel/control_v11f1p_sd15_depth",
+    "lllyasviel/control_v11p_sd15_openpose",
+    "lllyasviel/control_v11p_sd15_scribble",
+    "lllyasviel/control_v11p_sd15_mlsd",
+    "lllyasviel/control_v11e_sd15_shuffle",
+    "lllyasviel/control_v11e_sd15_ip2p",
+    "lllyasviel/control_v11p_sd15_lineart",
+    "lllyasviel/control_v11p_sd15s2_lineart_anime",
+    "lllyasviel/control_v11p_sd15_softedge",
+]

diffusion_webui/utils/preprocces_utils.py

@@ -0,0 +1,96 @@
+from controlnet_aux import (
+    CannyDetector,
+    ContentShuffleDetector,
+    HEDdetector,
+    LineartAnimeDetector,
+    LineartDetector,
+    MediapipeFaceDetector,
+    MidasDetector,
+    MLSDdetector,
+    NormalBaeDetector,
+    OpenposeDetector,
+    PidiNetDetector,
+    SamDetector,
+    ZoeDetector,
+)
+
+import numpy as np
+import cv2
+
+def pad64(x):
+    return int(np.ceil(float(x) / 64.0) * 64 - x)
+
+def HWC3(x):
+    assert x.dtype == np.uint8
+    if x.ndim == 2:
+        x = x[:, :, None]
+    assert x.ndim == 3
+    H, W, C = x.shape
+    assert C == 1 or C == 3 or C == 4
+    if C == 3:
+        return x
+    if C == 1:
+        return np.concatenate([x, x, x], axis=2)
+    if C == 4:
+        color = x[:, :, 0:3].astype(np.float32)
+        alpha = x[:, :, 3:4].astype(np.float32) / 255.0
+        y = color * alpha + 255.0 * (1.0 - alpha)
+        y = y.clip(0, 255).astype(np.uint8)
+        return y
+        
+def safer_memory(x):
+    return np.ascontiguousarray(x.copy()).copy()
+
+
+def resize_image_with_pad(input_image, resolution, skip_hwc3=False):
+    if skip_hwc3:
+        img = input_image
+    else:
+        img = HWC3(input_image)
+        
+    H_raw, W_raw, _ = img.shape
+    k = float(resolution) / float(min(H_raw, W_raw))
+    interpolation = cv2.INTER_CUBIC if k > 1 else cv2.INTER_AREA
+    H_target = int(np.round(float(H_raw) * k))
+    W_target = int(np.round(float(W_raw) * k))
+    img = cv2.resize(img, (W_target, H_target), interpolation=interpolation)
+    H_pad, W_pad = pad64(H_target), pad64(W_target)
+    img_padded = np.pad(img, [[0, H_pad], [0, W_pad], [0, 0]], mode='edge')
+
+    def remove_pad(x):
+        return safer_memory(x[:H_target, :W_target])
+
+    return safer_memory(img_padded), remove_pad
+
+
+def scribble_xdog(img, res=512, thr_a=32, **kwargs):
+    img, remove_pad = resize_image_with_pad(img, res)
+    g1 = cv2.GaussianBlur(img.astype(np.float32), (0, 0), 0.5)
+    g2 = cv2.GaussianBlur(img.astype(np.float32), (0, 0), 5.0)
+    dog = (255 - np.min(g2 - g1, axis=2)).clip(0, 255).astype(np.uint8)
+    result = np.zeros_like(img, dtype=np.uint8)
+    result[2 * (255 - dog) > thr_a] = 255
+    return remove_pad(result), True
+
+def none_preprocces(image_path:str):
+    return Image.open(image_path)
+
+PREPROCCES_DICT = {
+    "Hed": HEDdetector.from_pretrained("lllyasviel/Annotators"),
+    "Midas": MidasDetector.from_pretrained("lllyasviel/Annotators"),
+    "MLSD": MLSDdetector.from_pretrained("lllyasviel/Annotators"),
+    "Openpose": OpenposeDetector.from_pretrained("lllyasviel/Annotators"),
+    "PidiNet": PidiNetDetector.from_pretrained("lllyasviel/Annotators"),
+    "NormalBae": NormalBaeDetector.from_pretrained("lllyasviel/Annotators"),
+    "Lineart": LineartDetector.from_pretrained("lllyasviel/Annotators"),
+    "LineartAnime": LineartAnimeDetector.from_pretrained(
+        "lllyasviel/Annotators"
+    ),
+    "Zoe": ZoeDetector.from_pretrained("lllyasviel/Annotators"),
+    "Canny": CannyDetector(),
+    "ContentShuffle": ContentShuffleDetector(),
+    "MediapipeFace": MediapipeFaceDetector(),
+    "ScribbleXDOG": scribble_xdog,
+    "None": none_preprocces
+}
+    

diffusion_webui/utils/scheduler_list.py

@@ -0,0 +1,39 @@
+from diffusers import (
+    DDIMScheduler,
+    DDPMScheduler,
+    DEISMultistepScheduler,
+    DPMSolverMultistepScheduler,
+    DPMSolverSinglestepScheduler,
+    EulerAncestralDiscreteScheduler,
+    EulerDiscreteScheduler,
+    HeunDiscreteScheduler,
+    KDPM2AncestralDiscreteScheduler,
+    KDPM2DiscreteScheduler,
+    PNDMScheduler,
+    UniPCMultistepScheduler,
+)
+
+SCHEDULER_MAPPING = {
+    "DDIM": DDIMScheduler,
+    "DDPMScheduler": DDPMScheduler,
+    "DEISMultistep": DEISMultistepScheduler,
+    "DPMSolverMultistep": DPMSolverMultistepScheduler,
+    "DPMSolverSinglestep": DPMSolverSinglestepScheduler,
+    "EulerAncestralDiscrete": EulerAncestralDiscreteScheduler,
+    "EulerDiscrete": EulerDiscreteScheduler,
+    "HeunDiscrete": HeunDiscreteScheduler,
+    "KDPM2AncestralDiscrete": KDPM2AncestralDiscreteScheduler,
+    "KDPM2Discrete": KDPM2DiscreteScheduler,
+    "PNDMScheduler": PNDMScheduler,
+    "UniPCMultistep": UniPCMultistepScheduler,
+}
+
+
+def get_scheduler(pipe, scheduler):
+    if scheduler in SCHEDULER_MAPPING:
+        SchedulerClass = SCHEDULER_MAPPING[scheduler]
+        pipe.scheduler = SchedulerClass.from_config(pipe.scheduler.config)
+    else:
+        raise ValueError(f"Invalid scheduler name {scheduler}")
+
+    return pipe

requirements.txt

@@ -0,0 +1,9 @@
+transformers 
+bitsandbytes==0.35.0 
+xformers 
+controlnet_aux 
+git+https://github.com/huggingface/diffusers
+imageio
+gradio
+controlnet_aux
+mediapipe