init commit

.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+WIP/

app.py

@@ -0,0 +1,328 @@
+
+# inpaint pipeline with fix to avoid noise added to latents during final iteration of denoising loop
+from inpaint_pipeline import SDInpaintPipeline as StableDiffusionInpaintPipelineLegacy
+
+from diffusers import (
+    StableDiffusionPipeline,
+    StableDiffusionImg2ImgPipeline,
+    StableDiffusionInpaintPipelineLegacy # uncomment this line to use original inpaint pipeline
+)
+
+import diffusers.schedulers
+import gradio as gr
+import torch
+import random
+from multiprocessing import cpu_count
+import json
+
+import importlib
+
+_xformers_available = importlib.util.find_spec("xformers") is not None
+device = "cuda" if torch.cuda.is_available() else "cpu"
+low_vram_mode = False
+
+# scheduler dict includes superclass SchedulerMixin (it still generates reasonable images)
+scheduler_dict = {
+    k: v
+    for k, v in diffusers.schedulers.__dict__.items()
+    if "Scheduler" in k and "Flax" not in k
+}
+scheduler_dict.pop(
+    "VQDiffusionScheduler"
+)  # requires unique parameter, unlike other schedulers
+scheduler_names = list(scheduler_dict.keys())
+default_scheduler = scheduler_names[3]  # expected to be DPM Multistep
+
+model_ids = [
+  "andite/anything-v4.0",
+  "hakurei/waifu-diffusion",
+  "prompthero/openjourney-v2",
+  "runwayml/stable-diffusion-v1-5", 
+  "johnslegers/epic-diffusion",
+  "stabilityai/stable-diffusion-2-1",
+]
+
+loaded_model_id = ""
+
+
+def load_pipe(
+    model_id, scheduler_name, pipe_class=StableDiffusionPipeline, pipe_kwargs="{}"
+):
+    global pipe, loaded_model_id
+
+    scheduler = scheduler_dict[scheduler_name]
+
+    # load new weights from disk only when changing model_id
+    if model_id != loaded_model_id:
+        pipe = pipe_class.from_pretrained(
+            model_id,
+            # torch_dtype=torch.float16,
+            # revision='fp16',
+            safety_checker=None,
+            requires_safety_checker=False,
+            scheduler=scheduler.from_pretrained(model_id, subfolder="scheduler"),
+            **json.loads(pipe_kwargs),
+        )
+        loaded_model_id = model_id
+
+    # if same model_id, instantiate new pipeline with same underlying pytorch objects to avoid reloading weights from disk
+    elif pipe_class != pipe.__class__ or not isinstance(pipe.scheduler, scheduler):
+        pipe.components["scheduler"] = scheduler.from_pretrained(
+            model_id, subfolder="scheduler"
+        )
+        pipe = pipe_class(**pipe.components)
+
+    if device == 'cuda':
+        pipe = pipe.to(device)
+        if _xformers_available:
+            pipe.enable_xformers_memory_efficient_attention()
+            print("using xformers")
+        if low_vram_mode:
+            pipe.enable_attention_slicing()
+            print("using attention slicing to lower VRAM")
+
+    return pipe
+
+
+pipe = None
+pipe = load_pipe(model_ids[0], default_scheduler)
+
+
+def generate(
+    model_name,
+    scheduler_name,
+    prompt,
+    guidance,
+    steps,
+    n_images=1,
+    width=512,
+    height=512,
+    seed=0,
+    image=None,
+    strength=0.5,
+    inpaint_image=None,
+    inpaint_strength=0.5,
+    neg_prompt="",
+    pipe_class=StableDiffusionPipeline,
+    pipe_kwargs="{}",
+):
+
+    if seed == -1:
+        seed = random.randint(0, 2147483647)
+
+    generator = torch.Generator("cuda").manual_seed(seed)
+
+    pipe = load_pipe(
+        model_id=model_name,
+        scheduler_name=scheduler_name,
+        pipe_class=pipe_class,
+        pipe_kwargs=pipe_kwargs,
+    )
+
+    status_message = (
+        f"Prompt: '{prompt}' | Seed: {seed} | Guidance: {guidance} | Scheduler: {scheduler_name} | Steps: {steps}"
+    )
+
+    if pipe_class == StableDiffusionPipeline:
+        status_message = "Text to Image " + status_message
+
+        result = pipe(
+            prompt,
+            negative_prompt=neg_prompt,
+            num_images_per_prompt=n_images,
+            num_inference_steps=int(steps),
+            guidance_scale=guidance,
+            width=width,
+            height=height,
+            generator=generator,
+        )
+
+    elif pipe_class == StableDiffusionImg2ImgPipeline:
+
+        status_message = "Image to Image " + status_message
+        print(image.size)
+        image = image.resize((width, height))
+        print(image.size)
+
+        result = pipe(
+            prompt,
+            negative_prompt=neg_prompt,
+            num_images_per_prompt=n_images,
+            image=image,
+            num_inference_steps=int(steps),
+            strength=strength,
+            guidance_scale=guidance,
+            generator=generator,
+        )
+
+    elif pipe_class == StableDiffusionInpaintPipelineLegacy:
+        status_message = "Inpainting " + status_message
+
+        init_image = inpaint_image["image"].resize((width, height))
+        mask = inpaint_image["mask"].resize((width, height))
+
+        result = pipe(
+            prompt,
+            negative_prompt=neg_prompt,
+            num_images_per_prompt=n_images,
+            image=init_image,
+            mask_image=mask,
+            num_inference_steps=int(steps),
+            strength=inpaint_strength,
+            guidance_scale=guidance,
+            generator=generator,
+        )
+
+    else:
+        None, f"Unhandled pipeline class: {pipe_class}"
+
+    return result.images, status_message
+
+
+default_img_size = 512
+
+with open("header.html") as fp:
+    header = fp.read()
+
+with open("footer.html") as fp:
+    footer = fp.read()
+
+with gr.Blocks(css="style.css") as demo:
+
+    pipe_state = gr.State(lambda: StableDiffusionPipeline)
+
+    gr.HTML(header)
+
+    with gr.Row():
+
+        with gr.Column(scale=70):
+
+            # with gr.Row():
+            prompt = gr.Textbox(
+                label="Prompt", placeholder="<Shift+Enter> to generate", lines=2
+            )
+            neg_prompt = gr.Textbox(label="Negative Prompt", placeholder="", lines=2)
+
+        with gr.Column(scale=30):
+            model_name = gr.Dropdown(
+                label="Model", choices=model_ids, value=loaded_model_id
+            )
+            scheduler_name = gr.Dropdown(
+                label="Scheduler", choices=scheduler_names, value=default_scheduler
+            )
+            generate_button = gr.Button(value="Generate", elem_id="generate-button")
+
+    with gr.Row():
+
+        with gr.Column():
+
+            with gr.Tab("Text to Image") as tab:
+                tab.select(lambda: StableDiffusionPipeline, [], pipe_state)
+
+            with gr.Tab("Image to image") as tab:
+                tab.select(lambda: StableDiffusionImg2ImgPipeline, [], pipe_state)
+
+                image = gr.Image(
+                    label="Image to Image",
+                    source="upload",
+                    tool="editor",
+                    type="pil",
+                    elem_id="image_upload",
+                ).style(height=default_img_size)
+                strength = gr.Slider(
+                    label="Denoising strength",
+                    minimum=0,
+                    maximum=1,
+                    step=0.02,
+                    value=0.8,
+                )
+
+            with gr.Tab("Inpainting") as tab:
+                tab.select(lambda: StableDiffusionInpaintPipelineLegacy, [], pipe_state)
+
+                inpaint_image = gr.Image(
+                    label="Inpainting",
+                    source="upload",
+                    tool="sketch",
+                    type="pil",
+                    elem_id="image_upload",
+                ).style(height=default_img_size)
+                inpaint_strength = gr.Slider(
+                    label="Denoising strength",
+                    minimum=0,
+                    maximum=1,
+                    step=0.02,
+                    value=0.8,
+                )
+
+            with gr.Row():
+                batch_size = gr.Slider(
+                    label="Batch Size", value=1, minimum=1, maximum=8, step=1
+                )
+                seed = gr.Slider(-1, 2147483647, label="Seed", value=-1, step=1)
+
+            with gr.Row():
+                guidance = gr.Slider(
+                    label="Guidance scale", value=7.5, minimum=0, maximum=20
+                )
+                steps = gr.Slider(
+                    label="Steps", value=20, minimum=1, maximum=100, step=1
+                )
+
+            with gr.Row():
+                width = gr.Slider(
+                    label="Width",
+                    value=default_img_size,
+                    minimum=64,
+                    maximum=1024,
+                    step=32,
+                )
+                height = gr.Slider(
+                    label="Height",
+                    value=default_img_size,
+                    minimum=64,
+                    maximum=1024,
+                    step=32,
+                )
+
+        with gr.Column():
+            gallery = gr.Gallery(
+                label="Generated images", show_label=False, elem_id="gallery"
+            ).style(height=default_img_size, grid=2)
+
+            generation_details = gr.Markdown()
+
+            pipe_kwargs = gr.Textbox(label="Pipe kwargs", value="{\n\t\n}")
+
+            # if torch.cuda.is_available():
+            #  giga = 2**30
+            #  vram_guage = gr.Slider(0, torch.cuda.memory_reserved(0)/giga, label='VRAM Allocated to Reserved (GB)', value=0, step=1)
+            #  demo.load(lambda : torch.cuda.memory_allocated(0)/giga, inputs=[], outputs=vram_guage, every=0.5, show_progress=False)
+
+    gr.HTML(footer)
+
+    inputs = [
+        model_name,
+        scheduler_name,
+        prompt,
+        guidance,
+        steps,
+        batch_size,
+        width,
+        height,
+        seed,
+        image,
+        strength,
+        inpaint_image,
+        inpaint_strength,
+        neg_prompt,
+        pipe_state,
+        pipe_kwargs,
+    ]
+    outputs = [gallery, generation_details]
+
+    prompt.submit(generate, inputs=inputs, outputs=outputs)
+    generate_button.click(generate, inputs=inputs, outputs=outputs)
+
+demo.queue(concurrency_count=cpu_count())
+demo.launch()

footer.html

@@ -0,0 +1,15 @@
+
+<!-- based on https://huggingface.co/spaces/stabilityai/stable-diffusion/blob/main/app.py -->
+
+
+<div class="footer">
+    <p>Model Architecture by <a href="https://huggingface.co/stabilityai" style="text-decoration: underline;" target="_blank">StabilityAI</a> - Pipelines by 🤗 Hugging Face
+    </p>
+</div>
+<div class="acknowledgments">
+    <p><h4>LICENSE</h4>
+The model is licensed with a <a href="https://huggingface.co/stabilityai/stable-diffusion-2/blob/main/LICENSE-MODEL" style="text-decoration: underline;" target="_blank">CreativeML OpenRAIL++</a> license. The authors claim no rights on the outputs you generate, you are free to use them and are accountable for their use which must not go against the provisions set in this license. The license forbids you from sharing any content that violates any laws, produce any harm to a person, disseminate any personal information that would be meant for harm, spread misinformation and target vulnerable groups. For the full list of restrictions please <a href="https://huggingface.co/spaces/CompVis/stable-diffusion-license" target="_blank" style="text-decoration: underline;" target="_blank">read the license</a></p>
+    <p><h4>Biases and content acknowledgment</h4>
+Despite how impressive being able to turn text into image is, beware to the fact that this model may output content that reinforces or exacerbates societal biases, as well as realistic faces, pornography and violence. The model was trained on the <a href="https://laion.ai/blog/laion-5b/" style="text-decoration: underline;" target="_blank">LAION-5B dataset</a>, which scraped non-curated image-text-pairs from the internet (the exception being the removal of illegal content) and is meant for research purposes. You can read more in the <a href="https://huggingface.co/CompVis/stable-diffusion-v1-4" style="text-decoration: underline;" target="_blank">model card</a></p>
+</div>
+

header.html

@@ -0,0 +1,25 @@
+
+<!-- based on https://huggingface.co/spaces/stabilityai/stable-diffusion/blob/main/app.py -->
+
+<div style="text-align: center; margin: 0 auto;">
+    <div
+      style="
+        display: inline-flex;
+        align-items: center;
+        gap: 0.8rem;
+        font-size: 1.75rem;
+      "
+    >
+    <svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 32 32" style="enable-background:new 0 0 512 512;" xml:space="preserve" width="32" height="32"><path style="fill:#FCD577;" d="M29.545 29.791V2.21c-1.22 0 -2.21 -0.99 -2.21 -2.21H4.665c0 1.22 -0.99 2.21 -2.21 2.21v27.581c1.22 0 2.21 0.99 2.21 2.21H27.335C27.335 30.779 28.325 29.791 29.545 29.791z"/><path x="98.205" y="58.928" style="fill:#99B6C6;" width="315.577" height="394.144" d="M6.138 3.683H25.861V28.317H6.138V3.683z"/><path x="98.205" y="58.928" style="fill:#7BD4EF;" width="315.577" height="131.317" d="M6.138 3.683H25.861V11.89H6.138V3.683z"/><g><path style="fill:#7190A5;" d="M14.498 10.274c0 1.446 0.983 1.155 1.953 1.502l0.504 5.317c0 0 -5.599 0.989 -6.026 2.007l0.27 -2.526c0.924 -1.462 1.286 -4.864 1.419 -6.809l0.086 0.006C12.697 9.876 14.498 10.166 14.498 10.274z"/><path style="fill:#7190A5;" d="M21.96 17.647c0 0 -0.707 1.458 -1.716 1.903c0 0 -1.502 -0.827 -1.502 -0.827c-2.276 -1.557 -2.366 -8.3 -2.366 -8.3c0 -1.718 -0.185 -1.615 -1.429 -1.615c-1.167 0 -2.127 -0.606 -2.242 0.963l-0.086 -0.006c0.059 -0.859 0.074 -1.433 0.074 -1.433c0 -1.718 1.449 -3.11 3.237 -3.11s3.237 1.392 3.237 3.11C19.168 8.332 19.334 15.617 21.96 17.647z"/></g><path style="fill:#6C8793;" d="M12.248 24.739c1.538 0.711 3.256 1.591 3.922 2.258c-1.374 0.354 -2.704 0.798 -3.513 1.32h-2.156c-1.096 -0.606 -2.011 -1.472 -2.501 -2.702c-1.953 -4.907 2.905 -8.664 2.905 -8.664c0.001 -0.001 0.002 -0.002 0.003 -0.003c0.213 -0.214 0.523 -0.301 0.811 -0.21l0.02 0.006c-0.142 0.337 -0.03 0.71 0.517 1.108c1.264 0.919 3.091 1.131 4.416 1.143c-1.755 1.338 -3.42 3.333 -4.367 5.618L12.248 24.739z"/><path style="fill:#577484;" d="M16.17 26.997c-0.666 -0.666 -2.385 -1.548 -3.922 -2.258l0.059 -0.126c0.947 -2.284 2.612 -4.28 4.367 -5.618c0.001 0 0.001 0 0.001 0c0.688 -0.525 1.391 -0.948 2.068 -1.247c0.001 0 0.001 0 0.001 0c1.009 -0.446 1.964 -0.617 2.742 -0.44c0.61 0.138 1.109 0.492 1.439 1.095c1.752 3.205 0.601 9.913 0.601 9.913H12.657C13.466 27.796 14.796 27.352 16.17 26.997z"/><path style="fill:#F7DEB0;" d="M14.38 13.1c-0.971 -0.347 -1.687 -1.564 -1.687 -3.01c0 -0.107 0.004 -0.213 0.011 -0.318c0.116 -1.569 1.075 -2.792 2.242 -2.792c1.244 0 2.253 1.392 2.253 3.11c0 0 -0.735 6.103 1.542 7.66c-0.677 0.299 -1.38 0.722 -2.068 1.247c0 0 0 0 -0.001 0c-1.326 -0.012 -3.152 -0.223 -4.416 -1.143c-0.547 -0.398 -0.659 -0.771 -0.517 -1.108c0.426 -1.018 3.171 -1.697 3.171 -1.697L14.38 13.1z"/><path style="fill:#E5CA9E;" d="M14.38 13.1c0 0 1.019 0.216 1.544 -0.309c0 0 -0.401 1.04 -1.346 1.04"/><g><path style="fill:#EAC36E;" points="437.361,0 413.79,58.926 472.717,35.356 	" d="M27.335 0L25.862 3.683L29.545 2.21"/><path style="fill:#EAC36E;" points="437.361,512 413.79,453.074 472.717,476.644 	" d="M27.335 32L25.862 28.317L29.545 29.791"/><path style="fill:#EAC36E;" points="74.639,512 98.21,453.074 39.283,476.644 	" d="M4.665 32L6.138 28.317L2.455 29.791"/><path style="fill:#EAC36E;" points="39.283,35.356 98.21,58.926 74.639,0 	" d="M2.455 2.21L6.138 3.683L4.665 0"/><path style="fill:#EAC36E;" d="M26.425 28.881H5.574V3.119h20.851v25.761H26.425zM6.702 27.754h18.597V4.246H6.702V27.754z"/></g><g><path style="fill:#486572;" d="M12.758 21.613c-0.659 0.767 -1.245 1.613 -1.722 2.531l0.486 0.202C11.82 23.401 12.241 22.483 12.758 21.613z"/><path style="fill:#486572;" d="M21.541 25.576l-0.37 0.068c-0.553 0.101 -1.097 0.212 -1.641 0.331l-0.071 -0.201l-0.059 -0.167c-0.019 -0.056 -0.035 -0.112 -0.052 -0.169l-0.104 -0.338l-0.088 -0.342c-0.112 -0.457 -0.197 -0.922 -0.235 -1.393c-0.035 -0.47 -0.032 -0.947 0.042 -1.417c0.072 -0.47 0.205 -0.935 0.422 -1.369c-0.272 0.402 -0.469 0.856 -0.606 1.329c-0.138 0.473 -0.207 0.967 -0.234 1.462c-0.024 0.496 0.002 0.993 0.057 1.487l0.046 0.37l0.063 0.367c0.011 0.061 0.02 0.123 0.033 0.184l0.039 0.182l0.037 0.174c-0.677 0.157 -1.351 0.327 -2.019 0.514c-0.131 0.037 -0.262 0.075 -0.392 0.114l0.004 -0.004c-0.117 -0.095 -0.232 -0.197 -0.35 -0.275c-0.059 -0.041 -0.117 -0.084 -0.177 -0.122l-0.179 -0.112c-0.239 -0.147 -0.482 -0.279 -0.727 -0.406c-0.489 -0.252 -0.985 -0.479 -1.484 -0.697c-0.998 -0.433 -2.01 -0.825 -3.026 -1.196c0.973 0.475 1.937 0.969 2.876 1.499c0.469 0.266 0.932 0.539 1.379 0.832c0.223 0.146 0.442 0.297 0.648 0.456l0.154 0.119c0.05 0.041 0.097 0.083 0.145 0.124c0.002 0.002 0.004 0.003 0.005 0.005c-0.339 0.109 -0.675 0.224 -1.009 0.349c-0.349 0.132 -0.696 0.273 -1.034 0.431c-0.338 0.159 -0.668 0.337 -0.973 0.549c0.322 -0.186 0.662 -0.334 1.01 -0.463c0.347 -0.129 0.701 -0.239 1.056 -0.34c0.394 -0.111 0.79 -0.208 1.19 -0.297c0.006 0.006 0.013 0.013 0.019 0.019l0.03 -0.03c0.306 -0.068 0.614 -0.132 0.922 -0.192c0.727 -0.14 1.457 -0.258 2.189 -0.362c0.731 -0.103 1.469 -0.195 2.197 -0.265l0.374 -0.036L21.541 25.576z"/></g></svg>
+
+      <h1 style="font-weight: 1000; margin-bottom: 8px;margin-top:8px">
+        Stable Diffusion Pipeline UI
+      </h1>
+    </div>
+    <p style="margin-bottom: 4px; font-size: 100%; line-height: 24px;">
+      Stable Diffusion WebUI with first class support for HuggingFace Diffusers Pipelines and Diffusion Schedulers, made in the style of <a style="text-decoration: underline;" href="https://github.com/AUTOMATIC1111/stable-diffusion-webui">Automatic1111's WebUI</a> and <a style="text-decoration: underline;" href="https://huggingface.co/spaces/Evel/Evel_Space">Evel_Space</a>.
+    </p> 
+    <p> Supports Text-to-Image, Image to Image, and Inpainting modes, with fast switching between pipeline modes by reusing loaded model weights already in memory.
+    </p>
+  </div>
+

inpaint_pipeline.py

@@ -0,0 +1,247 @@
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+from typing import Optional, Union, List, Callable
+import PIL
+import numpy as np
+
+from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_inpaint_legacy import preprocess_image, deprecate, StableDiffusionInpaintPipelineLegacy, StableDiffusionPipelineOutput, PIL_INTERPOLATION
+
+def preprocess_mask(mask, scale_factor=8):
+    mask = mask.convert("L")
+    w, h = mask.size
+    w, h = map(lambda x: x - x % 32, (w, h))  # resize to integer multiple of 32
+
+    #input_mask = mask.resize((w, h), resample=PIL_INTERPOLATION["nearest"])
+    input_mask = np.array(mask).astype(np.float32) / 255.0
+    input_mask = np.tile(input_mask, (3, 1, 1))
+    input_mask = input_mask[None].transpose(0, 1, 2, 3)  # add batch dimension
+    input_mask = 1 - input_mask  # repaint white, keep black
+    input_mask = torch.round(torch.from_numpy(input_mask))
+
+    mask = mask.resize((w // scale_factor, h // scale_factor), resample=PIL_INTERPOLATION["nearest"])
+    mask = np.array(mask).astype(np.float32) / 255.0
+    mask = np.tile(mask, (4, 1, 1))
+    mask = mask[None].transpose(0, 1, 2, 3)  # add batch dimension
+    mask = 1 - mask  # repaint white, keep black
+    mask = torch.round(torch.from_numpy(mask))
+
+    return mask, input_mask
+
+
+
+class SDInpaintPipeline(StableDiffusionInpaintPipelineLegacy):
+
+    # forward call is same as StableDiffusionInpaintPipelineLegacy, but with line added to avoid noise added to final latents right before decoding step
+    @torch.no_grad()
+    def __call__(
+        self,
+        prompt: Union[str, List[str]],
+        image: Union[torch.FloatTensor, PIL.Image.Image] = None,
+        mask_image: Union[torch.FloatTensor, PIL.Image.Image] = None,
+        strength: float = 0.8,
+        num_inference_steps: Optional[int] = 50,
+        guidance_scale: Optional[float] = 7.5,
+        negative_prompt: Optional[Union[str, List[str]]] = None,
+        num_images_per_prompt: Optional[int] = 1,
+        add_predicted_noise: Optional[bool] = False,
+        eta: Optional[float] = 0.0,
+        generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
+        output_type: Optional[str] = "pil",
+        return_dict: bool = True,
+        callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
+        callback_steps: Optional[int] = 1,
+        preserve_unmasked_image: bool = True,
+        **kwargs,
+    ):
+        r"""
+        Function invoked when calling the pipeline for generation.
+
+        Args:
+            prompt (`str` or `List[str]`):
+                The prompt or prompts to guide the image generation.
+            image (`torch.FloatTensor` or `PIL.Image.Image`):
+                `Image`, or tensor representing an image batch, that will be used as the starting point for the
+                process. This is the image whose masked region will be inpainted.
+            mask_image (`torch.FloatTensor` or `PIL.Image.Image`):
+                `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be
+                replaced by noise and therefore repainted, while black pixels will be preserved. If `mask_image` is a
+                PIL image, it will be converted to a single channel (luminance) before use. If it's a tensor, it should
+                contain one color channel (L) instead of 3, so the expected shape would be `(B, H, W, 1)`.
+            strength (`float`, *optional*, defaults to 0.8):
+                Conceptually, indicates how much to inpaint the masked area. Must be between 0 and 1. When `strength`
+                is 1, the denoising process will be run on the masked area for the full number of iterations specified
+                in `num_inference_steps`. `image` will be used as a reference for the masked area, adding more noise to
+                that region the larger the `strength`. If `strength` is 0, no inpainting will occur.
+            num_inference_steps (`int`, *optional*, defaults to 50):
+                The reference number of denoising steps. More denoising steps usually lead to a higher quality image at
+                the expense of slower inference. This parameter will be modulated by `strength`, as explained above.
+            guidance_scale (`float`, *optional*, defaults to 7.5):
+                Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
+                `guidance_scale` is defined as `w` of equation 2. of [Imagen
+                Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
+                1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
+                usually at the expense of lower image quality.
+            negative_prompt (`str` or `List[str]`, *optional*):
+                The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored
+                if `guidance_scale` is less than `1`).
+            num_images_per_prompt (`int`, *optional*, defaults to 1):
+                The number of images to generate per prompt.
+            add_predicted_noise (`bool`, *optional*, defaults to True):
+                Use predicted noise instead of random noise when constructing noisy versions of the original image in
+                the reverse diffusion process
+            eta (`float`, *optional*, defaults to 0.0):
+                Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
+                [`schedulers.DDIMScheduler`], will be ignored for others.
+            generator (`torch.Generator`, *optional*):
+                One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
+                to make generation deterministic.
+            output_type (`str`, *optional*, defaults to `"pil"`):
+                The output format of the generate image. Choose between
+                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
+                plain tuple.
+            callback (`Callable`, *optional*):
+                A function that will be called every `callback_steps` steps during inference. The function will be
+                called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
+            callback_steps (`int`, *optional*, defaults to 1):
+                The frequency at which the `callback` function will be called. If not specified, the callback will be
+                called at every step.
+            preserve_unmasked_image (`bool`, *optional*, defaults to `True`):
+                Whether or not to preserve the unmasked portions of the original image in the inpainted output. If False, 
+                inpainting of the masked latents may produce noticeable distortion of unmasked portions of the decoded 
+                image.
+
+        Returns:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:
+            [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.
+            When returning a tuple, the first element is a list with the generated images, and the second element is a
+            list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"
+            (nsfw) content, according to the `safety_checker`.
+        """
+        message = "Please use `image` instead of `init_image`."
+        init_image = deprecate("init_image", "0.13.0", message, take_from=kwargs)
+        image = init_image or image
+
+        # 1. Check inputs
+        self.check_inputs(prompt, strength, callback_steps)
+
+        # 2. Define call parameters
+        batch_size = 1 if isinstance(prompt, str) else len(prompt)
+        device = self._execution_device
+        # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
+        # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
+        # corresponds to doing no classifier free guidance.
+        do_classifier_free_guidance = guidance_scale > 1.0
+
+        # 3. Encode input prompt
+        text_embeddings = self._encode_prompt(
+            prompt, device, num_images_per_prompt, do_classifier_free_guidance, negative_prompt
+        )
+
+        # 4. Preprocess image and mask
+        if not isinstance(image, torch.FloatTensor):
+            image = preprocess_image(image)
+
+        # get mask corresponding to input latents as well as image
+        if not isinstance(mask_image, torch.FloatTensor):
+            mask_image, input_mask_image = preprocess_mask(mask_image, self.vae_scale_factor)
+
+        # 5. set timesteps
+        self.scheduler.set_timesteps(num_inference_steps, device=device)
+        timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)
+        latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)
+
+        # 6. Prepare latent variables
+        # encode the init image into latents and scale the latents
+        latents, init_latents_orig, noise = self.prepare_latents(
+            image, latent_timestep, batch_size, num_images_per_prompt, text_embeddings.dtype, device, generator
+        )
+
+        # 7. Prepare mask latent
+        mask = mask_image.to(device=self.device, dtype=latents.dtype)
+        mask = torch.cat([mask] * batch_size * num_images_per_prompt)
+
+        # 8. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
+        extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
+
+        # 9. Denoising loop
+        num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order
+        with self.progress_bar(total=num_inference_steps) as progress_bar:
+            for i, t in enumerate(timesteps):
+                
+                # expand the latents if we are doing classifier free guidance
+                latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents
+                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
+
+                # predict the noise residual
+                noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings).sample
+
+                # perform guidance
+                if do_classifier_free_guidance:
+                    noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
+                    noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
+
+                # compute the previous noisy sample x_t -> x_t-1
+                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample
+                # masking
+                if add_predicted_noise:
+                    init_latents_proper = self.scheduler.add_noise(
+                        init_latents_orig, noise_pred_uncond, torch.tensor([t])
+                    )
+                else:
+                    init_latents_proper = self.scheduler.add_noise(init_latents_orig, noise, torch.tensor([t]))
+
+                latents = (init_latents_proper * mask) + (latents * (1 - mask))
+
+                # call the callback, if provided
+                if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
+                    progress_bar.update()
+                    if callback is not None and i % callback_steps == 0:
+                        callback(i, t, latents)
+
+        # use original latents corresponding to unmasked portions of the image
+        # necessary step because noise is still added to "init_latents_proper" after final denoising step
+        latents = (init_latents_orig * mask) + (latents * (1 - mask))
+
+        # 10. Post-processing
+        if preserve_unmasked_image:
+            # decode latents
+            latents = 1 / 0.18215 * latents
+            inpaint_image = self.vae.decode(latents).sample
+
+            # restore unmasked parts of image with original image
+            input_mask_image = input_mask_image.to(inpaint_image)
+            image = image.to(inpaint_image)
+            image = (image * input_mask_image) + (inpaint_image * (1 - input_mask_image)) # use original unmasked portions of image to avoid degradation
+
+            # post-processing of image
+            image = (image / 2 + 0.5).clamp(0, 1)
+            # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16
+            image = image.cpu().permute(0, 2, 3, 1).float().numpy()
+        else:
+            image = self.decode_latents(latents)
+
+        # 11. Run safety checker
+        image, has_nsfw_concept = self.run_safety_checker(image, device, text_embeddings.dtype)
+
+        # 12. Convert to PIL
+        if output_type == "pil":
+            image = self.numpy_to_pil(image)
+
+        if not return_dict:
+            return (image, has_nsfw_concept)
+
+        return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)

style.css

@@ -0,0 +1,38 @@
+
+#image_upload{min-height: 512px}
+#image_upload [data-testid="image"], #image_upload [data-testid="image"] > div{min-height: 512px}
+#image_upload .touch-none{display: flex}
+
+
+#generate-button {
+    color:white;
+    border-color: orangered;
+    background: orange;
+    height: 45px;
+}
+
+.footer {
+    margin-bottom: 45px;
+    margin-top: 35px;
+    text-align: center;
+    border-bottom: 1px solid #e5e5e5;
+}
+.footer>p {
+    font-size: .8rem;
+    display: inline-block;
+    padding: 0 10px;
+    transform: translateY(10px);
+    background: white;
+}
+.dark .footer {
+    border-color: #303030;
+}
+.dark .footer>p {
+    background: #0b0f19;
+}
+.acknowledgments h4{
+    margin: 1.25em 0 .25em 0;
+    font-weight: bold;
+    font-size: 115%;
+}
+