app.py

from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler
from diffusers.utils import load_image, make_image_grid
from PIL import Image
import numpy as np
import gradio as gr
import torch
import cv2
import random

def launch(image_input, prompt, negative_prompt, sd_models_dropdown):
    
    return predict(image_input, prompt, negative_prompt, sd_models_dropdown)

def predict(image_input, prompt, negative_prompt, sd_models_dropdown):
    
    image = np.array(image_input)

    # Get canny image
    t_lower = 100
    t_upper = 200
    image = cv2.Canny(image, t_lower, t_upper)
    # image = image[:, :, None]
    # image = np.concatenate([image, image, image], axis=2)
    canny_image = Image.fromarray(image)
    
    selected_model = f"""{sd_models_dropdown}"""

    # Load ControlNet model.
    controlnet = ControlNetModel.from_pretrained(
        "lllyasviel/sd-controlnet-canny",
        torch_dtype=torch.float16
    )

    # Load Stable Diffusion model.
    pipe = StableDiffusionControlNetPipeline.from_pretrained(
        selected_model,
        controlnet=controlnet,
        torch_dtype=torch.float16
    )
    
    pipe = pipe.to("cuda")
    
    # speed up diffusion process with faster scheduler and memory optimization
    pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)

    # prompt = "a photo of an astronaut riding a horse on mars"
    # negative_prompt=""
    
    generator = torch.manual_seed(random.randint(0, 1000))
    
    image = pipe(
        prompt=prompt,
        negative_prompt=negative_prompt,
        num_inference_steps=65,
        generator=generator,
        image=canny_image,
        strength=0.15,
        guidance_scale=15.5
        
    ).images[0]  

    return image

# SYSTEM INPUTS

image_input = gr.Image(label="Upload iamge candidate", type="pil")
prompt = gr.Textbox(type="text", label="Prompt")
negative_prompt = gr.Textbox(type="text", label="Negative prompt")

sd_models_dropdown = gr.Dropdown(
    [
        "runwayml/stable-diffusion-v1-5",
    ], 
    label="Stable Diffusion Models")

# SYSTEM OUTPUTS
output_image = gr.Image()

demo = gr.Interface(
    launch,
    inputs=[image_input, prompt, negative_prompt, sd_models_dropdown],
    outputs=output_image,
    title="Stable Diffusion with ControlNet",
)

if __name__ == "__main__":
    demo.launch()