from diffusers import ControlNetModel, StableDiffusionXLControlNetPipeline, AutoencoderKL from diffusers.utils import load_image from PIL import Image import torch import numpy as np import cv2 import gradio as gr controlnet_conditioning_scale = 0.5 # recommended for good generalization controlnet = ControlNetModel.from_pretrained( "diffusers/controlnet-canny-sdxl-1.0", torch_dtype=torch.float16 ) vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16) pipe = StableDiffusionXLControlNetPipeline.from_pretrained( "stabilityai/stable-diffusion-xl-base-1.0", controlnet=controlnet, vae=vae, torch_dtype=torch.float16, ) pipe.enable_model_cpu_offload() low_threshold = 100 high_threshold = 200 def get_canny_filter(image): if not isinstance(image, np.ndarray): image = np.array(image) image = cv2.Canny(image, low_threshold, high_threshold) image = image[:, :, None] image = np.concatenate([image, image, image], axis=2) canny_image = Image.fromarray(image) return canny_image def process(input_image, prompt): canny_image = get_canny_filter(input_image) images = pipe( prompt,image=canny_image, controlnet_conditioning_scale=controlnet_conditioning_scale, ).images return [canny_image,images[0]] block = gr.Blocks().queue() with block: gr.Markdown("## ControlNet SDXL Canny") gr.HTML('''

This is a demo for ControlNet SDXL, which is a neural network structure to control Stable Diffusion XL model by adding extra condition such as canny edge detection.

''') gr.HTML("

You can duplicate this Space to run it privately without a queue and load additional checkpoints. : Duplicate Space

") with gr.Row(): with gr.Column(): input_image = gr.Image(source='upload', type="numpy") prompt = gr.Textbox(label="Prompt") run_button = gr.Button(label="Run") with gr.Column(): result_gallery = gr.Gallery(label='Output', show_label=False, elem_id="gallery").style(grid_cols=2, height='auto') ips = [input_image, prompt] run_button.click(fn=process, inputs=ips, outputs=[result_gallery]) # examples_list = [ # # [ # # "bird.png", # # "bird", # # "Canny Edge Map" # # ], # # [ # # "turtle.png", # # "turtle", # # "Scribble", # # "best quality, extremely detailed", # # 'longbody, lowres, bad anatomy, bad hands, missing fingers, pubic hair,extra digit, fewer digits, cropped, worst quality, low quality', # # 1, # # 512, # # 20, # # 9.0, # # 123490213, # # 0.0, # # 100, # # 200 # # ], # [ # "pose1.png", # "Chef in the Kitchen", # "Pose", # # "best quality, extremely detailed", # # 'longbody, lowres, bad anatomy, bad hands, missing fingers, pubic hair,extra digit, fewer digits, cropped, worst quality, low quality', # # 1, # # 512, # # 20, # # 9.0, # # 123490213, # # 0.0, # # 100, # # 200 # ] # ] # examples = gr.Examples(examples=examples_list,inputs = [input_image, prompt], outputs = [result_gallery], cache_examples = True, fn = process) gr.Markdown("![visitor badge](https://visitor-badge.glitch.me/badge?page_id=RamAnanth1.ControlNet)") block.launch(debug = True)