import cv2 import gradio as gr import numpy as np import torch from diffusers import StableDiffusionControlNetPipeline, StableDiffusionLatentUpscalePipeline, ControlNetModel, AutoencoderKL from diffusers import UniPCMultistepScheduler from PIL import Image from lpw import _encode_prompt controlnet_ColorCanny = ControlNetModel.from_pretrained("ghoskno/Color-Canny-Controlnet-model", torch_dtype=torch.float16) vae = AutoencoderKL.from_pretrained("stabilityai/sd-vae-ft-mse", torch_dtype=torch.float16) pipe = StableDiffusionControlNetPipeline.from_pretrained("Lykon/DreamShaper", vae=vae, controlnet=controlnet_ColorCanny, torch_dtype=torch.float16) pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) pipe.enable_model_cpu_offload() pipe.enable_xformers_memory_efficient_attention() pipe.enable_attention_slicing() # Generator seed generator = torch.manual_seed(0) def HWC3(x): assert x.dtype == np.uint8 if x.ndim == 2: x = x[:, :, None] assert x.ndim == 3 H, W, C = x.shape assert C == 1 or C == 3 or C == 4 if C == 3: return x if C == 1: return np.concatenate([x, x, x], axis=2) if C == 4: color = x[:, :, 0:3].astype(np.float32) alpha = x[:, :, 3:4].astype(np.float32) / 255.0 y = color * alpha + 255.0 * (1.0 - alpha) y = y.clip(0, 255).astype(np.uint8) return y def resize_image(input_image, resolution, max_edge=False, edge_limit=False): H, W, C = input_image.shape H = float(H) W = float(W) if max_edge: k = float(resolution) / max(H, W) else: k = float(resolution) / min(H, W) H *= k W *= k H, W = int(H), int(W) img = cv2.resize(input_image, (W, H), interpolation=cv2.INTER_LANCZOS4 if k > 1 else cv2.INTER_AREA) if not edge_limit: return img pH = int(np.round(H / 64.0)) * 64 pW = int(np.round(W / 64.0)) * 64 pimg = np.zeros((pH, pW, 3), dtype=img.dtype) oH, oW = (pH-H)//2, (pW-W)//2 pimg[oH:oH+H, oW:oW+W] = img return pimg def get_canny_filter(image, format='pil', low_threshold=100, high_threshold=200): if not isinstance(image, np.ndarray): image = np.array(image) image = cv2.Canny(image, low_threshold, high_threshold) image = image[:, :, None] image = np.concatenate([image, image, image], axis=2) if format == 'pil': image = Image.fromarray(image) return image def get_color_filter(cond_image, mask_size=64): H, W = cond_image.shape[:2] cond_image = cv2.resize(cond_image, (W // mask_size, H // mask_size), interpolation=cv2.INTER_CUBIC) color = cv2.resize(cond_image, (W, H), interpolation=cv2.INTER_NEAREST) return color def get_colorcanny(image, mask_size): if not isinstance(image, np.ndarray): image = np.array(image) canny_img = get_canny_filter(image, format='np') color_img = get_color_filter(image, int(mask_size)) color_img[np.where(canny_img > 128)] = 255 color_img = Image.fromarray(color_img) return color_img def process(input_image, prompt, n_prompt, strength=1.0, color_mask_size=96, size=512, scale=6.0, ddim_steps=20): prompt_embeds, negative_prompt_embeds = _encode_prompt(pipe, prompt, pipe.device, 1, True, n_prompt, 3) input_image = resize_image(input_image, size, max_edge=True, edge_limit=True) cond_img = get_colorcanny(input_image, color_mask_size) output = pipe( prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_prompt_embeds, image=cond_img, generator=generator, num_images_per_prompt=1, num_inference_steps=ddim_steps, guidance_scale=scale, controlnet_conditioning_scale=float(strength) ) return [output.images[0], cond_img] block = gr.Blocks().queue() with block: gr.Markdown(""" # 🧨 Color-Canny-ControlNet This is an extended model of ControlNet that not only utilizes the Canny edge of images but also incorporates the color features. We trained this model on the cleaned laion-art dataset that contains 2.6 million images with 2 epochs, using the Canny edge and color mosaic of the images as input. The processed dataset and pretrained model can be found in [ghoskno/laion-art-en-colorcanny](https://huggingface.co/datasets/ghoskno/laion-art-en-colorcanny) and [ghoskno/Color-Canny-Controlnet-model](https://huggingface.co/ghoskno/Color-Canny-Controlnet-model). This allows generated images to maintain the same color composition as the original images. If you are looking to control both the contours and colors of the original image while using ControlNet to generate images, then this is the best option for you! You can try out this model or test the examples provided below 🤗. """) with gr.Row(): with gr.Column(): input_image = gr.Image(source='upload', type="numpy") prompt = gr.Textbox(label="Prompt", value='') n_prompt = gr.Textbox(label="Negative Prompt", value='') run_button = gr.Button(label="Run") with gr.Accordion('Advanced', open=False): strength = gr.Slider(label="Control Strength", minimum=0.0, maximum=2.0, value=1.0, step=0.01) color_mask_size = gr.Slider(label="Color Mask Size", minimum=32, maximum=256, value=96, step=16) size = gr.Slider(label="Size", minimum=256, maximum=768, value=512, step=128) scale = gr.Slider(label="Guidance Scale", minimum=0.1, maximum=30.0, value=6.0, step=0.1) ddim_steps = gr.Slider(label="Steps", minimum=1, maximum=50, value=20, step=1) with gr.Column(): result_gallery = gr.Gallery(label='Output', show_label=False, elem_id="gallery").style(grid=2, height='auto') ips = [input_image, prompt, n_prompt, strength, color_mask_size, size, scale, ddim_steps] run_button.click(fn=process, inputs=ips, outputs=[result_gallery]) gr.Examples( examples=[ ["./asserts/1.png", "a concept art of by Makoto Shinkai, a girl is standing in the middle of the sea", "text, bad anatomy, blurry, (low quality, blurry)"], ["./asserts/2.png", "a concept illustration with saturated vivid watercolors by Erin Hanson and Moebius stylized graphic scene", "text, bad anatomy, blurry, (low quality, blurry)"], ["./asserts/3.png", "sky city on the sea, with waves churning and wind power plants on the island", "text, bad anatomy, blurry, (low quality, blurry)"], ], inputs=[ input_image, prompt, n_prompt ], outputs=result_gallery, fn=process, cache_examples=True, ) block.launch(debug = True, server_name='0.0.0.0')