import gradio as gr import spaces import torch from diffusers import AutoencoderKL, ControlNetModel, TCDScheduler from gradio_imageslider import ImageSlider from image_gen_aux import LineArtPreprocessor from PIL import Image, ImageEnhance from controlnet_union import ControlNetModel_Union from pipeline_sdxl_recolor import StableDiffusionXLRecolorPipeline lineart_preprocessor = LineArtPreprocessor.from_pretrained("OzzyGT/lineart").to("cuda") controlnet = [ ControlNetModel.from_pretrained( "OzzyGT/ControlNet-recolorXL", torch_dtype=torch.float16, variant="fp16" ), ControlNetModel_Union.from_pretrained( "OzzyGT/controlnet-union-promax-sdxl-1.0", torch_dtype=torch.float16, variant="fp16", ), ] vae = AutoencoderKL.from_pretrained( "madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16 ).to("cuda") pipe = StableDiffusionXLRecolorPipeline.from_pretrained( "recoilme/ColorfulXL-Lightning", torch_dtype=torch.float16, vae=vae, controlnet=controlnet, variant="fp16", ).to("cuda") pipe.scheduler = TCDScheduler.from_config(pipe.scheduler.config) pipe.load_ip_adapter( "h94/IP-Adapter", subfolder="sdxl_models", weight_name="ip-adapter_sdxl_vit-h.safetensors", image_encoder_folder="models/image_encoder", ) scale = { "up": {"block_0": [1.0, 0.0, 1.0]}, } pipe.set_ip_adapter_scale(scale) prompt = "high quality color photo, sharp, detailed, 4k, colorized, remastered" negative_prompt = "blurry, low resolution, bad quality, pixelated, black and white, b&w, grayscale, monochrome, sepia" ( prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds, ) = pipe.encode_prompt(prompt, negative_prompt, "cuda", True) @spaces.GPU(duration=16) def recolor_image(image): source_image = image["background"] lineart_image = lineart_preprocessor(source_image, resolution_scale=0.7)[0] for image in pipe( prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_prompt_embeds, pooled_prompt_embeds=pooled_prompt_embeds, negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, image=[source_image, lineart_image], ip_adapter_image=source_image, num_inference_steps=8, guidance_scale=2.0, controlnet_conditioning_scale=[1.0, 0.5], control_guidance_end=[1.0, 0.9], ): yield source_image, image image = image.convert("RGBA") source_image = source_image.convert("RGBA") enhancer = ImageEnhance.Color(image) image = enhancer.enhance(4.0) alpha = image.split()[3] alpha = alpha.point(lambda p: p * 0.20) image.putalpha(alpha) merged_image = Image.alpha_composite(source_image, image) yield source_image, merged_image def clear_result(): return gr.update(value=None) css = """ .gradio-container { width: 1024px !important; } """ title = """

Diffusers Image Recolor

Upload a grayscale image to colorize it.
This space is a PoC made for the guide Recoloring photos with diffusers.
""" with gr.Blocks(css=css) as demo: gr.HTML(title) run_button = gr.Button("Generate") with gr.Row(): input_image = gr.ImageEditor( type="pil", label="Input Image", crop_size=(1024, 1024), canvas_size=(1024, 1024), layers=False, eraser=False, brush=False, sources=["upload"], image_mode="RGB", ) result = ImageSlider(interactive=False, label="Generated Image", type="pil") run_button.click( fn=clear_result, inputs=None, outputs=result, ).then( fn=recolor_image, inputs=[input_image], outputs=result, ) demo.launch(share=False)