#!/usr/bin/env python from __future__ import annotations import requests import os import random import gradio as gr import numpy as np import spaces import torch import cv2 import xformers import triton from PIL import Image from io import BytesIO from diffusers.utils import load_image from diffusers import StableDiffusionXLControlNetPipeline, StableDiffusionXLControlNetInpaintPipeline, ControlNetModel, AutoencoderKL, DiffusionPipeline, AutoPipelineForImage2Image, AutoPipelineForInpainting, EulerDiscreteScheduler, DPMSolverMultistepScheduler DESCRIPTION = "# Run any LoRA or SD Model" if not torch.cuda.is_available(): DESCRIPTION += "\n
⚠️ This space is running on the CPU. This demo doesn't work on CPU 😞! Run on a GPU by duplicating this space or test our website for free and unlimited by clicking here, which provides these and more options.
" MAX_SEED = np.iinfo(np.int32).max MAX_IMAGE_SIZE = int(os.getenv("MAX_IMAGE_SIZE", "1824")) USE_TORCH_COMPILE = os.getenv("USE_TORCH_COMPILE") == "1" ENABLE_CPU_OFFLOAD = os.getenv("ENABLE_CPU_OFFLOAD") == "1" ENABLE_USE_LORA = os.getenv("ENABLE_USE_LORA", "1") == "1" ENABLE_USE_LORA2 = os.getenv("ENABLE_USE_LORA2", "1") == "1" ENABLE_USE_VAE = os.getenv("ENABLE_USE_VAE", "1") == "1" ENABLE_USE_IMG2IMG = os.getenv("ENABLE_USE_IMG2IMG", "1") == "1" ENABLE_USE_CONTROLNET = os.getenv("ENABLE_USE_CONTROLNET", "1") == "1" ENABLE_USE_CONTROLNETIMG2IMG = os.getenv("ENABLE_USE_CONTROLNET", "1") == "1" device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") def randomize_seed_fn(seed: int, randomize_seed: bool) -> int: if randomize_seed: seed = random.randint(0, MAX_SEED) return seed @spaces.GPU def generate( prompt: str, negative_prompt: str = "", prompt_2: str = "", negative_prompt_2: str = "", use_negative_prompt: bool = False, use_prompt_2: bool = False, use_negative_prompt_2: bool = False, seed: int = 0, width: int = 1024, height: int = 1024, guidance_scale_base: float = 5.0, num_inference_steps_base: int = 25, controlnet_conditioning_scale: float = 1, control_guidance_start: float = 0, control_guidance_end: float = 1, strength_img2img: float = 0.7, use_vae: bool = False, use_lora: bool = False, use_lora2: bool = False, model = 'stabilityai/stable-diffusion-xl-base-1.0', vaecall = 'madebyollin/sdxl-vae-fp16-fix', lora = '', lora2 = '', controlnet_model = 'diffusers/controlnet-canny-sdxl-1.0', lora_scale: float = 0.7, lora_scale2: float = 0.7, use_img2img: bool = False, use_controlnet: bool = False, use_controlnetimg2img: bool = False, url = '', controlnet_img = '', controlnet_img2img = '', ): if torch.cuda.is_available(): if not use_img2img: scheduler = DPMSolverMultistepScheduler.from_pretrained(model, subfolder="scheduler") pipe = DiffusionPipeline.from_pretrained(model, scheduler=scheduler, torch_dtype=torch.float16) pipe.to(device) if use_vae: vae = AutoencoderKL.from_pretrained(vaecall, torch_dtype=torch.float16) pipe = DiffusionPipeline.from_pretrained(model, vae=vae, torch_dtype=torch.float16) pipe.to(device) if use_img2img: pipe = AutoPipelineForImage2Image.from_pretrained(model, torch_dtype=torch.float16) init_image = load_image(url) if use_vae: vae = AutoencoderKL.from_pretrained(vaecall, torch_dtype=torch.float16) pipe = AutoPipelineForImage2Image.from_pretrained(model, vae=vae, torch_dtype=torch.float16) if use_controlnet: controlnet = ControlNetModel.from_pretrained(controlnet_model, torch_dtype=torch.float16) pipe = StableDiffusionXLControlNetPipeline.from_pretrained(model, controlnet=controlnet, torch_dtype=torch.float16) image = load_image(controlnet_img) image = np.array(image) image = cv2.Canny(image, 100, 200) image = image[:, :, None] image = np.concatenate([image, image, image], axis=2) image = Image.fromarray(image) if use_vae: vae = AutoencoderKL.from_pretrained(vaecall, torch_dtype=torch.float16) pipe = StableDiffusionXLControlNetPipeline.from_pretrained(model, controlnet=controlnet, vae=vae, torch_dtype=torch.float16) if use_controlnetimg2img: controlnet = ControlNetModel.from_pretrained(controlnet_model, torch_dtype=torch.float16) pipe = StableDiffusionXLControlNetInpaintPipeline.from_pretrained(model, controlnet=controlnet, torch_dtype=torch.float16) image_start = load_image(controlnet_img) image = load_image(controlnet_img) image_mask = load_image(controlnet_img2img) image = np.array(image) image = cv2.Canny(image, 100, 200) image = image[:, :, None] image = np.concatenate([image, image, image], axis=2) image = Image.fromarray(image) if use_vae: vae = AutoencoderKL.from_pretrained(vaecall, torch_dtype=torch.float16) pipe = StableDiffusionXLControlNetInpaintPipeline.from_pretrained(model, controlnet=controlnet, vae=vae, torch_dtype=torch.float16) if use_lora: pipe.load_lora_weights(lora) pipe.fuse_lora(lora_scale) if use_lora2: pipe.load_lora_weights(lora, adapter_name="1") pipe.load_lora_weights(lora2, adapter_name="2") pipe.set_adapters(["1", "2"], adapter_weights=[lora_scale, lora_scale2]) generator = torch.Generator().manual_seed(seed) if not use_negative_prompt: negative_prompt = None # type: ignore if not use_prompt_2: prompt_2 = None # type: ignore if not use_negative_prompt_2: negative_prompt_2 = None # type: ignore if use_controlnetimg2img: image = pipe( prompt=prompt, strength=strength_img2img, controlnet_conditioning_scale=controlnet_conditioning_scale, eta=0.0, mask_image=image_mask, image=image_start, control_image=image, negative_prompt=negative_prompt, width=width, height=height, guidance_scale=guidance_scale_base, num_inference_steps=num_inference_steps_base, generator=generator, ).images[0] return image if use_controlnet: image = pipe( prompt=prompt, controlnet_conditioning_scale=controlnet_conditioning_scale, control_guidance_start=control_guidance_start, control_guidance_end=control_guidance_end, image=image, negative_prompt=negative_prompt, prompt_2=prompt_2, width=width, height=height, negative_prompt_2=negative_prompt_2, guidance_scale=guidance_scale_base, num_inference_steps=num_inference_steps_base, generator=generator, ).images[0] return image elif use_img2img: images = pipe( prompt=prompt, image=init_image, strength=strength_img2img, negative_prompt=negative_prompt, prompt_2=prompt_2, negative_prompt_2=negative_prompt_2, width=width, height=height, guidance_scale=guidance_scale_base, num_inference_steps=num_inference_steps_base, generator=generator, output_type="pil", ).images[0] return images else: return pipe( prompt=prompt, negative_prompt=negative_prompt, prompt_2=prompt_2, negative_prompt_2=negative_prompt_2, width=width, height=height, guidance_scale=guidance_scale_base, num_inference_steps=num_inference_steps_base, generator=generator, output_type="pil", ).images[0] with gr.Blocks(theme=gr.themes.Soft(), css="style.css") as demo: gr.HTML( "