import gc import os import traceback from pathlib import Path from typing import Dict, List, Literal, Optional, Union import numpy as np import torch from diffusers import AutoencoderTiny from PIL import Image from live2diff import StreamAnimateDiffusionDepth from live2diff.image_utils import postprocess_image from live2diff.pipeline_stream_animation_depth import WARMUP_FRAMES class StreamAnimateDiffusionDepthWrapper: def __init__( self, config_path: str, few_step_model_type: str, num_inference_steps: int, t_index_list: Optional[List[int]] = None, strength: Optional[float] = None, dreambooth_path: Optional[str] = None, lora_dict: Optional[Dict[str, float]] = None, output_type: Literal["pil", "pt", "np", "latent"] = "pil", vae_id: Optional[str] = None, device: Literal["cpu", "cuda"] = "cuda", dtype: torch.dtype = torch.float16, frame_buffer_size: int = 1, width: int = 512, height: int = 512, acceleration: Literal["none", "xformers", "tensorrt"] = "tensorrt", do_add_noise: bool = True, device_ids: Optional[List[int]] = None, use_tiny_vae: bool = True, enable_similar_image_filter: bool = False, similar_image_filter_threshold: float = 0.98, similar_image_filter_max_skip_frame: int = 10, use_denoising_batch: bool = True, cfg_type: Literal["none", "full", "self", "initialize"] = "self", seed: int = 42, engine_dir: Optional[Union[str, Path]] = "engines", opt_unet: bool = False, ): """ Initializes the StreamAnimateDiffusionWrapper. Parameters ---------- config_path : str The model id or path to load. few_step_model_type : str The few step model type to use. num_inference_steps : int The number of inference steps to perform. If `t_index_list` is passed, `num_infernce_steps` will parsed as the number of denoising steps before apply few-step lora. Otherwise, `num_inference_steps` will be parsed as the number of steps after applying few-step lora. t_index_list : List[int] The t_index_list to use for inference. strength : Optional[float] The strength to use for inference. dreambooth_path : Optional[str] The dreambooth path to use for inference. If not passed, will use dreambooth from config. lora_dict : Optional[Dict[str, float]], optional The lora_dict to load, by default None. Keys are the LoRA names and values are the LoRA scales. Example: {'LoRA_1' : 0.5 , 'LoRA_2' : 0.7 ,...} output_type : Literal["pil", "pt", "np", "latent"], optional The output type of image, by default "pil". vae_id : Optional[str], optional The vae_id to load, by default None. If None, the default TinyVAE ("madebyollin/taesd") will be used. device : Literal["cpu", "cuda"], optional The device to use for inference, by default "cuda". dtype : torch.dtype, optional The dtype for inference, by default torch.float16. frame_buffer_size : int, optional The frame buffer size for denoising batch, by default 1. width : int, optional The width of the image, by default 512. height : int, optional The height of the image, by default 512. acceleration : Literal["none", "xformers", "tensorrt"], optional The acceleration method, by default "tensorrt". do_add_noise : bool, optional Whether to add noise for following denoising steps or not, by default True. device_ids : Optional[List[int]], optional The device ids to use for DataParallel, by default None. use_lcm_lora : bool, optional Whether to use LCM-LoRA or not, by default True. use_tiny_vae : bool, optional Whether to use TinyVAE or not, by default True. enable_similar_image_filter : bool, optional Whether to enable similar image filter or not, by default False. similar_image_filter_threshold : float, optional The threshold for similar image filter, by default 0.98. similar_image_filter_max_skip_frame : int, optional The max skip frame for similar image filter, by default 10. use_denoising_batch : bool, optional Whether to use denoising batch or not, by default True. cfg_type : Literal["none", "full", "self", "initialize"], optional The cfg_type for img2img mode, by default "self". You cannot use anything other than "none" for txt2img mode. seed : int, optional The seed, by default 42. engine_dir : Optional[Union[str, Path]], optional The directory to save TensorRT engines, by default "engines". opt_unet : bool, optional Whether to optimize UNet or not, by default False. """ self.sd_turbo = False self.device = device self.dtype = dtype self.width = width self.height = height self.output_type = output_type self.frame_buffer_size = frame_buffer_size self.use_denoising_batch = use_denoising_batch self.stream: StreamAnimateDiffusionDepth = self._load_model( config_path=config_path, lora_dict=lora_dict, dreambooth_path=dreambooth_path, few_step_model_type=few_step_model_type, vae_id=vae_id, num_inference_steps=num_inference_steps, t_index_list=t_index_list, strength=strength, height=height, width=width, acceleration=acceleration, do_add_noise=do_add_noise, use_tiny_vae=use_tiny_vae, cfg_type=cfg_type, seed=seed, engine_dir=engine_dir, opt_unet=opt_unet, ) self.batch_size = len(self.stream.t_list) * frame_buffer_size if use_denoising_batch else frame_buffer_size if device_ids is not None: self.stream.unet = torch.nn.DataParallel(self.stream.unet, device_ids=device_ids) if enable_similar_image_filter: self.stream.enable_similar_image_filter( similar_image_filter_threshold, similar_image_filter_max_skip_frame ) def prepare( self, warmup_frames: torch.Tensor, prompt: str, negative_prompt: str = "", guidance_scale: float = 1.2, delta: float = 1.0, ) -> torch.Tensor: """ Prepares the model for inference. Parameters ---------- prompt : str The prompt to generate images from. num_inference_steps : int, optional The number of inference steps to perform, by default 50. guidance_scale : float, optional The guidance scale to use, by default 1.2. delta : float, optional The delta multiplier of virtual residual noise, by default 1.0. Returns ---------- warmup_frames : torch.Tensor generated warmup-frames. """ warmup_frames = self.stream.prepare( warmup_frames=warmup_frames, prompt=prompt, negative_prompt=negative_prompt, guidance_scale=guidance_scale, delta=delta, ) warmup_frames = warmup_frames.permute(0, 2, 3, 1) warmup_frames = (warmup_frames.clip(-1, 1) + 1) / 2 return warmup_frames def __call__( self, image: Optional[Union[str, Image.Image, torch.Tensor]] = None, prompt: Optional[str] = None, ) -> Union[Image.Image, List[Image.Image]]: """ Performs img2img or txt2img based on the mode. Parameters ---------- image : Optional[Union[str, Image.Image, torch.Tensor]] The image to generate from. prompt : Optional[str] The prompt to generate images from. Returns ------- Union[Image.Image, List[Image.Image]] The generated image. """ return self.img2img(image, prompt) def img2img( self, image: Union[str, Image.Image, torch.Tensor], prompt: Optional[str] = None ) -> Union[Image.Image, List[Image.Image], torch.Tensor, np.ndarray]: """ Performs img2img. Parameters ---------- image : Union[str, Image.Image, torch.Tensor] The image to generate from. Returns ------- Image.Image The generated image. """ if prompt is not None: self.stream.update_prompt(prompt) if isinstance(image, str) or isinstance(image, Image.Image): image = self.preprocess_image(image) image_tensor = self.stream(image) image = self.postprocess_image(image_tensor, output_type=self.output_type) return image def preprocess_image(self, image: Union[str, Image.Image]) -> torch.Tensor: """ Preprocesses the image. Parameters ---------- image : Union[str, Image.Image, torch.Tensor] The image to preprocess. Returns ------- torch.Tensor The preprocessed image. """ if isinstance(image, str): image = Image.open(image).convert("RGB").resize((self.width, self.height)) if isinstance(image, Image.Image): image = image.convert("RGB").resize((self.width, self.height)) return self.stream.image_processor.preprocess(image, self.height, self.width).to( device=self.device, dtype=self.dtype ) def postprocess_image( self, image_tensor: torch.Tensor, output_type: str = "pil" ) -> Union[Image.Image, List[Image.Image], torch.Tensor, np.ndarray]: """ Postprocesses the image. Parameters ---------- image_tensor : torch.Tensor The image tensor to postprocess. Returns ------- Union[Image.Image, List[Image.Image]] The postprocessed image. """ if self.frame_buffer_size > 1: output = postprocess_image(image_tensor, output_type=output_type) else: output = postprocess_image(image_tensor, output_type=output_type)[0] if output_type not in ["pil", "np"]: return output.cpu() else: return output @staticmethod def get_model_prefix( config_path: str, few_step_model_type: str, use_tiny_vae: bool, num_denoising_steps: int, height: int, width: int, dreambooth: Optional[str] = None, lora_dict: Optional[dict] = None, ) -> str: from omegaconf import OmegaConf config = OmegaConf.load(config_path) third_party = config.third_party_dict dreambooth_path = dreambooth or third_party.dreambooth if dreambooth_path is None: dreambooth_name = "sd15" else: dreambooth_name = Path(dreambooth_path).stem base_lora_list = third_party.get("lora_list", []) lora_dict = lora_dict or {} for lora_alpha in base_lora_list: lora_name = lora_alpha["lora"] alpha = lora_alpha["lora_alpha"] if lora_name not in lora_dict: lora_dict[lora_name] = alpha prefix = f"{dreambooth_name}--{few_step_model_type}--step{num_denoising_steps}--" for k, v in lora_dict.items(): prefix += f"{Path(k).stem}-{v}--" prefix += f"tiny_vae-{use_tiny_vae}--h-{height}--w-{width}" return prefix def _load_model( self, config_path: str, num_inference_steps: int, height: int, width: int, t_index_list: Optional[List[int]] = None, strength: Optional[float] = None, dreambooth_path: Optional[str] = None, lora_dict: Optional[Dict[str, float]] = None, vae_id: Optional[str] = None, acceleration: Literal["none", "xformers", "tensorrt"] = "tensorrt", do_add_noise: bool = True, few_step_model_type: Optional[str] = None, use_tiny_vae: bool = True, cfg_type: Literal["none", "full", "self", "initialize"] = "self", seed: int = 2, engine_dir: Optional[Union[str, Path]] = "engines", opt_unet: bool = False, ) -> StreamAnimateDiffusionDepth: """ Loads the model. This method does the following: 1. Loads the model from the model_id_or_path. 3. Loads the VAE model from the vae_id if needed. 4. Enables acceleration if needed. 6. Load the safety checker if needed. Parameters ---------- config_path : str The path to config, all needed checkpoints are list in config file. t_index_list : List[int] The t_index_list to use for inference. dreambooth_path : Optional[str] The dreambooth path to use for inference. If not passed, will use dreambooth from config. lora_dict : Optional[Dict[str, float]], optional The lora_dict to load, by default None. Keys are the LoRA names and values are the LoRA scales. Example: {'LoRA_1' : 0.5 , 'LoRA_2' : 0.7 ,...} vae_id : Optional[str], optional The vae_id to load, by default None. acceleration : Literal["none", "xfomers", "sfast", "tensorrt"], optional The acceleration method, by default "tensorrt". warmup : int, optional The number of warmup steps to perform, by default 10. do_add_noise : bool, optional Whether to add noise for following denoising steps or not, by default True. use_lcm_lora : bool, optional Whether to use LCM-LoRA or not, by default True. use_tiny_vae : bool, optional Whether to use TinyVAE or not, by default True. cfg_type : Literal["none", "full", "self", "initialize"], optional The cfg_type for img2img mode, by default "self". You cannot use anything other than "none" for txt2img mode. seed : int, optional The seed, by default 2. opt_unet : bool, optional Whether to optimize UNet or not, by default False. Returns ------- AnimatePipeline The loaded pipeline. """ supported_few_step_model = ["LCM"] assert ( few_step_model_type.upper() in supported_few_step_model ), f"Only support few_step_model: {supported_few_step_model}, but receive {few_step_model_type}." # NOTE: build animatediff pipeline from live2diff.animatediff.pipeline import AnimationDepthPipeline try: pipe = AnimationDepthPipeline.build_pipeline( config_path, ).to(device=self.device, dtype=self.dtype) except Exception: # No model found traceback.print_exc() print("Model load has failed. Doesn't exist.") exit() if few_step_model_type.upper() == "LCM": few_step_lora = "latent-consistency/lcm-lora-sdv1-5" stream_pipeline_cls = StreamAnimateDiffusionDepth print(f"Pipeline class: {stream_pipeline_cls}") print(f"Few-step LoRA: {few_step_lora}") # parse clip skip from config from .config import load_config cfg = load_config(config_path) third_party_dict = cfg.third_party_dict clip_skip = third_party_dict.get("clip_skip", 1) stream = stream_pipeline_cls( pipe=pipe, num_inference_steps=num_inference_steps, t_index_list=t_index_list, strength=strength, torch_dtype=self.dtype, width=self.width, height=self.height, do_add_noise=do_add_noise, frame_buffer_size=self.frame_buffer_size, use_denoising_batch=self.use_denoising_batch, cfg_type=cfg_type, clip_skip=clip_skip, ) stream.load_warmup_unet(config_path) stream.load_lora(few_step_lora) stream.fuse_lora() denoising_steps_num = len(stream.t_list) stream.prepare_cache( height=height, width=width, denoising_steps_num=denoising_steps_num, ) kv_cache_list = stream.kv_cache_list if lora_dict is not None: for lora_name, lora_scale in lora_dict.items(): stream.load_lora(lora_name) stream.fuse_lora(lora_scale=lora_scale) print(f"Use LoRA: {lora_name} in weights {lora_scale}") if use_tiny_vae: vae_id = "madebyollin/taesd" if vae_id is None else vae_id stream.vae = AutoencoderTiny.from_pretrained(vae_id).to(device=pipe.device, dtype=pipe.dtype) try: if acceleration == "none": stream.pipe.unet = torch.compile(stream.pipe.unet, options={"triton.cudagraphs": True}, fullgraph=True) stream.vae = torch.compile(stream.vae, options={"triton.cudagraphs": True}, fullgraph=True) if acceleration == "xformers": stream.pipe.enable_xformers_memory_efficient_attention() if acceleration == "tensorrt": from polygraphy import cuda from live2diff.acceleration.tensorrt import ( TorchVAEEncoder, compile_engine, ) from live2diff.acceleration.tensorrt.engine import ( AutoencoderKLEngine, MidasEngine, UNet2DConditionModelDepthEngine, ) from live2diff.acceleration.tensorrt.models import ( VAE, InflatedUNetDepth, Midas, VAEEncoder, ) prefix = self.get_model_prefix( config_path=config_path, few_step_model_type=few_step_model_type, use_tiny_vae=use_tiny_vae, num_denoising_steps=denoising_steps_num, height=height, width=width, dreambooth=dreambooth_path, lora_dict=lora_dict, ) engine_dir = os.path.join(Path(engine_dir), prefix) unet_path = os.path.join(engine_dir, "unet", "unet.engine") unet_opt_path = os.path.join(engine_dir, "unet-opt", "unet.engine.opt") midas_path = os.path.join(engine_dir, "depth", "midas.engine") vae_encoder_path = os.path.join(engine_dir, "vae", "vae_encoder.engine") vae_decoder_path = os.path.join(engine_dir, "vae", "vae_decoder.engine") if not os.path.exists(unet_path): os.makedirs(os.path.dirname(unet_path), exist_ok=True) os.makedirs(os.path.dirname(unet_opt_path), exist_ok=True) unet_model = InflatedUNetDepth( fp16=True, device=stream.device, max_batch_size=stream.trt_unet_batch_size, min_batch_size=stream.trt_unet_batch_size, embedding_dim=stream.text_encoder.config.hidden_size, unet_dim=stream.unet.config.in_channels, kv_cache_list=kv_cache_list, ) compile_engine( torch_model=stream.unet, model_data=unet_model, onnx_path=unet_path + ".onnx", onnx_opt_path=unet_opt_path, # use specific folder for external data engine_path=unet_path, opt_image_height=height, opt_image_width=width, opt_batch_size=stream.trt_unet_batch_size, engine_build_options={"ignore_onnx_optimize": not opt_unet}, ) if not os.path.exists(vae_decoder_path): os.makedirs(os.path.dirname(vae_decoder_path), exist_ok=True) stream.vae.forward = stream.vae.decode max_bz = WARMUP_FRAMES opt_bz = min_bz = 1 vae_decoder_model = VAE( device=stream.device, max_batch_size=max_bz, min_batch_size=min_bz, ) compile_engine( torch_model=stream.vae, model_data=vae_decoder_model, onnx_path=vae_decoder_path + ".onnx", onnx_opt_path=vae_decoder_path + ".opt.onnx", engine_path=vae_decoder_path, opt_image_height=height, opt_image_width=width, opt_batch_size=opt_bz, ) delattr(stream.vae, "forward") if not os.path.exists(midas_path): os.makedirs(os.path.dirname(midas_path), exist_ok=True) max_bz = WARMUP_FRAMES opt_bz = min_bz = 1 midas = Midas( fp16=True, device=stream.device, max_batch_size=max_bz, min_batch_size=min_bz, ) compile_engine( torch_model=stream.depth_detector.half(), model_data=midas, onnx_path=midas_path + ".onnx", onnx_opt_path=midas_path + ".opt.onnx", engine_path=midas_path, opt_batch_size=opt_bz, opt_image_height=384, opt_image_width=384, engine_build_options={ "auto_cast": False, "handle_batch_norm": True, "ignore_onnx_optimize": True, }, ) if not os.path.exists(vae_encoder_path): os.makedirs(os.path.dirname(vae_encoder_path), exist_ok=True) vae_encoder = TorchVAEEncoder(stream.vae).to(torch.device("cuda")) max_bz = WARMUP_FRAMES opt_bz = min_bz = 1 vae_encoder_model = VAEEncoder( device=stream.device, max_batch_size=max_bz, min_batch_size=min_bz, ) compile_engine( torch_model=vae_encoder, model_data=vae_encoder_model, onnx_path=vae_encoder_path + ".onnx", onnx_opt_path=vae_encoder_path + ".opt.onnx", engine_path=vae_encoder_path, opt_batch_size=opt_bz, opt_image_height=height, opt_image_width=width, ) cuda_stream = cuda.Stream() vae_config = stream.vae.config vae_dtype = stream.vae.dtype midas_dtype = stream.depth_detector.dtype stream.unet = UNet2DConditionModelDepthEngine(unet_path, cuda_stream, use_cuda_graph=False) stream.depth_detector = MidasEngine(midas_path, cuda_stream, use_cuda_graph=False) setattr(stream.depth_detector, "dtype", midas_dtype) stream.vae = AutoencoderKLEngine( vae_encoder_path, vae_decoder_path, cuda_stream, stream.pipe.vae_scale_factor, use_cuda_graph=False, ) setattr(stream.vae, "config", vae_config) setattr(stream.vae, "dtype", vae_dtype) stream.is_tensorrt = True gc.collect() torch.cuda.empty_cache() print("TensorRT acceleration enabled.") except Exception: traceback.print_exc() print("Acceleration has failed. Falling back to normal mode.") if seed < 0: # Random seed seed = np.random.randint(0, 1000000) return stream