diff --git "a/easyanimate/ui/ui.py" "b/easyanimate/ui/ui.py" --- "a/easyanimate/ui/ui.py" +++ "b/easyanimate/ui/ui.py" @@ -8,6 +8,7 @@ import random from datetime import datetime from glob import glob +import cv2 import gradio as gr import numpy as np import pkg_resources @@ -21,19 +22,28 @@ from diffusers.utils.import_utils import is_xformers_available from omegaconf import OmegaConf from PIL import Image from safetensors import safe_open -from transformers import (CLIPImageProcessor, CLIPVisionModelWithProjection, +from transformers import (BertModel, BertTokenizer, CLIPImageProcessor, + CLIPVisionModelWithProjection, T5Tokenizer, T5EncoderModel, T5Tokenizer) from easyanimate.data.bucket_sampler import ASPECT_RATIO_512, get_closest_ratio +from easyanimate.models import (name_to_autoencoder_magvit, + name_to_transformer3d) from easyanimate.models.autoencoder_magvit import AutoencoderKLMagvit -from easyanimate.models.transformer3d import Transformer3DModel +from easyanimate.models.transformer3d import (HunyuanTransformer3DModel, + Transformer3DModel) from easyanimate.pipeline.pipeline_easyanimate import EasyAnimatePipeline from easyanimate.pipeline.pipeline_easyanimate_inpaint import \ EasyAnimateInpaintPipeline +from easyanimate.pipeline.pipeline_easyanimate_multi_text_encoder import \ + EasyAnimatePipeline_Multi_Text_Encoder +from easyanimate.pipeline.pipeline_easyanimate_multi_text_encoder_inpaint import \ + EasyAnimatePipeline_Multi_Text_Encoder_Inpaint from easyanimate.utils.lora_utils import merge_lora, unmerge_lora from easyanimate.utils.utils import ( - get_image_to_video_latent, + get_image_to_video_latent, get_video_to_video_latent, get_width_and_height_from_image_and_base_resolution, save_videos_grid) +from easyanimate.utils.fp8_optimization import convert_weight_dtype_wrapper scheduler_dict = { "Euler": EulerDiscreteScheduler, @@ -56,7 +66,7 @@ css = """ """ class EasyAnimateController: - def __init__(self): + def __init__(self, GPU_memory_mode, weight_dtype): # config dirs self.basedir = os.getcwd() self.config_dir = os.path.join(self.basedir, "config") @@ -65,8 +75,7 @@ class EasyAnimateController: self.personalized_model_dir = os.path.join(self.basedir, "models", "Personalized_Model") self.savedir = os.path.join(self.basedir, "samples", datetime.now().strftime("Gradio-%Y-%m-%dT%H-%M-%S")) self.savedir_sample = os.path.join(self.savedir, "sample") - self.edition = "v3" - self.inference_config = OmegaConf.load(os.path.join(self.config_dir, "easyanimate_video_slicevae_motion_module_v3.yaml")) + self.model_type = "Inpaint" os.makedirs(self.savedir, exist_ok=True) self.diffusion_transformer_list = [] @@ -86,8 +95,11 @@ class EasyAnimateController: self.motion_module_path = "none" self.base_model_path = "none" self.lora_model_path = "none" + self.GPU_memory_mode = GPU_memory_mode - self.weight_dtype = torch.bfloat16 + self.weight_dtype = weight_dtype + self.edition = "v5" + self.inference_config = OmegaConf.load(os.path.join(self.config_dir, "easyanimate_video_v5_magvit_multi_text_encoder.yaml")) def refresh_diffusion_transformer(self): self.diffusion_transformer_list = sorted(glob(os.path.join(self.diffusion_transformer_dir, "*/"))) @@ -99,72 +111,179 @@ class EasyAnimateController: def refresh_personalized_model(self): personalized_model_list = sorted(glob(os.path.join(self.personalized_model_dir, "*.safetensors"))) self.personalized_model_list = [os.path.basename(p) for p in personalized_model_list] + + def update_model_type(self, model_type): + self.model_type = model_type def update_edition(self, edition): print("Update edition of EasyAnimate") self.edition = edition if edition == "v1": - self.inference_config = OmegaConf.load(os.path.join(self.config_dir, "easyanimate_video_motion_module_v1.yaml")) + self.inference_config = OmegaConf.load(os.path.join(self.config_dir, "easyanimate_video_v1_motion_module.yaml")) return gr.update(), gr.update(value="none"), gr.update(visible=True), gr.update(visible=True), \ gr.update(value=512, minimum=384, maximum=704, step=32), \ gr.update(value=512, minimum=384, maximum=704, step=32), gr.update(value=80, minimum=40, maximum=80, step=1) elif edition == "v2": - self.inference_config = OmegaConf.load(os.path.join(self.config_dir, "easyanimate_video_magvit_motion_module_v2.yaml")) + self.inference_config = OmegaConf.load(os.path.join(self.config_dir, "easyanimate_video_v2_magvit_motion_module.yaml")) return gr.update(), gr.update(value="none"), gr.update(visible=False), gr.update(visible=False), \ - gr.update(value=672, minimum=128, maximum=1280, step=16), \ - gr.update(value=384, minimum=128, maximum=1280, step=16), gr.update(value=144, minimum=9, maximum=144, step=9) - else: - self.inference_config = OmegaConf.load(os.path.join(self.config_dir, "easyanimate_video_slicevae_motion_module_v3.yaml")) + gr.update(value=672, minimum=128, maximum=1344, step=16), \ + gr.update(value=384, minimum=128, maximum=1344, step=16), gr.update(value=144, minimum=9, maximum=144, step=9) + elif edition == "v3": + self.inference_config = OmegaConf.load(os.path.join(self.config_dir, "easyanimate_video_v3_slicevae_motion_module.yaml")) + return gr.update(), gr.update(value="none"), gr.update(visible=False), gr.update(visible=False), \ + gr.update(value=672, minimum=128, maximum=1344, step=16), \ + gr.update(value=384, minimum=128, maximum=1344, step=16), gr.update(value=144, minimum=8, maximum=144, step=8) + elif edition == "v4": + self.inference_config = OmegaConf.load(os.path.join(self.config_dir, "easyanimate_video_v4_slicevae_multi_text_encoder.yaml")) return gr.update(), gr.update(value="none"), gr.update(visible=False), gr.update(visible=False), \ - gr.update(value=672, minimum=128, maximum=1280, step=16), \ - gr.update(value=384, minimum=128, maximum=1280, step=16), gr.update(value=144, minimum=8, maximum=144, step=8) + gr.update(value=672, minimum=128, maximum=1344, step=16), \ + gr.update(value=384, minimum=128, maximum=1344, step=16), gr.update(value=144, minimum=8, maximum=144, step=8) + elif edition == "v5": + self.inference_config = OmegaConf.load(os.path.join(self.config_dir, "easyanimate_video_v5_magvit_multi_text_encoder.yaml")) + return gr.update(), gr.update(value="none"), gr.update(visible=False), gr.update(visible=False), \ + gr.update(value=672, minimum=128, maximum=1344, step=16), \ + gr.update(value=384, minimum=128, maximum=1344, step=16), gr.update(value=49, minimum=1, maximum=49, step=4) def update_diffusion_transformer(self, diffusion_transformer_dropdown): print("Update diffusion transformer") if diffusion_transformer_dropdown == "none": return gr.update() - if OmegaConf.to_container(self.inference_config['vae_kwargs'])['enable_magvit']: - Choosen_AutoencoderKL = AutoencoderKLMagvit - else: - Choosen_AutoencoderKL = AutoencoderKL + Choosen_AutoencoderKL = name_to_autoencoder_magvit[ + self.inference_config['vae_kwargs'].get('vae_type', 'AutoencoderKL') + ] self.vae = Choosen_AutoencoderKL.from_pretrained( diffusion_transformer_dropdown, subfolder="vae", ).to(self.weight_dtype) - self.transformer = Transformer3DModel.from_pretrained_2d( + if self.inference_config['vae_kwargs'].get('vae_type', 'AutoencoderKL') == 'AutoencoderKLMagvit' and self.weight_dtype == torch.float16: + self.vae.upcast_vae = True + + transformer_additional_kwargs = OmegaConf.to_container(self.inference_config['transformer_additional_kwargs']) + if self.weight_dtype == torch.float16: + transformer_additional_kwargs["upcast_attention"] = True + + # Get Transformer + Choosen_Transformer3DModel = name_to_transformer3d[ + self.inference_config['transformer_additional_kwargs'].get('transformer_type', 'Transformer3DModel') + ] + + self.transformer = Choosen_Transformer3DModel.from_pretrained_2d( diffusion_transformer_dropdown, subfolder="transformer", - transformer_additional_kwargs=OmegaConf.to_container(self.inference_config.transformer_additional_kwargs) + transformer_additional_kwargs=transformer_additional_kwargs ).to(self.weight_dtype) - self.tokenizer = T5Tokenizer.from_pretrained(diffusion_transformer_dropdown, subfolder="tokenizer") - self.text_encoder = T5EncoderModel.from_pretrained(diffusion_transformer_dropdown, subfolder="text_encoder", torch_dtype=self.weight_dtype) + + if self.inference_config['text_encoder_kwargs'].get('enable_multi_text_encoder', False): + tokenizer = BertTokenizer.from_pretrained( + diffusion_transformer_dropdown, subfolder="tokenizer" + ) + tokenizer_2 = T5Tokenizer.from_pretrained( + diffusion_transformer_dropdown, subfolder="tokenizer_2" + ) + else: + tokenizer = T5Tokenizer.from_pretrained( + diffusion_transformer_dropdown, subfolder="tokenizer" + ) + tokenizer_2 = None - # Get pipeline - if self.transformer.config.in_channels != 12: - self.pipeline = EasyAnimatePipeline( - vae=self.vae, - text_encoder=self.text_encoder, - tokenizer=self.tokenizer, - transformer=self.transformer, - scheduler=scheduler_dict["Euler"](**OmegaConf.to_container(self.inference_config.noise_scheduler_kwargs)) + if self.inference_config['text_encoder_kwargs'].get('enable_multi_text_encoder', False): + text_encoder = BertModel.from_pretrained( + diffusion_transformer_dropdown, subfolder="text_encoder", torch_dtype=self.weight_dtype + ) + text_encoder_2 = T5EncoderModel.from_pretrained( + diffusion_transformer_dropdown, subfolder="text_encoder_2", torch_dtype=self.weight_dtype ) else: + text_encoder = T5EncoderModel.from_pretrained( + diffusion_transformer_dropdown, subfolder="text_encoder", torch_dtype=self.weight_dtype + ) + text_encoder_2 = None + + # Get pipeline + if self.transformer.config.in_channels != self.vae.config.latent_channels and self.inference_config['transformer_additional_kwargs'].get('enable_clip_in_inpaint', True): clip_image_encoder = CLIPVisionModelWithProjection.from_pretrained( diffusion_transformer_dropdown, subfolder="image_encoder" - ).to("cuda", self.weight_dtype) + ).to(self.weight_dtype) clip_image_processor = CLIPImageProcessor.from_pretrained( diffusion_transformer_dropdown, subfolder="image_encoder" ) - self.pipeline = EasyAnimateInpaintPipeline( - vae=self.vae, - text_encoder=self.text_encoder, - tokenizer=self.tokenizer, - transformer=self.transformer, - scheduler=scheduler_dict["Euler"](**OmegaConf.to_container(self.inference_config.noise_scheduler_kwargs)), - clip_image_encoder=clip_image_encoder, - clip_image_processor=clip_image_processor, - ) - + else: + clip_image_encoder = None + clip_image_processor = None + + # Get Scheduler + Choosen_Scheduler = scheduler_dict = { + "Euler": EulerDiscreteScheduler, + "Euler A": EulerAncestralDiscreteScheduler, + "DPM++": DPMSolverMultistepScheduler, + "PNDM": PNDMScheduler, + "DDIM": DDIMScheduler, + }["Euler"] + + scheduler = Choosen_Scheduler.from_pretrained( + diffusion_transformer_dropdown, + subfolder="scheduler" + ) + + if self.inference_config['text_encoder_kwargs'].get('enable_multi_text_encoder', False): + if self.transformer.config.in_channels != self.vae.config.latent_channels: + self.pipeline = EasyAnimatePipeline_Multi_Text_Encoder_Inpaint.from_pretrained( + diffusion_transformer_dropdown, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + vae=self.vae, + transformer=self.transformer, + scheduler=scheduler, + torch_dtype=self.weight_dtype, + clip_image_encoder=clip_image_encoder, + clip_image_processor=clip_image_processor, + ) + else: + self.pipeline = EasyAnimatePipeline_Multi_Text_Encoder.from_pretrained( + diffusion_transformer_dropdown, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + vae=self.vae, + transformer=self.transformer, + scheduler=scheduler, + torch_dtype=self.weight_dtype + ) + else: + if self.transformer.config.in_channels != self.vae.config.latent_channels: + self.pipeline = EasyAnimateInpaintPipeline( + diffusion_transformer_dropdown, + text_encoder=text_encoder, + tokenizer=tokenizer, + vae=self.vae, + transformer=self.transformer, + scheduler=scheduler, + torch_dtype=self.weight_dtype, + clip_image_encoder=clip_image_encoder, + clip_image_processor=clip_image_processor, + ) + else: + self.pipeline = EasyAnimatePipeline( + diffusion_transformer_dropdown, + text_encoder=text_encoder, + tokenizer=tokenizer, + vae=self.vae, + transformer=self.transformer, + scheduler=scheduler, + torch_dtype=self.weight_dtype + ) + + if self.GPU_memory_mode == "sequential_cpu_offload": + self.pipeline.enable_sequential_cpu_offload() + elif self.GPU_memory_mode == "model_cpu_offload_and_qfloat8": + self.pipeline.enable_model_cpu_offload() + self.pipeline.enable_autocast_float8_transformer() + convert_weight_dtype_wrapper(self.pipeline.transformer, self.weight_dtype) + else: + self.GPU_memory_mode.enable_model_cpu_offload() print("Update diffusion transformer done") return gr.update() @@ -238,6 +357,10 @@ class EasyAnimateController: cfg_scale_slider, start_image, end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, seed_textbox, is_api = False, ): @@ -251,33 +374,47 @@ class EasyAnimateController: if self.base_model_path != base_model_dropdown: self.update_base_model(base_model_dropdown) - if self.motion_module_path != motion_module_dropdown: - self.update_motion_module(motion_module_dropdown) - if self.lora_model_path != lora_model_dropdown: print("Update lora model") self.update_lora_model(lora_model_dropdown) - - if resize_method == "Resize to the Start Image": - if start_image is None: + + if control_video is not None and self.model_type == "Inpaint": + if is_api: + return "", f"If specifying the control video, please set the model_type == \"Control\". " + else: + raise gr.Error(f"If specifying the control video, please set the model_type == \"Control\". ") + + if control_video is None and self.model_type == "Control": + if is_api: + return "", f"If set the model_type == \"Control\", please specifying the control video. " + else: + raise gr.Error(f"If set the model_type == \"Control\", please specifying the control video. ") + + if resize_method == "Resize according to Reference": + if start_image is None and validation_video is None and control_video is None: if is_api: - return "", f"Please upload an image when using \"Resize to the Start Image\"." + return "", f"Please upload an image when using \"Resize according to Reference\"." else: - raise gr.Error(f"Please upload an image when using \"Resize to the Start Image\".") + raise gr.Error(f"Please upload an image when using \"Resize according to Reference\".") aspect_ratio_sample_size = {key : [x / 512 * base_resolution for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} - - original_width, original_height = start_image[0].size if type(start_image) is list else Image.open(start_image).size + if self.model_type == "Inpaint": + if validation_video is not None: + original_width, original_height = Image.fromarray(cv2.VideoCapture(validation_video).read()[1]).size + else: + original_width, original_height = start_image[0].size if type(start_image) is list else Image.open(start_image).size + else: + original_width, original_height = Image.fromarray(cv2.VideoCapture(control_video).read()[1]).size closest_size, closest_ratio = get_closest_ratio(original_height, original_width, ratios=aspect_ratio_sample_size) height_slider, width_slider = [int(x / 16) * 16 for x in closest_size] - if self.transformer.config.in_channels != 12 and start_image is not None: + if self.transformer.config.in_channels == self.vae.config.latent_channels and start_image is not None: if is_api: return "", f"Please select an image to video pretrained model while using image to video." else: raise gr.Error(f"Please select an image to video pretrained model while using image to video.") - if self.transformer.config.in_channels != 12 and generation_method == "Long Video Generation": + if self.transformer.config.in_channels == self.vae.config.latent_channels and generation_method == "Long Video Generation": if is_api: return "", f"Please select an image to video pretrained model while using long video generation." else: @@ -289,88 +426,118 @@ class EasyAnimateController: else: raise gr.Error(f"If specifying the ending image of the video, please specify a starting image of the video.") + fps = {"v1": 12, "v2": 24, "v3": 24, "v4": 24, "v5": 8}[self.edition] is_image = True if generation_method == "Image Generation" else False - if is_xformers_available(): self.transformer.enable_xformers_memory_efficient_attention() + if is_xformers_available() and not self.inference_config['text_encoder_kwargs'].get('enable_multi_text_encoder', False): self.transformer.enable_xformers_memory_efficient_attention() - self.pipeline.scheduler = scheduler_dict[sampler_dropdown](**OmegaConf.to_container(self.inference_config.noise_scheduler_kwargs)) + self.pipeline.scheduler = scheduler_dict[sampler_dropdown].from_config(self.pipeline.scheduler.config) if self.lora_model_path != "none": # lora part self.pipeline = merge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) - self.pipeline.to("cuda") if int(seed_textbox) != -1 and seed_textbox != "": torch.manual_seed(int(seed_textbox)) else: seed_textbox = np.random.randint(0, 1e10) generator = torch.Generator(device="cuda").manual_seed(int(seed_textbox)) try: - if self.transformer.config.in_channels == 12: - if generation_method == "Long Video Generation": - init_frames = 0 - last_frames = init_frames + partial_video_length - while init_frames < length_slider: - if last_frames >= length_slider: - if self.pipeline.vae.quant_conv.weight.ndim==5: - mini_batch_encoder = self.pipeline.vae.mini_batch_encoder + if self.model_type == "Inpaint": + if self.transformer.config.in_channels != self.vae.config.latent_channels: + if generation_method == "Long Video Generation": + if validation_video is not None: + raise gr.Error(f"Video to Video is not Support Long Video Generation now.") + init_frames = 0 + last_frames = init_frames + partial_video_length + while init_frames < length_slider: + if last_frames >= length_slider: _partial_video_length = length_slider - init_frames - _partial_video_length = int(_partial_video_length // mini_batch_encoder * mini_batch_encoder) + if self.vae.cache_mag_vae: + _partial_video_length = int((_partial_video_length - 1) // self.vae.mini_batch_encoder * self.vae.mini_batch_encoder) + 1 + else: + _partial_video_length = int(_partial_video_length // self.vae.mini_batch_encoder * self.vae.mini_batch_encoder) + + if _partial_video_length <= 0: + break else: - _partial_video_length = length_slider - init_frames - - if _partial_video_length <= 0: - break - else: - _partial_video_length = partial_video_length + _partial_video_length = partial_video_length - if last_frames >= length_slider: - input_video, input_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, video_length=_partial_video_length, sample_size=(height_slider, width_slider)) - else: - input_video, input_video_mask, clip_image = get_image_to_video_latent(start_image, None, video_length=_partial_video_length, sample_size=(height_slider, width_slider)) - - with torch.no_grad(): - sample = self.pipeline( - prompt_textbox, - negative_prompt = negative_prompt_textbox, - num_inference_steps = sample_step_slider, - guidance_scale = cfg_scale_slider, - width = width_slider, - height = height_slider, - video_length = _partial_video_length, - generator = generator, - - video = input_video, - mask_video = input_video_mask, - clip_image = clip_image, - strength = 1, - ).videos - - if init_frames != 0: - mix_ratio = torch.from_numpy( - np.array([float(_index) / float(overlap_video_length) for _index in range(overlap_video_length)], np.float32) - ).unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) + if last_frames >= length_slider: + input_video, input_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, video_length=_partial_video_length, sample_size=(height_slider, width_slider)) + else: + input_video, input_video_mask, clip_image = get_image_to_video_latent(start_image, None, video_length=_partial_video_length, sample_size=(height_slider, width_slider)) + + with torch.no_grad(): + sample = self.pipeline( + prompt_textbox, + negative_prompt = negative_prompt_textbox, + num_inference_steps = sample_step_slider, + guidance_scale = cfg_scale_slider, + width = width_slider, + height = height_slider, + video_length = _partial_video_length, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + strength = 1, + ).videos - new_sample[:, :, -overlap_video_length:] = new_sample[:, :, -overlap_video_length:] * (1 - mix_ratio) + \ - sample[:, :, :overlap_video_length] * mix_ratio - new_sample = torch.cat([new_sample, sample[:, :, overlap_video_length:]], dim = 2) + if init_frames != 0: + mix_ratio = torch.from_numpy( + np.array([float(_index) / float(overlap_video_length) for _index in range(overlap_video_length)], np.float32) + ).unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) + + new_sample[:, :, -overlap_video_length:] = new_sample[:, :, -overlap_video_length:] * (1 - mix_ratio) + \ + sample[:, :, :overlap_video_length] * mix_ratio + new_sample = torch.cat([new_sample, sample[:, :, overlap_video_length:]], dim = 2) - sample = new_sample - else: - new_sample = sample + sample = new_sample + else: + new_sample = sample - if last_frames >= length_slider: - break + if last_frames >= length_slider: + break - start_image = [ - Image.fromarray( - (sample[0, :, _index].transpose(0, 1).transpose(1, 2) * 255).numpy().astype(np.uint8) - ) for _index in range(-overlap_video_length, 0) - ] + start_image = [ + Image.fromarray( + (sample[0, :, _index].transpose(0, 1).transpose(1, 2) * 255).numpy().astype(np.uint8) + ) for _index in range(-overlap_video_length, 0) + ] - init_frames = init_frames + _partial_video_length - overlap_video_length - last_frames = init_frames + _partial_video_length + init_frames = init_frames + _partial_video_length - overlap_video_length + last_frames = init_frames + _partial_video_length + else: + if self.vae.cache_mag_vae: + length_slider = int((length_slider - 1) // self.vae.mini_batch_encoder * self.vae.mini_batch_encoder) + 1 + else: + length_slider = int(length_slider // self.vae.mini_batch_encoder * self.vae.mini_batch_encoder) + if validation_video is not None: + input_video, input_video_mask, clip_image = get_video_to_video_latent(validation_video, length_slider if not is_image else 1, sample_size=(height_slider, width_slider), validation_video_mask=validation_video_mask, fps=fps) + strength = denoise_strength + else: + input_video, input_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, length_slider if not is_image else 1, sample_size=(height_slider, width_slider)) + strength = 1 + + sample = self.pipeline( + prompt_textbox, + negative_prompt = negative_prompt_textbox, + num_inference_steps = sample_step_slider, + guidance_scale = cfg_scale_slider, + width = width_slider, + height = height_slider, + video_length = length_slider if not is_image else 1, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + strength = strength, + ).videos else: - input_video, input_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, length_slider if not is_image else 1, sample_size=(height_slider, width_slider)) - + if self.vae.cache_mag_vae: + length_slider = int((length_slider - 1) // self.vae.mini_batch_encoder * self.vae.mini_batch_encoder) + 1 + else: + length_slider = int(length_slider // self.vae.mini_batch_encoder * self.vae.mini_batch_encoder) + sample = self.pipeline( prompt_textbox, negative_prompt = negative_prompt_textbox, @@ -379,13 +546,15 @@ class EasyAnimateController: width = width_slider, height = height_slider, video_length = length_slider if not is_image else 1, - generator = generator, - - video = input_video, - mask_video = input_video_mask, - clip_image = clip_image, + generator = generator ).videos else: + if self.vae.cache_mag_vae: + length_slider = int((length_slider - 1) // self.vae.mini_batch_encoder * self.vae.mini_batch_encoder) + 1 + else: + length_slider = int(length_slider // self.vae.mini_batch_encoder * self.vae.mini_batch_encoder) + input_video, input_video_mask, clip_image = get_video_to_video_latent(control_video, length_slider if not is_image else 1, sample_size=(height_slider, width_slider), fps=fps) + sample = self.pipeline( prompt_textbox, negative_prompt = negative_prompt_textbox, @@ -394,7 +563,9 @@ class EasyAnimateController: width = width_slider, height = height_slider, video_length = length_slider if not is_image else 1, - generator = generator + generator = generator, + + control_video = input_video, ).videos except Exception as e: gc.collect() @@ -457,7 +628,7 @@ class EasyAnimateController: return gr.Image.update(value=save_sample_path, visible=True), gr.Video.update(value=None, visible=False), "Success" else: save_sample_path = os.path.join(self.savedir_sample, prefix + f".mp4") - save_videos_grid(sample, save_sample_path, fps=12 if self.edition == "v1" else 24) + save_videos_grid(sample, save_sample_path, fps=fps) if is_api: return save_sample_path, "Success" @@ -468,8 +639,8 @@ class EasyAnimateController: return gr.Image.update(visible=False, value=None), gr.Video.update(value=save_sample_path, visible=True), "Success" -def ui(): - controller = EasyAnimateController() +def ui(GPU_memory_mode, weight_dtype): + controller = EasyAnimateController(GPU_memory_mode, weight_dtype) with gr.Blocks(css=css) as demo: gr.Markdown( @@ -486,19 +657,32 @@ def ui(): with gr.Column(variant="panel"): gr.Markdown( """ - ### 1. EasyAnimate Edition (EasyAnimate版本). + ### 1. EasyAnimate Model Type (EasyAnimate模型的种类，正常模型还是控制模型). + """ + ) + with gr.Row(): + model_type = gr.Dropdown( + label="The model type of EasyAnimate (EasyAnimate模型的种类，正常模型还是控制模型)", + choices=["Inpaint", "Control"], + value="Inpaint", + interactive=True, + ) + with gr.Column(variant="panel"): + gr.Markdown( + """ + ### 2. EasyAnimate Edition (EasyAnimate版本). """ ) with gr.Row(): easyanimate_edition_dropdown = gr.Dropdown( label="The config of EasyAnimate Edition (EasyAnimate版本配置)", - choices=["v1", "v2", "v3"], - value="v3", + choices=["v1", "v2", "v3", "v4", "v5"], + value="v5", interactive=True, ) gr.Markdown( """ - ### 2. Model checkpoints (模型路径). + ### 3. Model checkpoints (模型路径). """ ) with gr.Row(): @@ -568,21 +752,21 @@ def ui(): ) prompt_textbox = gr.Textbox(label="Prompt (正向提示词)", lines=2, value="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.") - negative_prompt_textbox = gr.Textbox(label="Negative prompt (负向提示词)", lines=2, value="The video is not of a high quality, it has a low resolution, and the audio quality is not clear. Strange motion trajectory, a poor composition and deformed video, low resolution, duplicate and ugly, strange body structure, long and strange neck, bad teeth, bad eyes, bad limbs, bad hands, rotating camera, blurry camera, shaking camera. Deformation, low-resolution, blurry, ugly, distortion." ) + negative_prompt_textbox = gr.Textbox(label="Negative prompt (负向提示词)", lines=2, value="Unclear, mutated, deformed, distorted, dark frames, fixed frames, comic book, comic book, small and indistinguishable subject." ) with gr.Row(): with gr.Column(): with gr.Row(): sampler_dropdown = gr.Dropdown(label="Sampling method (采样器种类)", choices=list(scheduler_dict.keys()), value=list(scheduler_dict.keys())[0]) - sample_step_slider = gr.Slider(label="Sampling steps (生成步数)", value=30, minimum=10, maximum=100, step=1) + sample_step_slider = gr.Slider(label="Sampling steps (生成步数)", value=50, minimum=10, maximum=100, step=1) resize_method = gr.Radio( - ["Generate by", "Resize to the Start Image"], + ["Generate by", "Resize according to Reference"], value="Generate by", show_label=False, ) - width_slider = gr.Slider(label="Width (视频宽度)", value=672, minimum=128, maximum=1280, step=16) - height_slider = gr.Slider(label="Height (视频高度)", value=384, minimum=128, maximum=1280, step=16) + width_slider = gr.Slider(label="Width (视频宽度)", value=672, minimum=128, maximum=1344, step=16) + height_slider = gr.Slider(label="Height (视频高度)", value=384, minimum=128, maximum=1344, step=16) base_resolution = gr.Radio(label="Base Resolution of Pretrained Models", value=512, choices=[512, 768, 960], visible=False) with gr.Group(): @@ -592,17 +776,25 @@ def ui(): show_label=False, ) with gr.Row(): - length_slider = gr.Slider(label="Animation length (视频帧数)", value=144, minimum=8, maximum=144, step=8) + length_slider = gr.Slider(label="Animation length (视频帧数)", value=49, minimum=1, maximum=49, step=4) overlap_video_length = gr.Slider(label="Overlap length (视频续写的重叠帧数)", value=4, minimum=1, maximum=4, step=1, visible=False) - partial_video_length = gr.Slider(label="Partial video generation length (每个部分的视频生成帧数)", value=72, minimum=8, maximum=144, step=8, visible=False) + partial_video_length = gr.Slider(label="Partial video generation length (每个部分的视频生成帧数)", value=25, minimum=5, maximum=49, step=4, visible=False) - with gr.Accordion("Image to Video (图片到视频)", open=False): - start_image = gr.Image(label="The image at the beginning of the video (图片到视频的开始图片)", show_label=True, elem_id="i2v_start", sources="upload", type="filepath") + source_method = gr.Radio( + ["Text to Video (文本到视频)", "Image to Video (图片到视频)", "Video to Video (视频到视频)", "Video Control (视频控制)"], + value="Text to Video (文本到视频)", + show_label=False, + ) + with gr.Column(visible = False) as image_to_video_col: + start_image = gr.Image( + label="The image at the beginning of the video (图片到视频的开始图片)", show_label=True, + elem_id="i2v_start", sources="upload", type="filepath", + ) template_gallery_path = ["asset/1.png", "asset/2.png", "asset/3.png", "asset/4.png", "asset/5.png"] def select_template(evt: gr.SelectData): text = { - "asset/1.png": "The dog is looking at camera and smiling. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", + "asset/1.png": "The dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", "asset/2.png": "a sailboat sailing in rough seas with a dramatic sunset. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", "asset/3.png": "a beautiful woman with long hair and a dress blowing in the wind. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", "asset/4.png": "a man in an astronaut suit playing a guitar. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", @@ -623,7 +815,37 @@ def ui(): with gr.Accordion("The image at the ending of the video (图片到视频的结束图片[非必需, Optional])", open=False): end_image = gr.Image(label="The image at the ending of the video (图片到视频的结束图片[非必需, Optional])", show_label=False, elem_id="i2v_end", sources="upload", type="filepath") - cfg_scale_slider = gr.Slider(label="CFG Scale (引导系数)", value=7.0, minimum=0, maximum=20) + with gr.Column(visible = False) as video_to_video_col: + with gr.Row(): + validation_video = gr.Video( + label="The video to convert (视频转视频的参考视频)", show_label=True, + elem_id="v2v", sources="upload", + ) + with gr.Accordion("The mask of the video to inpaint (视频重新绘制的mask[非必需, Optional])", open=False): + gr.Markdown( + """ + - Please set a larger denoise_strength when using validation_video_mask, such as 1.00 instead of 0.70 + - (请设置更大的denoise_strength，当使用validation_video_mask的时候，比如1而不是0.70) + """ + ) + validation_video_mask = gr.Image( + label="The mask of the video to inpaint (视频重新绘制的mask[非必需, Optional])", + show_label=False, elem_id="v2v_mask", sources="upload", type="filepath" + ) + denoise_strength = gr.Slider(label="Denoise strength (重绘系数)", value=0.70, minimum=0.10, maximum=1.00, step=0.01) + + with gr.Column(visible = False) as control_video_col: + gr.Markdown( + """ + Demo pose control video can be downloaded here [URL](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1.1/pose.mp4). + """ + ) + control_video = gr.Video( + label="The control video (用于提供控制信号的video)", show_label=True, + elem_id="v2v_control", sources="upload", + ) + + cfg_scale_slider = gr.Slider(label="CFG Scale (引导系数)", value=6.0, minimum=0, maximum=20) with gr.Row(): seed_textbox = gr.Textbox(label="Seed (随机种子)", value=43) @@ -645,17 +867,49 @@ def ui(): interactive=False ) - def upload_generation_method(generation_method): + model_type.change( + fn=controller.update_model_type, + inputs=[model_type], + outputs=[] + ) + + def upload_generation_method(generation_method, easyanimate_edition_dropdown): + if easyanimate_edition_dropdown == "v1": + f_maximum = 80 + f_value = 80 + elif easyanimate_edition_dropdown in ["v2", "v3", "v4"]: + f_maximum = 144 + f_value = 144 + else: + f_maximum = 49 + f_value = 49 + if generation_method == "Video Generation": - return [gr.update(visible=True, maximum=144, value=144), gr.update(visible=False), gr.update(visible=False)] + return [gr.update(visible=True, maximum=f_maximum, value=f_value), gr.update(visible=False), gr.update(visible=False)] elif generation_method == "Image Generation": return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=False)] else: - return [gr.update(visible=True, maximum=1440), gr.update(visible=True), gr.update(visible=True)] + return [gr.update(visible=True, maximum=1200), gr.update(visible=True), gr.update(visible=True)] generation_method.change( upload_generation_method, generation_method, [length_slider, overlap_video_length, partial_video_length] ) + def upload_source_method(source_method): + if source_method == "Text to Video (文本到视频)": + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Image to Video (图片到视频)": + return [gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Video to Video (视频到视频)": + return [gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(), gr.update(), gr.update(value=None)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update()] + source_method.change( + upload_source_method, source_method, [ + image_to_video_col, video_to_video_col, control_video_col, start_image, end_image, + validation_video, validation_video_mask, control_video + ] + ) + def upload_resize_method(resize_method): if resize_method == "Generate by": return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] @@ -701,6 +955,10 @@ def ui(): cfg_scale_slider, start_image, end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, seed_textbox, ], outputs=[result_image, result_video, infer_progress] @@ -709,10 +967,7 @@ def ui(): class EasyAnimateController_Modelscope: - def __init__(self, edition, config_path, model_name, savedir_sample): - # Weight Dtype - weight_dtype = torch.bfloat16 - + def __init__(self, model_type, edition, config_path, model_name, savedir_sample, GPU_memory_mode, weight_dtype): # Basic dir self.basedir = os.getcwd() self.personalized_model_dir = os.path.join(self.basedir, "models", "Personalized_Model") @@ -722,65 +977,152 @@ class EasyAnimateController_Modelscope: os.makedirs(self.savedir_sample, exist_ok=True) # Config and model path + self.model_type = model_type self.edition = edition + self.weight_dtype = weight_dtype self.inference_config = OmegaConf.load(config_path) - # Get Transformer - self.transformer = Transformer3DModel.from_pretrained_2d( - model_name, - subfolder="transformer", - transformer_additional_kwargs=OmegaConf.to_container(self.inference_config['transformer_additional_kwargs']) - ).to(weight_dtype) - if OmegaConf.to_container(self.inference_config['vae_kwargs'])['enable_magvit']: - Choosen_AutoencoderKL = AutoencoderKLMagvit - else: - Choosen_AutoencoderKL = AutoencoderKL + Choosen_AutoencoderKL = name_to_autoencoder_magvit[ + self.inference_config['vae_kwargs'].get('vae_type', 'AutoencoderKL') + ] self.vae = Choosen_AutoencoderKL.from_pretrained( model_name, - subfolder="vae" - ).to(weight_dtype) - self.tokenizer = T5Tokenizer.from_pretrained( - model_name, - subfolder="tokenizer" - ) - self.text_encoder = T5EncoderModel.from_pretrained( + subfolder="vae", + ).to(self.weight_dtype) + if self.inference_config['vae_kwargs'].get('vae_type', 'AutoencoderKL') == 'AutoencoderKLMagvit' and weight_dtype == torch.float16: + self.vae.upcast_vae = True + + transformer_additional_kwargs = OmegaConf.to_container(self.inference_config['transformer_additional_kwargs']) + if self.weight_dtype == torch.float16: + transformer_additional_kwargs["upcast_attention"] = True + + # Get Transformer + Choosen_Transformer3DModel = name_to_transformer3d[ + self.inference_config['transformer_additional_kwargs'].get('transformer_type', 'Transformer3DModel') + ] + + self.transformer = Choosen_Transformer3DModel.from_pretrained_2d( model_name, - subfolder="text_encoder", - torch_dtype=weight_dtype - ) - # Get pipeline - if self.transformer.config.in_channels != 12: - self.pipeline = EasyAnimatePipeline( - vae=self.vae, - text_encoder=self.text_encoder, - tokenizer=self.tokenizer, - transformer=self.transformer, - scheduler=scheduler_dict["Euler"](**OmegaConf.to_container(self.inference_config.noise_scheduler_kwargs)) + subfolder="transformer", + transformer_additional_kwargs=transformer_additional_kwargs + ).to(self.weight_dtype) + + if self.inference_config['text_encoder_kwargs'].get('enable_multi_text_encoder', False): + tokenizer = BertTokenizer.from_pretrained( + model_name, subfolder="tokenizer" + ) + tokenizer_2 = T5Tokenizer.from_pretrained( + model_name, subfolder="tokenizer_2" + ) + else: + tokenizer = T5Tokenizer.from_pretrained( + model_name, subfolder="tokenizer" + ) + tokenizer_2 = None + + if self.inference_config['text_encoder_kwargs'].get('enable_multi_text_encoder', False): + text_encoder = BertModel.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=self.weight_dtype + ) + text_encoder_2 = T5EncoderModel.from_pretrained( + model_name, subfolder="text_encoder_2", torch_dtype=self.weight_dtype ) else: + text_encoder = T5EncoderModel.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=self.weight_dtype + ) + text_encoder_2 = None + + # Get pipeline + if self.transformer.config.in_channels != self.vae.config.latent_channels and self.inference_config['transformer_additional_kwargs'].get('enable_clip_in_inpaint', True): clip_image_encoder = CLIPVisionModelWithProjection.from_pretrained( model_name, subfolder="image_encoder" - ).to("cuda", weight_dtype) + ).to(self.weight_dtype) clip_image_processor = CLIPImageProcessor.from_pretrained( model_name, subfolder="image_encoder" ) - self.pipeline = EasyAnimateInpaintPipeline( - vae=self.vae, - text_encoder=self.text_encoder, - tokenizer=self.tokenizer, - transformer=self.transformer, - scheduler=scheduler_dict["Euler"](**OmegaConf.to_container(self.inference_config.noise_scheduler_kwargs)), - clip_image_encoder=clip_image_encoder, - clip_image_processor=clip_image_processor, - ) - - print("Update diffusion transformer done") + else: + clip_image_encoder = None + clip_image_processor = None + + # Get Scheduler + Choosen_Scheduler = scheduler_dict = { + "Euler": EulerDiscreteScheduler, + "Euler A": EulerAncestralDiscreteScheduler, + "DPM++": DPMSolverMultistepScheduler, + "PNDM": PNDMScheduler, + "DDIM": DDIMScheduler, + }["Euler"] + + scheduler = Choosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" + ) + if self.inference_config['text_encoder_kwargs'].get('enable_multi_text_encoder', False): + if self.transformer.config.in_channels != self.vae.config.latent_channels: + self.pipeline = EasyAnimatePipeline_Multi_Text_Encoder_Inpaint.from_pretrained( + model_name, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + vae=self.vae, + transformer=self.transformer, + scheduler=scheduler, + torch_dtype=self.weight_dtype, + clip_image_encoder=clip_image_encoder, + clip_image_processor=clip_image_processor, + ) + else: + self.pipeline = EasyAnimatePipeline_Multi_Text_Encoder.from_pretrained( + model_name, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + vae=self.vae, + transformer=self.transformer, + scheduler=scheduler, + torch_dtype=self.weight_dtype + ) + else: + if self.transformer.config.in_channels != self.vae.config.latent_channels: + self.pipeline = EasyAnimateInpaintPipeline( + model_name, + text_encoder=text_encoder, + tokenizer=tokenizer, + vae=self.vae, + transformer=self.transformer, + scheduler=scheduler, + torch_dtype=self.weight_dtype, + clip_image_encoder=clip_image_encoder, + clip_image_processor=clip_image_processor, + ) + else: + self.pipeline = EasyAnimatePipeline( + model_name, + text_encoder=text_encoder, + tokenizer=tokenizer, + vae=self.vae, + transformer=self.transformer, + scheduler=scheduler, + torch_dtype=self.weight_dtype + ) + + if GPU_memory_mode == "sequential_cpu_offload": + self.pipeline.enable_sequential_cpu_offload() + elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + self.pipeline.enable_model_cpu_offload() + self.pipeline.enable_autocast_float8_transformer() + convert_weight_dtype_wrapper(self.pipeline.transformer, weight_dtype) + else: + GPU_memory_mode.enable_model_cpu_offload() + print("Update diffusion transformer done") def refresh_personalized_model(self): personalized_model_list = sorted(glob(os.path.join(self.personalized_model_dir, "*.safetensors"))) self.personalized_model_list = [os.path.basename(p) for p in personalized_model_list] - def update_lora_model(self, lora_model_dropdown): print("Update lora model") if lora_model_dropdown == "none": @@ -789,7 +1131,6 @@ class EasyAnimateController_Modelscope: lora_model_dropdown = os.path.join(self.personalized_model_dir, lora_model_dropdown) self.lora_model_path = lora_model_dropdown return gr.update() - def generate( self, @@ -808,9 +1149,15 @@ class EasyAnimateController_Modelscope: base_resolution, generation_method, length_slider, + overlap_video_length, + partial_video_length, cfg_scale_slider, start_image, end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, seed_textbox, is_api = False, ): @@ -824,55 +1171,109 @@ class EasyAnimateController_Modelscope: if self.lora_model_path != lora_model_dropdown: print("Update lora model") self.update_lora_model(lora_model_dropdown) + + if control_video is not None and self.model_type == "Inpaint": + if is_api: + return "", f"If specifying the control video, please set the model_type == \"Control\". " + else: + raise gr.Error(f"If specifying the control video, please set the model_type == \"Control\". ") + + if control_video is None and self.model_type == "Control": + if is_api: + return "", f"If set the model_type == \"Control\", please specifying the control video. " + else: + raise gr.Error(f"If set the model_type == \"Control\", please specifying the control video. ") - if resize_method == "Resize to the Start Image": - if start_image is None: - raise gr.Error(f"Please upload an image when using \"Resize to the Start Image\".") + if resize_method == "Resize according to Reference": + if start_image is None and validation_video is None and control_video is None: + if is_api: + return "", f"Please upload an image when using \"Resize according to Reference\"." + else: + raise gr.Error(f"Please upload an image when using \"Resize according to Reference\".") aspect_ratio_sample_size = {key : [x / 512 * base_resolution for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} - original_width, original_height = start_image[0].size if type(start_image) is list else Image.open(start_image).size + if self.model_type == "Inpaint": + if validation_video is not None: + original_width, original_height = Image.fromarray(cv2.VideoCapture(validation_video).read()[1]).size + else: + original_width, original_height = start_image[0].size if type(start_image) is list else Image.open(start_image).size + else: + original_width, original_height = Image.fromarray(cv2.VideoCapture(control_video).read()[1]).size closest_size, closest_ratio = get_closest_ratio(original_height, original_width, ratios=aspect_ratio_sample_size) height_slider, width_slider = [int(x / 16) * 16 for x in closest_size] - if self.transformer.config.in_channels != 12 and start_image is not None: - raise gr.Error(f"Please select an image to video pretrained model while using image to video.") - + if self.transformer.config.in_channels == self.vae.config.latent_channels and start_image is not None: + if is_api: + return "", f"Please select an image to video pretrained model while using image to video." + else: + raise gr.Error(f"Please select an image to video pretrained model while using image to video.") + if start_image is None and end_image is not None: - raise gr.Error(f"If specifying the ending image of the video, please specify a starting image of the video.") + if is_api: + return "", f"If specifying the ending image of the video, please specify a starting image of the video." + else: + raise gr.Error(f"If specifying the ending image of the video, please specify a starting image of the video.") + fps = {"v1": 12, "v2": 24, "v3": 24, "v4": 24, "v5": 8}[self.edition] is_image = True if generation_method == "Image Generation" else False - if is_xformers_available(): self.transformer.enable_xformers_memory_efficient_attention() - - self.pipeline.scheduler = scheduler_dict[sampler_dropdown](**OmegaConf.to_container(self.inference_config.noise_scheduler_kwargs)) + self.pipeline.scheduler = scheduler_dict[sampler_dropdown].from_config(self.pipeline.scheduler.config) if self.lora_model_path != "none": # lora part self.pipeline = merge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) - self.pipeline.to("cuda") if int(seed_textbox) != -1 and seed_textbox != "": torch.manual_seed(int(seed_textbox)) else: seed_textbox = np.random.randint(0, 1e10) generator = torch.Generator(device="cuda").manual_seed(int(seed_textbox)) try: - if self.transformer.config.in_channels == 12: - input_video, input_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, length_slider if not is_image else 1, sample_size=(height_slider, width_slider)) + if self.model_type == "Inpaint": + if self.vae.cache_mag_vae: + length_slider = int((length_slider - 1) // self.vae.mini_batch_encoder * self.vae.mini_batch_encoder) + 1 + else: + length_slider = int(length_slider // self.vae.mini_batch_encoder * self.vae.mini_batch_encoder) + + if self.transformer.config.in_channels != self.vae.config.latent_channels: + if validation_video is not None: + input_video, input_video_mask, clip_image = get_video_to_video_latent(validation_video, length_slider if not is_image else 1, sample_size=(height_slider, width_slider), validation_video_mask=validation_video_mask, fps=fps) + strength = denoise_strength + else: + input_video, input_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, length_slider if not is_image else 1, sample_size=(height_slider, width_slider)) + strength = 1 - sample = self.pipeline( - prompt_textbox, - negative_prompt = negative_prompt_textbox, - num_inference_steps = sample_step_slider, - guidance_scale = cfg_scale_slider, - width = width_slider, - height = height_slider, - video_length = length_slider if not is_image else 1, - generator = generator, + sample = self.pipeline( + prompt_textbox, + negative_prompt = negative_prompt_textbox, + num_inference_steps = sample_step_slider, + guidance_scale = cfg_scale_slider, + width = width_slider, + height = height_slider, + video_length = length_slider if not is_image else 1, + generator = generator, - video = input_video, - mask_video = input_video_mask, - clip_image = clip_image, - ).videos + video = input_video, + mask_video = input_video_mask, + strength = strength, + ).videos + else: + sample = self.pipeline( + prompt_textbox, + negative_prompt = negative_prompt_textbox, + num_inference_steps = sample_step_slider, + guidance_scale = cfg_scale_slider, + width = width_slider, + height = height_slider, + video_length = length_slider if not is_image else 1, + generator = generator + ).videos else: + if self.vae.cache_mag_vae: + length_slider = int((length_slider - 1) // self.vae.mini_batch_encoder * self.vae.mini_batch_encoder) + 1 + else: + length_slider = int(length_slider // self.vae.mini_batch_encoder * self.vae.mini_batch_encoder) + + input_video, input_video_mask, clip_image = get_video_to_video_latent(control_video, length_slider if not is_image else 1, sample_size=(height_slider, width_slider), fps=fps) + sample = self.pipeline( prompt_textbox, negative_prompt = negative_prompt_textbox, @@ -881,7 +1282,9 @@ class EasyAnimateController_Modelscope: width = width_slider, height = height_slider, video_length = length_slider if not is_image else 1, - generator = generator + generator = generator, + + control_video = input_video, ).videos except Exception as e: gc.collect() @@ -927,7 +1330,7 @@ class EasyAnimateController_Modelscope: return gr.Image.update(value=save_sample_path, visible=True), gr.Video.update(value=None, visible=False), "Success" else: save_sample_path = os.path.join(self.savedir_sample, prefix + f".mp4") - save_videos_grid(sample, save_sample_path, fps=12 if self.edition == "v1" else 24) + save_videos_grid(sample, save_sample_path, fps=fps) if is_api: return save_sample_path, "Success" else: @@ -937,8 +1340,8 @@ class EasyAnimateController_Modelscope: return gr.Image.update(visible=False, value=None), gr.Video.update(value=save_sample_path, visible=True), "Success" -def ui_modelscope(edition, config_path, model_name, savedir_sample): - controller = EasyAnimateController_Modelscope(edition, config_path, model_name, savedir_sample) +def ui_modelscope(model_type, edition, config_path, model_name, savedir_sample, GPU_memory_mode, weight_dtype): + controller = EasyAnimateController_Modelscope(model_type, edition, config_path, model_name, savedir_sample, GPU_memory_mode, weight_dtype) with gr.Blocks(css=css) as demo: gr.Markdown( @@ -989,9 +1392,9 @@ def ui_modelscope(edition, config_path, model_name, savedir_sample): with gr.Row(): lora_model_dropdown = gr.Dropdown( label="Select LoRA model", - choices=["none", "easyanimatev2_minimalism_lora.safetensors"], + choices=["none"], value="none", - interactive=True, + interactive=False, ) lora_alpha_slider = gr.Slider(label="LoRA alpha (LoRA权重)", value=0.55, minimum=0, maximum=2, interactive=True) @@ -1004,13 +1407,13 @@ def ui_modelscope(edition, config_path, model_name, savedir_sample): ) prompt_textbox = gr.Textbox(label="Prompt (正向提示词)", lines=2, value="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.") - negative_prompt_textbox = gr.Textbox(label="Negative prompt (负向提示词)", lines=2, value="The video is not of a high quality, it has a low resolution, and the audio quality is not clear. Strange motion trajectory, a poor composition and deformed video, low resolution, duplicate and ugly, strange body structure, long and strange neck, bad teeth, bad eyes, bad limbs, bad hands, rotating camera, blurry camera, shaking camera. Deformation, low-resolution, blurry, ugly, distortion." ) + negative_prompt_textbox = gr.Textbox(label="Negative prompt (负向提示词)", lines=2, value="Unclear, mutated, deformed, distorted, dark frames, fixed frames, comic book, comic book, small and indistinguishable subject." ) with gr.Row(): with gr.Column(): with gr.Row(): sampler_dropdown = gr.Dropdown(label="Sampling method (采样器种类)", choices=list(scheduler_dict.keys()), value=list(scheduler_dict.keys())[0]) - sample_step_slider = gr.Slider(label="Sampling steps (生成步数)", value=20, minimum=10, maximum=30, step=1, interactive=False) + sample_step_slider = gr.Slider(label="Sampling steps (生成步数)", value=50, minimum=10, maximum=50, step=1, interactive=False) if edition == "v1": width_slider = gr.Slider(label="Width (视频宽度)", value=512, minimum=384, maximum=704, step=32) @@ -1024,25 +1427,17 @@ def ui_modelscope(edition, config_path, model_name, savedir_sample): visible=False, ) length_slider = gr.Slider(label="Animation length (视频帧数)", value=80, minimum=40, maximum=96, step=1) + overlap_video_length = gr.Slider(label="Overlap length (视频续写的重叠帧数)", value=4, minimum=1, maximum=4, step=1, visible=False) + partial_video_length = gr.Slider(label="Partial video generation length (每个部分的视频生成帧数)", value=72, minimum=8, maximum=144, step=8, visible=False) cfg_scale_slider = gr.Slider(label="CFG Scale (引导系数)", value=6.0, minimum=0, maximum=20) else: resize_method = gr.Radio( - ["Generate by", "Resize to the Start Image"], + ["Generate by", "Resize according to Reference"], value="Generate by", show_label=False, - ) - with gr.Column(): - gr.Markdown( - """ - We support video generation up to 720p with 144 frames, but for the trial experience, we have set certain limitations. We fix the max resolution of video to 384x672x48 (2s). - - If the start image you uploaded does not match this resolution, you can use the "Resize to the Start Image" option above. - - If you want to experience longer and larger video generation, you can go to our [Github](https://github.com/aigc-apps/EasyAnimate/). - """ - ) - width_slider = gr.Slider(label="Width (视频宽度)", value=672, minimum=128, maximum=1280, step=16, interactive=False) - height_slider = gr.Slider(label="Height (视频高度)", value=384, minimum=128, maximum=1280, step=16, interactive=False) + ) + width_slider = gr.Slider(label="Width (视频宽度)", value=672, minimum=128, maximum=1344, step=16, interactive=False) + height_slider = gr.Slider(label="Height (视频高度)", value=384, minimum=128, maximum=1344, step=16, interactive=False) base_resolution = gr.Radio(label="Base Resolution of Pretrained Models", value=512, choices=[512, 768, 960], interactive=False, visible=False) with gr.Group(): @@ -1052,16 +1447,26 @@ def ui_modelscope(edition, config_path, model_name, savedir_sample): show_label=False, visible=True, ) - length_slider = gr.Slider(label="Animation length (视频帧数)", value=48, minimum=8, maximum=48, step=8) + if edition in ["v2", "v3", "v4"]: + length_slider = gr.Slider(label="Animation length (视频帧数)", value=144, minimum=8, maximum=144, step=8) + else: + length_slider = gr.Slider(label="Animation length (视频帧数)", value=49, minimum=5, maximum=49, step=4) + overlap_video_length = gr.Slider(label="Overlap length (视频续写的重叠帧数)", value=4, minimum=1, maximum=4, step=1, visible=False) + partial_video_length = gr.Slider(label="Partial video generation length (每个部分的视频生成帧数)", value=72, minimum=8, maximum=144, step=8, visible=False) - with gr.Accordion("Image to Video (图片到视频)", open=True): + source_method = gr.Radio( + ["Text to Video (文本到视频)", "Image to Video (图片到视频)", "Video to Video (视频到视频)", "Video Control (视频控制)"], + value="Text to Video (文本到视频)", + show_label=False, + ) + with gr.Column(visible = False) as image_to_video_col: with gr.Row(): start_image = gr.Image(label="The image at the beginning of the video (图片到视频的开始图片)", show_label=True, elem_id="i2v_start", sources="upload", type="filepath") template_gallery_path = ["asset/1.png", "asset/2.png", "asset/3.png", "asset/4.png", "asset/5.png"] def select_template(evt: gr.SelectData): text = { - "asset/1.png": "The dog is looking at camera and smiling. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", + "asset/1.png": "The dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", "asset/2.png": "a sailboat sailing in rough seas with a dramatic sunset. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", "asset/3.png": "a beautiful woman with long hair and a dress blowing in the wind. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", "asset/4.png": "a man in an astronaut suit playing a guitar. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", @@ -1082,8 +1487,37 @@ def ui_modelscope(edition, config_path, model_name, savedir_sample): with gr.Accordion("The image at the ending of the video (图片到视频的结束图片[非必需, Optional])", open=False): end_image = gr.Image(label="The image at the ending of the video (图片到视频的结束图片[非必需, Optional])", show_label=False, elem_id="i2v_end", sources="upload", type="filepath") + with gr.Column(visible = False) as video_to_video_col: + with gr.Row(): + validation_video = gr.Video( + label="The video to convert (视频转视频的参考视频)", show_label=True, + elem_id="v2v", sources="upload", + ) + with gr.Accordion("The mask of the video to inpaint (视频重新绘制的mask[非必需, Optional])", open=False): + gr.Markdown( + """ + - Please set a larger denoise_strength when using validation_video_mask, such as 1.00 instead of 0.70 + - (请设置更大的denoise_strength，当使用validation_video_mask的时候，比如1而不是0.70) + """ + ) + validation_video_mask = gr.Image( + label="The mask of the video to inpaint (视频重新绘制的mask[非必需, Optional])", + show_label=False, elem_id="v2v_mask", sources="upload", type="filepath" + ) + denoise_strength = gr.Slider(label="Denoise strength (重绘系数)", value=0.70, minimum=0.10, maximum=1.00, step=0.01) + + with gr.Column(visible = False) as control_video_col: + gr.Markdown( + """ + Demo pose control video can be downloaded here [URL](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1.1/pose.mp4). + """ + ) + control_video = gr.Video( + label="The control video (用于提供控制信号的video)", show_label=True, + elem_id="v2v_control", sources="upload", + ) - cfg_scale_slider = gr.Slider(label="CFG Scale (引导系数)", value=7.0, minimum=0, maximum=20) + cfg_scale_slider = gr.Slider(label="CFG Scale (引导系数)", value=6.0, minimum=0, maximum=20) with gr.Row(): seed_textbox = gr.Textbox(label="Seed (随机种子)", value=43) @@ -1106,14 +1540,40 @@ def ui_modelscope(edition, config_path, model_name, savedir_sample): ) def upload_generation_method(generation_method): + if edition == "v1": + f_maximum = 80 + f_value = 80 + elif edition in ["v2", "v3", "v4"]: + f_maximum = 144 + f_value = 144 + else: + f_maximum = 49 + f_value = 49 + if generation_method == "Video Generation": - return gr.update(visible=True, minimum=8, maximum=48, value=48, interactive=True) + return gr.update(visible=True, maximum=f_maximum, value=f_value) elif generation_method == "Image Generation": - return gr.update(minimum=1, maximum=1, value=1, interactive=False) + return gr.update(visible=False) generation_method.change( upload_generation_method, generation_method, [length_slider] ) + def upload_source_method(source_method): + if source_method == "Text to Video (文本到视频)": + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Image to Video (图片到视频)": + return [gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Video to Video (视频到视频)": + return [gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(), gr.update(), gr.update(value=None)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update()] + source_method.change( + upload_source_method, source_method, [ + image_to_video_col, video_to_video_col, control_video_col, start_image, end_image, + validation_video, validation_video_mask, control_video + ] + ) + def upload_resize_method(resize_method): if resize_method == "Generate by": return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] @@ -1141,9 +1601,15 @@ def ui_modelscope(edition, config_path, model_name, savedir_sample): base_resolution, generation_method, length_slider, + overlap_video_length, + partial_video_length, cfg_scale_slider, start_image, end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, seed_textbox, ], outputs=[result_image, result_video, infer_progress] @@ -1157,7 +1623,7 @@ def post_eas( prompt_textbox, negative_prompt_textbox, sampler_dropdown, sample_step_slider, resize_method, width_slider, height_slider, base_resolution, generation_method, length_slider, cfg_scale_slider, - start_image, end_image, seed_textbox, + start_image, end_image, validation_video, validation_video_mask, denoise_strength, seed_textbox, ): if start_image is not None: with open(start_image, 'rb') as file: @@ -1171,6 +1637,19 @@ def post_eas( end_image_encoded_content = base64.b64encode(file_content) end_image = end_image_encoded_content.decode('utf-8') + if validation_video is not None: + with open(validation_video, 'rb') as file: + file_content = file.read() + validation_video_encoded_content = base64.b64encode(file_content) + validation_video = validation_video_encoded_content.decode('utf-8') + + if validation_video_mask is not None: + with open(validation_video_mask, 'rb') as file: + file_content = file.read() + validation_video_mask_encoded_content = base64.b64encode(file_content) + validation_video_mask = validation_video_mask_encoded_content.decode('utf-8') + + datas = { "base_model_path": base_model_dropdown, "motion_module_path": motion_module_dropdown, @@ -1189,6 +1668,9 @@ def post_eas( "cfg_scale_slider": cfg_scale_slider, "start_image": start_image, "end_image": end_image, + "validation_video": validation_video, + "validation_video_mask": validation_video_mask, + "denoise_strength": denoise_strength, "seed_textbox": seed_textbox, } @@ -1226,6 +1708,9 @@ class EasyAnimateController_EAS: cfg_scale_slider, start_image, end_image, + validation_video, + validation_video_mask, + denoise_strength, seed_textbox ): is_image = True if generation_method == "Image Generation" else False @@ -1236,7 +1721,7 @@ class EasyAnimateController_EAS: prompt_textbox, negative_prompt_textbox, sampler_dropdown, sample_step_slider, resize_method, width_slider, height_slider, base_resolution, generation_method, length_slider, cfg_scale_slider, - start_image, end_image, + start_image, end_image, validation_video, validation_video_mask, denoise_strength, seed_textbox ) try: @@ -1321,9 +1806,9 @@ def ui_eas(edition, config_path, model_name, savedir_sample): with gr.Row(): lora_model_dropdown = gr.Dropdown( label="Select LoRA model", - choices=["none", "easyanimatev2_minimalism_lora.safetensors"], + choices=["none"], value="none", - interactive=True, + interactive=False, ) lora_alpha_slider = gr.Slider(label="LoRA alpha (LoRA权重)", value=0.55, minimum=0, maximum=2, interactive=True) @@ -1336,13 +1821,13 @@ def ui_eas(edition, config_path, model_name, savedir_sample): ) prompt_textbox = gr.Textbox(label="Prompt", lines=2, value="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.") - negative_prompt_textbox = gr.Textbox(label="Negative prompt", lines=2, value="The video is not of a high quality, it has a low resolution, and the audio quality is not clear. Strange motion trajectory, a poor composition and deformed video, low resolution, duplicate and ugly, strange body structure, long and strange neck, bad teeth, bad eyes, bad limbs, bad hands, rotating camera, blurry camera, shaking camera. Deformation, low-resolution, blurry, ugly, distortion. " ) + negative_prompt_textbox = gr.Textbox(label="Negative prompt", lines=2, value="Unclear, mutated, deformed, distorted, dark frames, fixed frames, comic book, comic book, small and indistinguishable subject." ) with gr.Row(): with gr.Column(): with gr.Row(): sampler_dropdown = gr.Dropdown(label="Sampling method", choices=list(scheduler_dict.keys()), value=list(scheduler_dict.keys())[0]) - sample_step_slider = gr.Slider(label="Sampling steps", value=20, minimum=10, maximum=30, step=1, interactive=False) + sample_step_slider = gr.Slider(label="Sampling steps", value=40, minimum=10, maximum=40, step=1, interactive=False) if edition == "v1": width_slider = gr.Slider(label="Width", value=512, minimum=384, maximum=704, step=32) @@ -1359,22 +1844,12 @@ def ui_eas(edition, config_path, model_name, savedir_sample): cfg_scale_slider = gr.Slider(label="CFG Scale", value=6.0, minimum=0, maximum=20) else: resize_method = gr.Radio( - ["Generate by", "Resize to the Start Image"], + ["Generate by", "Resize according to Reference"], value="Generate by", show_label=False, - ) - with gr.Column(): - gr.Markdown( - """ - We support video generation up to 720p with 144 frames, but for the trial experience, we have set certain limitations. We fix the max resolution of video to 384x672x48 (2s). - - If the start image you uploaded does not match this resolution, you can use the "Resize to the Start Image" option above. - - If you want to experience longer and larger video generation, you can go to our [Github](https://github.com/aigc-apps/EasyAnimate/). - """ - ) - width_slider = gr.Slider(label="Width (视频宽度)", value=672, minimum=128, maximum=1280, step=16, interactive=False) - height_slider = gr.Slider(label="Height (视频高度)", value=384, minimum=128, maximum=1280, step=16, interactive=False) + ) + width_slider = gr.Slider(label="Width (视频宽度)", value=672, minimum=128, maximum=1344, step=16, interactive=False) + height_slider = gr.Slider(label="Height (视频高度)", value=384, minimum=128, maximum=1344, step=16, interactive=False) base_resolution = gr.Radio(label="Base Resolution of Pretrained Models", value=512, choices=[512, 768, 960], interactive=False, visible=False) with gr.Group(): @@ -1384,15 +1859,23 @@ def ui_eas(edition, config_path, model_name, savedir_sample): show_label=False, visible=True, ) - length_slider = gr.Slider(label="Animation length (视频帧数)", value=48, minimum=8, maximum=48, step=8) + if edition in ["v2", "v3", "v4"]: + length_slider = gr.Slider(label="Animation length (视频帧数)", value=144, minimum=8, maximum=144, step=8) + else: + length_slider = gr.Slider(label="Animation length (视频帧数)", value=21, minimum=5, maximum=21, step=4) - with gr.Accordion("Image to Video", open=True): + source_method = gr.Radio( + ["Text to Video (文本到视频)", "Image to Video (图片到视频)", "Video to Video (视频到视频)"], + value="Text to Video (文本到视频)", + show_label=False, + ) + with gr.Column(visible = False) as image_to_video_col: start_image = gr.Image(label="The image at the beginning of the video", show_label=True, elem_id="i2v_start", sources="upload", type="filepath") template_gallery_path = ["asset/1.png", "asset/2.png", "asset/3.png", "asset/4.png", "asset/5.png"] def select_template(evt: gr.SelectData): text = { - "asset/1.png": "The dog is looking at camera and smiling. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", + "asset/1.png": "The dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", "asset/2.png": "a sailboat sailing in rough seas with a dramatic sunset. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", "asset/3.png": "a beautiful woman with long hair and a dress blowing in the wind. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", "asset/4.png": "a man in an astronaut suit playing a guitar. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", @@ -1413,8 +1896,27 @@ def ui_eas(edition, config_path, model_name, savedir_sample): with gr.Accordion("The image at the ending of the video (Optional)", open=False): end_image = gr.Image(label="The image at the ending of the video (Optional)", show_label=True, elem_id="i2v_end", sources="upload", type="filepath") - cfg_scale_slider = gr.Slider(label="CFG Scale", value=7.0, minimum=0, maximum=20) - + with gr.Column(visible = False) as video_to_video_col: + with gr.Row(): + validation_video = gr.Video( + label="The video to convert (视频转视频的参考视频)", show_label=True, + elem_id="v2v", sources="upload", + ) + with gr.Accordion("The mask of the video to inpaint (视频重新绘制的mask[非必需, Optional])", open=False): + gr.Markdown( + """ + - Please set a larger denoise_strength when using validation_video_mask, such as 1.00 instead of 0.70 + - (请设置更大的denoise_strength，当使用validation_video_mask的时候，比如1而不是0.70) + """ + ) + validation_video_mask = gr.Image( + label="The mask of the video to inpaint (视频重新绘制的mask[非必需, Optional])", + show_label=False, elem_id="v2v_mask", sources="upload", type="filepath" + ) + denoise_strength = gr.Slider(label="Denoise strength (重绘��数)", value=0.70, minimum=0.10, maximum=1.00, step=0.01) + + cfg_scale_slider = gr.Slider(label="CFG Scale (引导系数)", value=6.0, minimum=0, maximum=20) + with gr.Row(): seed_textbox = gr.Textbox(label="Seed", value=43) seed_button = gr.Button(value="\U0001F3B2", elem_classes="toolbutton") @@ -1436,14 +1938,35 @@ def ui_eas(edition, config_path, model_name, savedir_sample): ) def upload_generation_method(generation_method): + if edition == "v1": + f_maximum = 80 + f_value = 80 + elif edition in ["v2", "v3", "v4"]: + f_maximum = 144 + f_value = 144 + else: + f_maximum = 49 + f_value = 49 + if generation_method == "Video Generation": - return gr.update(visible=True, minimum=8, maximum=48, value=48, interactive=True) + return gr.update(visible=True, maximum=f_maximum, value=f_value) elif generation_method == "Image Generation": - return gr.update(minimum=1, maximum=1, value=1, interactive=False) + return gr.update(visible=False) generation_method.change( upload_generation_method, generation_method, [length_slider] ) + def upload_source_method(source_method): + if source_method == "Text to Video (文本到视频)": + return [gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Image to Video (图片到视频)": + return [gr.update(visible=True), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None)] + else: + return [gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(), gr.update()] + source_method.change( + upload_source_method, source_method, [image_to_video_col, video_to_video_col, start_image, end_image, validation_video, validation_video_mask] + ) + def upload_resize_method(resize_method): if resize_method == "Generate by": return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] @@ -1474,6 +1997,9 @@ def ui_eas(edition, config_path, model_name, savedir_sample): cfg_scale_slider, start_image, end_image, + validation_video, + validation_video_mask, + denoise_strength, seed_textbox, ], outputs=[result_image, result_video, infer_progress]