svjack/Genshin_Impact_Yumemizuki_Mizuki_HunyuanVideo_lora

Installtion

sudo apt-get update && sudo apt-get install ffmpeg git-lfs 
pip install torch torchvision diffusers transformers moviepy==1.0.3 peft safetensors
git clone https://huggingface.co/svjack/Genshin_Impact_Yumemizuki_Mizuki_HunyuanVideo_lora

Inference

import torch
from diffusers import HunyuanVideoPipeline, HunyuanVideoTransformer3DModel
from diffusers.utils import export_to_video
from safetensors.torch import load_file
import os

def infer_video(
    pretrained_model,
    prompt,
    height,
    width,
    num_frames,
    num_inference_steps,
    seed,
    output_dir,
    use_lora=False,
    lora_path=None,
    alpha=None,
):
    """
    合并使用和不使用 LoRA 的视频生成函数。

    参数:
        pretrained_model (str): 预训练模型的路径。
        prompt (str): 生成视频的提示词。
        height (int): 生成视频的高度。
        width (int): 生成视频的宽度。
        num_frames (int): 生成视频的帧数。
        num_inference_steps (int): 推断步数。
        seed (int): 随机种子。
        output_dir (str): 输出视频的目录。
        use_lora (bool): 是否使用 LoRA，默认为 False。
        lora_path (str): LoRA 文件的路径，仅在 use_lora=True 时有效。
        alpha (int): LoRA 的 alpha 参数，仅在 use_lora=True 时有效。
    """
    # 加载模型
    transformer = HunyuanVideoTransformer3DModel.from_pretrained(
        pretrained_model,
        subfolder="transformer",
        torch_dtype=torch.bfloat16,
    )
    # 如果使用 LoRA
    if use_lora:
        if lora_path is None:
            raise ValueError("lora_path must be provided when use_lora is True")

        # 加载 LoRA 权重
        lora_sd = load_file(lora_path)
        rank = 0
        for key in lora_sd.keys():
            if ".lora_A.weight" in key:
                rank = lora_sd[key].shape[0]

        alpha = 1 if alpha is None else alpha
        lora_weight = alpha / rank

        print(f"lora rank = {rank}")
        print(f"alpha = {alpha}")
        print(f"lora weight = {lora_weight}")

        # 应用 LoRA
        transformer.load_lora_adapter(lora_sd, adapter_name="default_lora")
        transformer.set_adapters(adapter_names="default_lora", weights=lora_weight)

    pipe = HunyuanVideoPipeline.from_pretrained(pretrained_model, transformer=transformer, torch_dtype=torch.float16)
    pipe.transformer = transformer
    
    pipe.vae.enable_tiling(
        tile_sample_min_height=256,
        tile_sample_min_width=256,
        tile_sample_min_num_frames=64,
        tile_sample_stride_height=192,
        tile_sample_stride_width=192,
        tile_sample_stride_num_frames=16,
    )
    pipe.enable_sequential_cpu_offload()

    # 进行推断
    output = pipe(
        prompt=prompt,
        height=height,
        width=width,
        num_frames=num_frames,
        num_inference_steps=num_inference_steps,
        generator=torch.Generator(device="cpu").manual_seed(seed),
    ).frames[0]

    # 导出视频
    output_filename = "output_lora.mp4" if use_lora else "output_base.mp4"
    export_to_video(
        output,
        os.path.join(output_dir, output_filename),
        fps=15,
    )

infer_video(
    pretrained_model="hunyuanvideo-community/HunyuanVideo",
    prompt="In the style of Yumemizuki Mizuki, the character stands under moonlight in a serene courtyard, softly illuminated. Translucent curtains sway as she blinks slowly, her eyes open reflecting the moon. Ancient lanterns, moss, and distant cherry blossoms surround her. Her pure, haunting voice blends with the tranquil night, crafting dreamlike serenity. Moonlight weaves through the scene, harmonizing with her melody, creating timeless enchantment.",
    height=512,
    width=512,
    num_frames=33,
    num_inference_steps=20,
    seed=42,
    output_dir="./",
    use_lora=True,
    lora_path="Genshin_Impact_Yumemizuki_Mizuki_HunyuanVideo_lora/checkpoints/hyv-lora-00000600.safetensors",
    alpha=16,
)

infer_video(
    pretrained_model="hunyuanvideo-community/HunyuanVideo",
    prompt="In the style of Yumemizuki Mizuki, the character sits in a hot spring under daylight, wearing a swimsuit, softly illuminated. Steam rises gently as she blinks slowly, her eyes open reflecting the sunlight. Ancient rocks, moss, and distant trees surround her. Her pure, soothing voice blends with the warm air, crafting dreamlike tranquility. Sunlight weaves through the scene, harmonizing with her melody, creating timeless relaxation.",
    height=512,
    width=512,
    num_frames=33,
    num_inference_steps=20,
    seed=42,
    output_dir="./",
    use_lora=True,
    lora_path="Genshin_Impact_Yumemizuki_Mizuki_HunyuanVideo_lora/checkpoints/hyv-lora-00000600.safetensors",
    alpha=16,
)

Demo

Some Lora samples in checkpoints

Yumemizuki Mizuki Moon

Yumemizuki Mizuki Hot Spring

Demo

Some Lora samples in video_checkpoints

Yumemizuki Mizuki Moon

Yumemizuki Mizuki Hot Spring