Spaces:
Running
Running
File size: 2,785 Bytes
a174f44 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 |
import torch
from diffusers import AnimateDiffSparseControlNetPipeline
from diffusers.models import AutoencoderKL, MotionAdapter, SparseControlNetModel
from diffusers.schedulers import DPMSolverMultistepScheduler
from diffusers.utils import export_to_gif, load_image
torch.backends.cuda.matmul.allow_tf32 = True # Enable TF32 for speed
device = "cuda"
dtype = torch.float16
# Model IDs
model_id = "SG161222/Realistic_Vision_V5.1_noVAE"
motion_adapter_id = "guoyww/animatediff-motion-adapter-v1-5-3"
controlnet_id = "guoyww/animatediff-sparsectrl-scribble"
lora_adapter_id = "guoyww/animatediff-motion-lora-v1-5-3"
vae_id = "stabilityai/sd-vae-ft-mse"
# Load models to device once
motion_adapter = MotionAdapter.from_pretrained(motion_adapter_id, torch_dtype=dtype, device_map="auto")
controlnet = SparseControlNetModel.from_pretrained(controlnet_id, torch_dtype=dtype, device_map="auto")
vae = AutoencoderKL.from_pretrained(vae_id, torch_dtype=dtype, device_map="auto")
# Use DPMSolverMultistepScheduler with optimizations
scheduler = DPMSolverMultistepScheduler.from_pretrained(
model_id, subfolder="scheduler", beta_schedule="linear",
algorithm_type="dpmsolver++", use_karras_sigmas=True,
)
pipe = AnimateDiffSparseControlNetPipeline.from_pretrained(
model_id, motion_adapter=motion_adapter, controlnet=controlnet,
vae=vae, scheduler=scheduler, torch_dtype=dtype,
).to(device)
# Enable memory optimizations
pipe.enable_xformers_memory_efficient_attention()
pipe.load_lora_weights(lora_adapter_id, adapter_name="motion_lora")
pipe.fuse_lora(lora_scale=1.0)
# Preload conditioning frames
image_files = [
"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/animatediff-scribble-1.png",
"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/animatediff-scribble-2.png",
"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/animatediff-scribble-3.png"
]
condition_frame_indices = [0, 8, 15]
conditioning_frames = [load_image(img) for img in image_files]
# Generator for reproducibility
generator = torch.Generator(device).manual_seed(1337)
# Inference with memory optimizations
with torch.inference_mode():
video = pipe(
prompt="an aerial view of a cyberpunk city, night time, neon lights, masterpiece, high quality",
negative_prompt="low quality, worst quality, letterboxed",
num_inference_steps=25,
conditioning_frames=conditioning_frames,
controlnet_conditioning_scale=1.0,
controlnet_frame_indices=condition_frame_indices,
generator=generator,
).frames[0]
export_to_gif(video, "output.gif")
# Free memory
del pipe, motion_adapter, controlnet, vae
torch.cuda.empty_cache()
|