wangfuyun commited on
Commit
992a789
1 Parent(s): c9a6a5b

Upload 54 files

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +2 -0
  2. app.py +256 -0
  3. enviroment.yaml +125 -0
  4. gradio_cached_examples/19/component 0/3725061b1c373489a048/000003.mp4 +0 -0
  5. gradio_cached_examples/19/component 0/a21547779ff20817de06/000002.mp4 +0 -0
  6. gradio_cached_examples/19/component 0/ab669c2acaeb6f957c50/000001.mp4 +0 -0
  7. gradio_cached_examples/19/component 0/ceca750cda163ac6f548/000000.mp4 +0 -0
  8. gradio_cached_examples/19/log.csv +5 -0
  9. lcm_scheduler.py +468 -0
  10. outputs_gradio/000000.mp4 +0 -0
  11. outputs_gradio/000001.mp4 +0 -0
  12. outputs_gradio/000002.mp4 +0 -0
  13. outputs_gradio/000003.mp4 +0 -0
  14. outputs_gradio/000004.mp4 +0 -0
  15. outputs_gradio/000005.mp4 +0 -0
  16. pipeline.py +711 -0
  17. requirements.txt +4 -0
  18. safetensors/AnimateLCM-SVD-xt-1.1.safetensors +3 -0
  19. safetensors/AnimateLCM-SVD-xt.safetensors +3 -0
  20. test_imgs/ai-generated-8255456_1280.png +3 -0
  21. test_imgs/ai-generated-8411866_1280.jpg +0 -0
  22. test_imgs/ai-generated-8463496_1280.jpg +0 -0
  23. test_imgs/ai-generated-8476858_1280.png +0 -0
  24. test_imgs/ai-generated-8479572_1280.jpg +0 -0
  25. test_imgs/ai-generated-8481641_1280.jpg +0 -0
  26. test_imgs/ai-generated-8489879_1280.png +3 -0
  27. test_imgs/ai-generated-8496135_1280.jpg +0 -0
  28. test_imgs/ai-generated-8496952_1280.jpg +0 -0
  29. test_imgs/ai-generated-8498844_1280.jpg +0 -0
  30. test_imgs/bird-7411270_1280.jpg +0 -0
  31. test_imgs/bird-7586857_1280.jpg +0 -0
  32. test_imgs/bird-8014191_1280.jpg +0 -0
  33. test_imgs/couple-8019370_1280.jpg +0 -0
  34. test_imgs/cupcakes-380178_1280.jpg +0 -0
  35. test_imgs/dog-7330712_1280.jpg +0 -0
  36. test_imgs/dog-7396912_1280.jpg +0 -0
  37. test_imgs/girl-4898696_1280.jpg +0 -0
  38. test_imgs/grey-capped-flycatcher-8071233_1280.jpg +0 -0
  39. test_imgs/halloween-4585684_1280.jpg +0 -0
  40. test_imgs/leaf-7260246_1280.jpg +0 -0
  41. test_imgs/meerkat-7465819_1280.jpg +0 -0
  42. test_imgs/mobile-phone-1875813_1280.jpg +0 -0
  43. test_imgs/mother-8097324_1280.jpg +0 -0
  44. test_imgs/plane-8145957_1280.jpg +0 -0
  45. test_imgs/power-station-6579092_1280.jpg +0 -0
  46. test_imgs/ship-7833921_1280.jpg +0 -0
  47. test_imgs/sleep-7871915_1280.jpg +0 -0
  48. test_imgs/squirrel-7985502_1280.jpg +0 -0
  49. test_imgs/squirrel-8211238_1280.jpg +0 -0
  50. test_imgs/training-8122941_1280.jpg +0 -0
.gitattributes CHANGED
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ test_imgs/ai-generated-8255456_1280.png filter=lfs diff=lfs merge=lfs -text
37
+ test_imgs/ai-generated-8489879_1280.png filter=lfs diff=lfs merge=lfs -text
app.py ADDED
@@ -0,0 +1,256 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+
3
+ # import gradio.helpers
4
+ import torch
5
+ import os
6
+ from glob import glob
7
+ from pathlib import Path
8
+ from typing import Optional
9
+
10
+ from PIL import Image
11
+ from diffusers.utils import load_image, export_to_video
12
+ from pipeline import StableVideoDiffusionPipeline
13
+
14
+ import random
15
+ from safetensors import safe_open
16
+ from lcm_scheduler import AnimateLCMSVDStochasticIterativeScheduler
17
+
18
+
19
+ def get_safetensors_files():
20
+ models_dir = "./safetensors"
21
+ safetensors_files = [
22
+ f for f in os.listdir(models_dir) if f.endswith(".safetensors")
23
+ ]
24
+ return safetensors_files
25
+
26
+
27
+ def model_select(selected_file):
28
+ print("load model weights", selected_file)
29
+ pipe.unet.cpu()
30
+ file_path = os.path.join("./safetensors", selected_file)
31
+ state_dict = {}
32
+ with safe_open(file_path, framework="pt", device="cpu") as f:
33
+ for key in f.keys():
34
+ state_dict[key] = f.get_tensor(key)
35
+ missing, unexpected = pipe.unet.load_state_dict(state_dict, strict=True)
36
+ pipe.unet.cuda()
37
+ del state_dict
38
+ return
39
+
40
+
41
+ noise_scheduler = AnimateLCMSVDStochasticIterativeScheduler(
42
+ num_train_timesteps=40,
43
+ sigma_min=0.002,
44
+ sigma_max=700.0,
45
+ sigma_data=1.0,
46
+ s_noise=1.0,
47
+ rho=7,
48
+ clip_denoised=False,
49
+ )
50
+ pipe = StableVideoDiffusionPipeline.from_pretrained(
51
+ "stabilityai/stable-video-diffusion-img2vid-xt",
52
+ scheduler=noise_scheduler,
53
+ torch_dtype=torch.float16,
54
+ variant="fp16",
55
+ )
56
+ pipe.to("cuda")
57
+ pipe.enable_model_cpu_offload() # for smaller cost
58
+ model_select("AnimateLCM-SVD-xt.safetensors")
59
+ # pipe.unet = torch.compile(pipe.unet, mode="reduce-overhead", fullgraph=True) # for faster inference
60
+
61
+
62
+ max_64_bit_int = 2**63 - 1
63
+
64
+
65
+ def sample(
66
+ image: Image,
67
+ seed: Optional[int] = 42,
68
+ randomize_seed: bool = False,
69
+ motion_bucket_id: int = 80,
70
+ fps_id: int = 8,
71
+ max_guidance_scale: float = 1.2,
72
+ min_guidance_scale: float = 1,
73
+ width: int = 1024,
74
+ height: int = 576,
75
+ num_inference_steps: int = 4,
76
+ decoding_t: int = 4, # Number of frames decoded at a time! This eats most VRAM. Reduce if necessary.
77
+ output_folder: str = "outputs_gradio",
78
+ ):
79
+ if image.mode == "RGBA":
80
+ image = image.convert("RGB")
81
+
82
+ if randomize_seed:
83
+ seed = random.randint(0, max_64_bit_int)
84
+ generator = torch.manual_seed(seed)
85
+
86
+ os.makedirs(output_folder, exist_ok=True)
87
+ base_count = len(glob(os.path.join(output_folder, "*.mp4")))
88
+ video_path = os.path.join(output_folder, f"{base_count:06d}.mp4")
89
+
90
+ with torch.autocast("cuda"):
91
+ frames = pipe(
92
+ image,
93
+ decode_chunk_size=decoding_t,
94
+ generator=generator,
95
+ motion_bucket_id=motion_bucket_id,
96
+ height=height,
97
+ width=width,
98
+ num_inference_steps=num_inference_steps,
99
+ min_guidance_scale=min_guidance_scale,
100
+ max_guidance_scale=max_guidance_scale,
101
+ ).frames[0]
102
+ export_to_video(frames, video_path, fps=fps_id)
103
+ torch.manual_seed(seed)
104
+
105
+ return video_path, seed
106
+
107
+
108
+ def resize_image(image, output_size=(1024, 576)):
109
+ # Calculate aspect ratios
110
+ target_aspect = output_size[0] / output_size[1] # Aspect ratio of the desired size
111
+ image_aspect = image.width / image.height # Aspect ratio of the original image
112
+
113
+ # Resize then crop if the original image is larger
114
+ if image_aspect > target_aspect:
115
+ # Resize the image to match the target height, maintaining aspect ratio
116
+ new_height = output_size[1]
117
+ new_width = int(new_height * image_aspect)
118
+ resized_image = image.resize((new_width, new_height), Image.LANCZOS)
119
+ # Calculate coordinates for cropping
120
+ left = (new_width - output_size[0]) / 2
121
+ top = 0
122
+ right = (new_width + output_size[0]) / 2
123
+ bottom = output_size[1]
124
+ else:
125
+ # Resize the image to match the target width, maintaining aspect ratio
126
+ new_width = output_size[0]
127
+ new_height = int(new_width / image_aspect)
128
+ resized_image = image.resize((new_width, new_height), Image.LANCZOS)
129
+ # Calculate coordinates for cropping
130
+ left = 0
131
+ top = (new_height - output_size[1]) / 2
132
+ right = output_size[0]
133
+ bottom = (new_height + output_size[1]) / 2
134
+
135
+ # Crop the image
136
+ cropped_image = resized_image.crop((left, top, right, bottom))
137
+ return cropped_image
138
+
139
+
140
+ with gr.Blocks() as demo:
141
+ gr.Markdown(
142
+ """
143
+ # [AnimateLCM: Accelerating the Animation of Personalized Diffusion Models and Adapters with Decoupled Consistency Learning](https://arxiv.org/abs/2402.00769)
144
+ Fu-Yun Wang, Zhaoyang Huang (*Corresponding Author), Xiaoyu Shi, Weikang Bian, Guanglu Song, Yu Liu, Hongsheng Li (*Corresponding Author)<br>
145
+ [arXiv Report](https://arxiv.org/abs/2402.00769) | [Project Page](https://animatelcm.github.io/) | [Github](https://github.com/G-U-N/AnimateLCM) | [Civitai](https://civitai.com/models/290375/animatelcm-fast-video-generation) | [Replicate](https://replicate.com/camenduru/animate-lcm)
146
+ Related Models:
147
+ [AnimateLCM-t2v](https://huggingface.co/wangfuyun/AnimateLCM): Personalized Text-to-Video Generation
148
+ [AnimateLCM-SVD-xt](https://huggingface.co/wangfuyun/AnimateLCM-SVD-xt): General Image-to-Video Generation
149
+ [AnimateLCM-i2v](https://huggingface.co/wangfuyun/AnimateLCM-I2V): Personalized Image-to-Video Generation
150
+ """
151
+ )
152
+ with gr.Row():
153
+ with gr.Column():
154
+ image = gr.Image(label="Upload your image", type="pil")
155
+ generate_btn = gr.Button("Generate")
156
+ video = gr.Video()
157
+ with gr.Accordion("Advanced options", open=False):
158
+ safetensors_dropdown = gr.Dropdown(
159
+ label="Choose Safetensors", choices=get_safetensors_files()
160
+ )
161
+ seed = gr.Slider(
162
+ label="Seed",
163
+ value=42,
164
+ randomize=False,
165
+ minimum=0,
166
+ maximum=max_64_bit_int,
167
+ step=1,
168
+ )
169
+ randomize_seed = gr.Checkbox(label="Randomize seed", value=False)
170
+ motion_bucket_id = gr.Slider(
171
+ label="Motion bucket id",
172
+ info="Controls how much motion to add/remove from the image",
173
+ value=80,
174
+ minimum=1,
175
+ maximum=255,
176
+ )
177
+ fps_id = gr.Slider(
178
+ label="Frames per second",
179
+ info="The length of your video in seconds will be 25/fps",
180
+ value=8,
181
+ minimum=5,
182
+ maximum=30,
183
+ )
184
+ width = gr.Slider(
185
+ label="Width of input image",
186
+ info="It should be divisible by 64",
187
+ value=1024,
188
+ minimum=576,
189
+ maximum=2048,
190
+ )
191
+ height = gr.Slider(
192
+ label="Height of input image",
193
+ info="It should be divisible by 64",
194
+ value=576,
195
+ minimum=320,
196
+ maximum=1152,
197
+ )
198
+ max_guidance_scale = gr.Slider(
199
+ label="Max guidance scale",
200
+ info="classifier-free guidance strength",
201
+ value=1.2,
202
+ minimum=1,
203
+ maximum=2,
204
+ )
205
+ min_guidance_scale = gr.Slider(
206
+ label="Min guidance scale",
207
+ info="classifier-free guidance strength",
208
+ value=1,
209
+ minimum=1,
210
+ maximum=1.5,
211
+ )
212
+ num_inference_steps = gr.Slider(
213
+ label="Num inference steps",
214
+ info="steps for inference",
215
+ value=4,
216
+ minimum=1,
217
+ maximum=20,
218
+ step=1,
219
+ )
220
+
221
+ image.upload(fn=resize_image, inputs=image, outputs=image, queue=False)
222
+ generate_btn.click(
223
+ fn=sample,
224
+ inputs=[
225
+ image,
226
+ seed,
227
+ randomize_seed,
228
+ motion_bucket_id,
229
+ fps_id,
230
+ max_guidance_scale,
231
+ min_guidance_scale,
232
+ width,
233
+ height,
234
+ num_inference_steps,
235
+ ],
236
+ outputs=[video, seed],
237
+ api_name="video",
238
+ )
239
+ safetensors_dropdown.change(fn=model_select, inputs=safetensors_dropdown)
240
+
241
+ gr.Examples(
242
+ examples=[
243
+ "test_imgs/ai-generated-8255456_1280.png",
244
+ "test_imgs/ai-generated-8496135_1280.jpg",
245
+ "test_imgs/dog-7396912_1280.jpg",
246
+ "test_imgs/ship-7833921_1280.jpg",
247
+ ],
248
+ inputs=image,
249
+ outputs=[video, seed],
250
+ fn=sample,
251
+ cache_examples=True,
252
+ )
253
+
254
+ if __name__ == "__main__":
255
+ demo.queue(max_size=20, api_open=False)
256
+ demo.launch(share=True, show_api=False)
enviroment.yaml ADDED
@@ -0,0 +1,125 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: svd
2
+ channels:
3
+ - defaults
4
+ dependencies:
5
+ - _libgcc_mutex=0.1=main
6
+ - _openmp_mutex=5.1=1_gnu
7
+ - ca-certificates=2023.12.12=h06a4308_0
8
+ - ld_impl_linux-64=2.38=h1181459_1
9
+ - libffi=3.4.4=h6a678d5_0
10
+ - libgcc-ng=11.2.0=h1234567_1
11
+ - libgomp=11.2.0=h1234567_1
12
+ - libstdcxx-ng=11.2.0=h1234567_1
13
+ - ncurses=6.4=h6a678d5_0
14
+ - openssl=3.0.12=h7f8727e_0
15
+ - pip=23.3.1=py39h06a4308_0
16
+ - python=3.9.18=h955ad1f_0
17
+ - readline=8.2=h5eee18b_0
18
+ - setuptools=68.2.2=py39h06a4308_0
19
+ - sqlite=3.41.2=h5eee18b_0
20
+ - tk=8.6.12=h1ccaba5_0
21
+ - wheel=0.41.2=py39h06a4308_0
22
+ - xz=5.4.5=h5eee18b_0
23
+ - zlib=1.2.13=h5eee18b_0
24
+ - pip:
25
+ - accelerate==0.26.1
26
+ - albumentations==1.3.1
27
+ - antlr4-python3-runtime==4.9.3
28
+ - appdirs==1.4.4
29
+ - bitsandbytes==0.42.0
30
+ - braceexpand==0.1.7
31
+ - brotli==1.1.0
32
+ - certifi==2023.11.17
33
+ - cffi==1.16.0
34
+ - charset-normalizer==3.3.2
35
+ - click==8.1.7
36
+ - dataclasses==0.6
37
+ - decord==0.6.0
38
+ - diffusers==0.25.1
39
+ - docker-pycreds==0.4.0
40
+ - docopt==0.6.2
41
+ - einops==0.7.0
42
+ - exifread-nocycle==3.0.1
43
+ - ffmpeg-python==0.2.0
44
+ - filelock==3.13.1
45
+ - fire==0.5.0
46
+ - fsspec==2023.12.2
47
+ - future==0.18.3
48
+ - gitdb==4.0.11
49
+ - gitpython==3.1.41
50
+ - huggingface-hub==0.20.3
51
+ - idna==3.6
52
+ - imageio==2.33.1
53
+ - img2dataset==1.45.0
54
+ - importlib-metadata==7.0.1
55
+ - jinja2==3.1.3
56
+ - joblib==1.3.2
57
+ - langdetect==1.0.9
58
+ - lazy-loader==0.3
59
+ - markupsafe==2.1.4
60
+ - mpmath==1.3.0
61
+ - mutagen==1.47.0
62
+ - networkx==3.2.1
63
+ - numpy==1.26.3
64
+ - nvidia-cublas-cu12==12.1.3.1
65
+ - nvidia-cuda-cupti-cu12==12.1.105
66
+ - nvidia-cuda-nvrtc-cu12==12.1.105
67
+ - nvidia-cuda-runtime-cu12==12.1.105
68
+ - nvidia-cudnn-cu12==8.9.2.26
69
+ - nvidia-cufft-cu12==11.0.2.54
70
+ - nvidia-curand-cu12==10.3.2.106
71
+ - nvidia-cusolver-cu12==11.4.5.107
72
+ - nvidia-cusparse-cu12==12.1.0.106
73
+ - nvidia-nccl-cu12==2.19.3
74
+ - nvidia-nvjitlink-cu12==12.3.101
75
+ - nvidia-nvtx-cu12==12.1.105
76
+ - omegaconf==2.3.0
77
+ - opencv-python==4.9.0.80
78
+ - opencv-python-headless==4.9.0.80
79
+ - packaging==23.2
80
+ - pandas==2.2.0
81
+ - pillow==10.2.0
82
+ - platformdirs==4.1.0
83
+ - protobuf==4.25.2
84
+ - psutil==5.9.8
85
+ - pyarrow==15.0.0
86
+ - pycparser==2.21
87
+ - pycryptodomex==3.20.0
88
+ - python-dateutil==2.8.2
89
+ - pytz==2023.3.post1
90
+ - pyyaml==6.0.1
91
+ - qudida==0.0.4
92
+ - regex==2023.12.25
93
+ - requests==2.31.0
94
+ - safetensors==0.4.2
95
+ - scenedetect==0.6.2
96
+ - scikit-image==0.22.0
97
+ - scikit-learn==1.4.0
98
+ - scipy==1.12.0
99
+ - sentry-sdk==1.39.2
100
+ - setproctitle==1.3.3
101
+ - six==1.16.0
102
+ - smmap==5.0.1
103
+ - soundfile==0.12.1
104
+ - sympy==1.12
105
+ - termcolor==2.4.0
106
+ - threadpoolctl==3.2.0
107
+ - tifffile==2023.12.9
108
+ - timeout-decorator==0.5.0
109
+ - tokenizers==0.15.1
110
+ - torch==2.2.0
111
+ - torchdata==0.7.1
112
+ - torchvision==0.17.0
113
+ - tqdm==4.66.1
114
+ - transformers==4.37.0
115
+ - triton==2.2.0
116
+ - typing-extensions==4.9.0
117
+ - tzdata==2023.4
118
+ - urllib3==2.1.0
119
+ - wandb==0.16.2
120
+ - webdataset==0.2.86
121
+ - websockets==12.0
122
+ - webvtt-py==0.4.6
123
+ - xformers==0.0.24
124
+ - yt-dlp==2023.12.30
125
+ - zipp==3.17.0
gradio_cached_examples/19/component 0/3725061b1c373489a048/000003.mp4 ADDED
Binary file (734 kB). View file
 
gradio_cached_examples/19/component 0/a21547779ff20817de06/000002.mp4 ADDED
Binary file (393 kB). View file
 
gradio_cached_examples/19/component 0/ab669c2acaeb6f957c50/000001.mp4 ADDED
Binary file (848 kB). View file
 
gradio_cached_examples/19/component 0/ceca750cda163ac6f548/000000.mp4 ADDED
Binary file (641 kB). View file
 
gradio_cached_examples/19/log.csv ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ component 0,Seed,flag,username,timestamp
2
+ "{""video"":{""path"":""gradio_cached_examples/19/component 0/ceca750cda163ac6f548/000000.mp4"",""url"":""/file=/tmp/gradio/2879d2ca96d58c8931b302213e276f4955b51afa/000000.mp4"",""size"":null,""orig_name"":""000000.mp4"",""mime_type"":null,""is_stream"":false},""subtitles"":null}",42,,,2024-02-25 17:49:43.926703
3
+ "{""video"":{""path"":""gradio_cached_examples/19/component 0/ab669c2acaeb6f957c50/000001.mp4"",""url"":""/file=/tmp/gradio/3fe5de118a0bc4b9e389758b5bfb2a9682e9ec4f/000001.mp4"",""size"":null,""orig_name"":""000001.mp4"",""mime_type"":null,""is_stream"":false},""subtitles"":null}",42,,,2024-02-25 17:50:17.506490
4
+ "{""video"":{""path"":""gradio_cached_examples/19/component 0/a21547779ff20817de06/000002.mp4"",""url"":""/file=/tmp/gradio/7bb404b88df0715738e14aa3e0d5d6975d90ad87/000002.mp4"",""size"":null,""orig_name"":""000002.mp4"",""mime_type"":null,""is_stream"":false},""subtitles"":null}",42,,,2024-02-25 17:50:51.099873
5
+ "{""video"":{""path"":""gradio_cached_examples/19/component 0/3725061b1c373489a048/000003.mp4"",""url"":""/file=/tmp/gradio/515c7849b4b9b1de0fdded51f77517bd15f92734/000003.mp4"",""size"":null,""orig_name"":""000003.mp4"",""mime_type"":null,""is_stream"":false},""subtitles"":null}",42,,,2024-02-25 17:51:24.329419
lcm_scheduler.py ADDED
@@ -0,0 +1,468 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2023 The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from dataclasses import dataclass
16
+ from typing import List, Optional, Tuple, Union
17
+
18
+ import numpy as np
19
+ import torch
20
+
21
+ from diffusers.configuration_utils import ConfigMixin, register_to_config
22
+ from diffusers.utils import BaseOutput, logging
23
+ from diffusers.utils.torch_utils import randn_tensor
24
+ from diffusers.schedulers.scheduling_utils import SchedulerMixin
25
+
26
+
27
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
28
+
29
+
30
+ @dataclass
31
+ class AnimateLCMSVDStochasticIterativeSchedulerOutput(BaseOutput):
32
+ """
33
+ Output class for the scheduler's `step` function.
34
+
35
+ Args:
36
+ prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images):
37
+ Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the
38
+ denoising loop.
39
+ """
40
+
41
+ prev_sample: torch.FloatTensor
42
+
43
+
44
+ class AnimateLCMSVDStochasticIterativeScheduler(SchedulerMixin, ConfigMixin):
45
+ """
46
+ Multistep and onestep sampling for consistency models.
47
+
48
+ This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic
49
+ methods the library implements for all schedulers such as loading and saving.
50
+
51
+ Args:
52
+ num_train_timesteps (`int`, defaults to 40):
53
+ The number of diffusion steps to train the model.
54
+ sigma_min (`float`, defaults to 0.002):
55
+ Minimum noise magnitude in the sigma schedule. Defaults to 0.002 from the original implementation.
56
+ sigma_max (`float`, defaults to 80.0):
57
+ Maximum noise magnitude in the sigma schedule. Defaults to 80.0 from the original implementation.
58
+ sigma_data (`float`, defaults to 0.5):
59
+ The standard deviation of the data distribution from the EDM
60
+ [paper](https://huggingface.co/papers/2206.00364). Defaults to 0.5 from the original implementation.
61
+ s_noise (`float`, defaults to 1.0):
62
+ The amount of additional noise to counteract loss of detail during sampling. A reasonable range is [1.000,
63
+ 1.011]. Defaults to 1.0 from the original implementation.
64
+ rho (`float`, defaults to 7.0):
65
+ The parameter for calculating the Karras sigma schedule from the EDM
66
+ [paper](https://huggingface.co/papers/2206.00364). Defaults to 7.0 from the original implementation.
67
+ clip_denoised (`bool`, defaults to `True`):
68
+ Whether to clip the denoised outputs to `(-1, 1)`.
69
+ timesteps (`List` or `np.ndarray` or `torch.Tensor`, *optional*):
70
+ An explicit timestep schedule that can be optionally specified. The timesteps are expected to be in
71
+ increasing order.
72
+ """
73
+
74
+ order = 1
75
+
76
+ @register_to_config
77
+ def __init__(
78
+ self,
79
+ num_train_timesteps: int = 40,
80
+ sigma_min: float = 0.002,
81
+ sigma_max: float = 80.0,
82
+ sigma_data: float = 0.5,
83
+ s_noise: float = 1.0,
84
+ rho: float = 7.0,
85
+ clip_denoised: bool = True,
86
+ ):
87
+ # standard deviation of the initial noise distribution
88
+ self.init_noise_sigma = (sigma_max**2 + 1) ** 0.5
89
+ # self.init_noise_sigma = sigma_max
90
+
91
+ ramp = np.linspace(0, 1, num_train_timesteps)
92
+ sigmas = self._convert_to_karras(ramp)
93
+ sigmas = np.concatenate([sigmas, np.array([0])])
94
+ timesteps = self.sigma_to_t(sigmas)
95
+
96
+ # setable values
97
+ self.num_inference_steps = None
98
+ self.sigmas = torch.from_numpy(sigmas)
99
+ self.timesteps = torch.from_numpy(timesteps)
100
+ self.custom_timesteps = False
101
+ self.is_scale_input_called = False
102
+ self._step_index = None
103
+ self.sigmas.to("cpu") # to avoid too much CPU/GPU communication
104
+
105
+ def index_for_timestep(self, timestep, schedule_timesteps=None):
106
+ if schedule_timesteps is None:
107
+ schedule_timesteps = self.timesteps
108
+
109
+ indices = (schedule_timesteps == timestep).nonzero()
110
+ return indices.item()
111
+
112
+ @property
113
+ def step_index(self):
114
+ """
115
+ The index counter for current timestep. It will increae 1 after each scheduler step.
116
+ """
117
+ return self._step_index
118
+
119
+ def scale_model_input(
120
+ self, sample: torch.FloatTensor, timestep: Union[float, torch.FloatTensor]
121
+ ) -> torch.FloatTensor:
122
+ """
123
+ Scales the consistency model input by `(sigma**2 + sigma_data**2) ** 0.5`.
124
+
125
+ Args:
126
+ sample (`torch.FloatTensor`):
127
+ The input sample.
128
+ timestep (`float` or `torch.FloatTensor`):
129
+ The current timestep in the diffusion chain.
130
+
131
+ Returns:
132
+ `torch.FloatTensor`:
133
+ A scaled input sample.
134
+ """
135
+ # Get sigma corresponding to timestep
136
+ if self.step_index is None:
137
+ self._init_step_index(timestep)
138
+
139
+ sigma = self.sigmas[self.step_index]
140
+ sample = sample / ((sigma**2 + self.config.sigma_data**2) ** 0.5)
141
+
142
+ self.is_scale_input_called = True
143
+ return sample
144
+
145
+ # def _sigma_to_t(self, sigma, log_sigmas):
146
+ # # get log sigma
147
+ # log_sigma = np.log(np.maximum(sigma, 1e-10))
148
+
149
+ # # get distribution
150
+ # dists = log_sigma - log_sigmas[:, np.newaxis]
151
+
152
+ # # get sigmas range
153
+ # low_idx = np.cumsum((dists >= 0), axis=0).argmax(axis=0).clip(max=log_sigmas.shape[0] - 2)
154
+ # high_idx = low_idx + 1
155
+
156
+ # low = log_sigmas[low_idx]
157
+ # high = log_sigmas[high_idx]
158
+
159
+ # # interpolate sigmas
160
+ # w = (low - log_sigma) / (low - high)
161
+ # w = np.clip(w, 0, 1)
162
+
163
+ # # transform interpolation to time range
164
+ # t = (1 - w) * low_idx + w * high_idx
165
+ # t = t.reshape(sigma.shape)
166
+ # return t
167
+
168
+ def sigma_to_t(self, sigmas: Union[float, np.ndarray]):
169
+ """
170
+ Gets scaled timesteps from the Karras sigmas for input to the consistency model.
171
+
172
+ Args:
173
+ sigmas (`float` or `np.ndarray`):
174
+ A single Karras sigma or an array of Karras sigmas.
175
+
176
+ Returns:
177
+ `float` or `np.ndarray`:
178
+ A scaled input timestep or scaled input timestep array.
179
+ """
180
+ if not isinstance(sigmas, np.ndarray):
181
+ sigmas = np.array(sigmas, dtype=np.float64)
182
+
183
+ timesteps = 0.25 * np.log(sigmas + 1e-44)
184
+
185
+ return timesteps
186
+
187
+ def set_timesteps(
188
+ self,
189
+ num_inference_steps: Optional[int] = None,
190
+ device: Union[str, torch.device] = None,
191
+ timesteps: Optional[List[int]] = None,
192
+ ):
193
+ """
194
+ Sets the timesteps used for the diffusion chain (to be run before inference).
195
+
196
+ Args:
197
+ num_inference_steps (`int`):
198
+ The number of diffusion steps used when generating samples with a pre-trained model.
199
+ device (`str` or `torch.device`, *optional*):
200
+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
201
+ timesteps (`List[int]`, *optional*):
202
+ Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default
203
+ timestep spacing strategy of equal spacing between timesteps is used. If `timesteps` is passed,
204
+ `num_inference_steps` must be `None`.
205
+ """
206
+ if num_inference_steps is None and timesteps is None:
207
+ raise ValueError(
208
+ "Exactly one of `num_inference_steps` or `timesteps` must be supplied."
209
+ )
210
+
211
+ if num_inference_steps is not None and timesteps is not None:
212
+ raise ValueError(
213
+ "Can only pass one of `num_inference_steps` or `timesteps`."
214
+ )
215
+
216
+ # Follow DDPMScheduler custom timesteps logic
217
+ if timesteps is not None:
218
+ for i in range(1, len(timesteps)):
219
+ if timesteps[i] >= timesteps[i - 1]:
220
+ raise ValueError("`timesteps` must be in descending order.")
221
+
222
+ if timesteps[0] >= self.config.num_train_timesteps:
223
+ raise ValueError(
224
+ f"`timesteps` must start before `self.config.train_timesteps`:"
225
+ f" {self.config.num_train_timesteps}."
226
+ )
227
+
228
+ timesteps = np.array(timesteps, dtype=np.int64)
229
+ self.custom_timesteps = True
230
+ else:
231
+ if num_inference_steps > self.config.num_train_timesteps:
232
+ raise ValueError(
233
+ f"`num_inference_steps`: {num_inference_steps} cannot be larger than `self.config.train_timesteps`:"
234
+ f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle"
235
+ f" maximal {self.config.num_train_timesteps} timesteps."
236
+ )
237
+
238
+ self.num_inference_steps = num_inference_steps
239
+
240
+ step_ratio = self.config.num_train_timesteps // self.num_inference_steps
241
+ timesteps = (
242
+ (np.arange(0, num_inference_steps) * step_ratio)
243
+ .round()[::-1]
244
+ .copy()
245
+ .astype(np.int64)
246
+ )
247
+ self.custom_timesteps = False
248
+
249
+ # Map timesteps to Karras sigmas directly for multistep sampling
250
+ # See https://github.com/openai/consistency_models/blob/main/cm/karras_diffusion.py#L675
251
+ num_train_timesteps = self.config.num_train_timesteps
252
+ ramp = timesteps[::-1].copy()
253
+ ramp = ramp / (num_train_timesteps - 1)
254
+ sigmas = self._convert_to_karras(ramp)
255
+ timesteps = self.sigma_to_t(sigmas)
256
+
257
+ sigmas = np.concatenate([sigmas, [0]]).astype(np.float32)
258
+ self.sigmas = torch.from_numpy(sigmas).to(device=device)
259
+
260
+ if str(device).startswith("mps"):
261
+ # mps does not support float64
262
+ self.timesteps = torch.from_numpy(timesteps).to(device, dtype=torch.float32)
263
+ else:
264
+ self.timesteps = torch.from_numpy(timesteps).to(device=device)
265
+
266
+ self._step_index = None
267
+ self.sigmas.to("cpu") # to avoid too much CPU/GPU communication
268
+
269
+ # Modified _convert_to_karras implementation that takes in ramp as argument
270
+ def _convert_to_karras(self, ramp):
271
+ """Constructs the noise schedule of Karras et al. (2022)."""
272
+
273
+ sigma_min: float = self.config.sigma_min
274
+ sigma_max: float = self.config.sigma_max
275
+
276
+ rho = self.config.rho
277
+ min_inv_rho = sigma_min ** (1 / rho)
278
+ max_inv_rho = sigma_max ** (1 / rho)
279
+ sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho
280
+ return sigmas
281
+
282
+ def get_scalings(self, sigma):
283
+ sigma_data = self.config.sigma_data
284
+
285
+ c_skip = sigma_data**2 / (sigma**2 + sigma_data**2)
286
+ c_out = -sigma * sigma_data / (sigma**2 + sigma_data**2) ** 0.5
287
+ return c_skip, c_out
288
+
289
+ def get_scalings_for_boundary_condition(self, sigma):
290
+ """
291
+ Gets the scalings used in the consistency model parameterization (from Appendix C of the
292
+ [paper](https://huggingface.co/papers/2303.01469)) to enforce boundary condition.
293
+
294
+ <Tip>
295
+
296
+ `epsilon` in the equations for `c_skip` and `c_out` is set to `sigma_min`.
297
+
298
+ </Tip>
299
+
300
+ Args:
301
+ sigma (`torch.FloatTensor`):
302
+ The current sigma in the Karras sigma schedule.
303
+
304
+ Returns:
305
+ `tuple`:
306
+ A two-element tuple where `c_skip` (which weights the current sample) is the first element and `c_out`
307
+ (which weights the consistency model output) is the second element.
308
+ """
309
+ sigma_min = self.config.sigma_min
310
+ sigma_data = self.config.sigma_data
311
+
312
+ c_skip = sigma_data**2 / ((sigma) ** 2 + sigma_data**2)
313
+ c_out = -sigma * sigma_data / (sigma**2 + sigma_data**2) ** 0.5
314
+ return c_skip, c_out
315
+
316
+ # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._init_step_index
317
+ def _init_step_index(self, timestep):
318
+ if isinstance(timestep, torch.Tensor):
319
+ timestep = timestep.to(self.timesteps.device)
320
+
321
+ index_candidates = (self.timesteps == timestep).nonzero()
322
+
323
+ # The sigma index that is taken for the **very** first `step`
324
+ # is always the second index (or the last index if there is only 1)
325
+ # This way we can ensure we don't accidentally skip a sigma in
326
+ # case we start in the middle of the denoising schedule (e.g. for image-to-image)
327
+ if len(index_candidates) > 1:
328
+ step_index = index_candidates[1]
329
+ else:
330
+ step_index = index_candidates[0]
331
+
332
+ self._step_index = step_index.item()
333
+
334
+ def step(
335
+ self,
336
+ model_output: torch.FloatTensor,
337
+ timestep: Union[float, torch.FloatTensor],
338
+ sample: torch.FloatTensor,
339
+ generator: Optional[torch.Generator] = None,
340
+ return_dict: bool = True,
341
+ ) -> Union[AnimateLCMSVDStochasticIterativeSchedulerOutput, Tuple]:
342
+ """
343
+ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion
344
+ process from the learned model outputs (most often the predicted noise).
345
+
346
+ Args:
347
+ model_output (`torch.FloatTensor`):
348
+ The direct output from the learned diffusion model.
349
+ timestep (`float`):
350
+ The current timestep in the diffusion chain.
351
+ sample (`torch.FloatTensor`):
352
+ A current instance of a sample created by the diffusion process.
353
+ generator (`torch.Generator`, *optional*):
354
+ A random number generator.
355
+ return_dict (`bool`, *optional*, defaults to `True`):
356
+ Whether or not to return a
357
+ [`~schedulers.scheduling_consistency_models.AnimateLCMSVDStochasticIterativeSchedulerOutput`] or `tuple`.
358
+
359
+ Returns:
360
+ [`~schedulers.scheduling_consistency_models.AnimateLCMSVDStochasticIterativeSchedulerOutput`] or `tuple`:
361
+ If return_dict is `True`,
362
+ [`~schedulers.scheduling_consistency_models.AnimateLCMSVDStochasticIterativeSchedulerOutput`] is returned,
363
+ otherwise a tuple is returned where the first element is the sample tensor.
364
+ """
365
+
366
+ if (
367
+ isinstance(timestep, int)
368
+ or isinstance(timestep, torch.IntTensor)
369
+ or isinstance(timestep, torch.LongTensor)
370
+ ):
371
+ raise ValueError(
372
+ (
373
+ "Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to"
374
+ f" `{self.__class__}.step()` is not supported. Make sure to pass"
375
+ " one of the `scheduler.timesteps` as a timestep."
376
+ ),
377
+ )
378
+
379
+ if not self.is_scale_input_called:
380
+ logger.warning(
381
+ "The `scale_model_input` function should be called before `step` to ensure correct denoising. "
382
+ "See `StableDiffusionPipeline` for a usage example."
383
+ )
384
+
385
+ sigma_min = self.config.sigma_min
386
+ sigma_max = self.config.sigma_max
387
+
388
+ if self.step_index is None:
389
+ self._init_step_index(timestep)
390
+
391
+ # sigma_next corresponds to next_t in original implementation
392
+ sigma = self.sigmas[self.step_index]
393
+ if self.step_index + 1 < self.config.num_train_timesteps:
394
+ sigma_next = self.sigmas[self.step_index + 1]
395
+ else:
396
+ # Set sigma_next to sigma_min
397
+ sigma_next = self.sigmas[-1]
398
+
399
+ # Get scalings for boundary conditions
400
+
401
+ c_skip, c_out = self.get_scalings_for_boundary_condition(sigma)
402
+
403
+ # 1. Denoise model output using boundary conditions
404
+ denoised = c_out * model_output + c_skip * sample
405
+ if self.config.clip_denoised:
406
+ denoised = denoised.clamp(-1, 1)
407
+
408
+ # 2. Sample z ~ N(0, s_noise^2 * I)
409
+ # Noise is not used for onestep sampling.
410
+ if len(self.timesteps) > 1:
411
+ noise = randn_tensor(
412
+ model_output.shape,
413
+ dtype=model_output.dtype,
414
+ device=model_output.device,
415
+ generator=generator,
416
+ )
417
+ else:
418
+ noise = torch.zeros_like(model_output)
419
+ z = noise * self.config.s_noise
420
+
421
+ sigma_hat = sigma_next.clamp(min=0, max=sigma_max)
422
+
423
+ print("denoise currently")
424
+ print(sigma_hat)
425
+
426
+ # origin
427
+ prev_sample = denoised + z * sigma_hat
428
+
429
+ # upon completion increase step index by one
430
+ self._step_index += 1
431
+
432
+ if not return_dict:
433
+ return (prev_sample,)
434
+
435
+ return AnimateLCMSVDStochasticIterativeSchedulerOutput(prev_sample=prev_sample)
436
+
437
+ # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler.add_noise
438
+ def add_noise(
439
+ self,
440
+ original_samples: torch.FloatTensor,
441
+ noise: torch.FloatTensor,
442
+ timesteps: torch.FloatTensor,
443
+ ) -> torch.FloatTensor:
444
+ # Make sure sigmas and timesteps have the same device and dtype as original_samples
445
+ sigmas = self.sigmas.to(
446
+ device=original_samples.device, dtype=original_samples.dtype
447
+ )
448
+ if original_samples.device.type == "mps" and torch.is_floating_point(timesteps):
449
+ # mps does not support float64
450
+ schedule_timesteps = self.timesteps.to(
451
+ original_samples.device, dtype=torch.float32
452
+ )
453
+ timesteps = timesteps.to(original_samples.device, dtype=torch.float32)
454
+ else:
455
+ schedule_timesteps = self.timesteps.to(original_samples.device)
456
+ timesteps = timesteps.to(original_samples.device)
457
+
458
+ step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps]
459
+
460
+ sigma = sigmas[step_indices].flatten()
461
+ while len(sigma.shape) < len(original_samples.shape):
462
+ sigma = sigma.unsqueeze(-1)
463
+
464
+ noisy_samples = original_samples + noise * sigma
465
+ return noisy_samples
466
+
467
+ def __len__(self):
468
+ return self.config.num_train_timesteps
outputs_gradio/000000.mp4 ADDED
Binary file (641 kB). View file
 
outputs_gradio/000001.mp4 ADDED
Binary file (848 kB). View file
 
outputs_gradio/000002.mp4 ADDED
Binary file (393 kB). View file
 
outputs_gradio/000003.mp4 ADDED
Binary file (734 kB). View file
 
outputs_gradio/000004.mp4 ADDED
Binary file (498 kB). View file
 
outputs_gradio/000005.mp4 ADDED
Binary file (373 kB). View file
 
pipeline.py ADDED
@@ -0,0 +1,711 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2023 The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import inspect
16
+ from dataclasses import dataclass
17
+ from typing import Callable, Dict, List, Optional, Union
18
+
19
+ import numpy as np
20
+ import PIL.Image
21
+ import torch
22
+ from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection
23
+
24
+ from diffusers.image_processor import VaeImageProcessor
25
+ from diffusers.models import (
26
+ AutoencoderKLTemporalDecoder,
27
+ UNetSpatioTemporalConditionModel,
28
+ )
29
+ from diffusers.schedulers import EulerDiscreteScheduler
30
+ from diffusers.utils import BaseOutput, logging
31
+ from diffusers.utils.torch_utils import is_compiled_module, randn_tensor
32
+ from diffusers.pipelines.pipeline_utils import DiffusionPipeline
33
+
34
+
35
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
36
+
37
+
38
+ def _append_dims(x, target_dims):
39
+ """Appends dimensions to the end of a tensor until it has target_dims dimensions."""
40
+ dims_to_append = target_dims - x.ndim
41
+ if dims_to_append < 0:
42
+ raise ValueError(
43
+ f"input has {x.ndim} dims but target_dims is {target_dims}, which is less"
44
+ )
45
+ return x[(...,) + (None,) * dims_to_append]
46
+
47
+
48
+ def tensor2vid(video: torch.Tensor, processor, output_type="np"):
49
+ # Based on:
50
+ # https://github.com/modelscope/modelscope/blob/1509fdb973e5871f37148a4b5e5964cafd43e64d/modelscope/pipelines/multi_modal/text_to_video_synthesis_pipeline.py#L78
51
+
52
+ batch_size, channels, num_frames, height, width = video.shape
53
+ outputs = []
54
+ for batch_idx in range(batch_size):
55
+ batch_vid = video[batch_idx].permute(1, 0, 2, 3)
56
+ batch_output = processor.postprocess(batch_vid, output_type)
57
+
58
+ outputs.append(batch_output)
59
+
60
+ return outputs
61
+
62
+
63
+ @dataclass
64
+ class StableVideoDiffusionPipelineOutput(BaseOutput):
65
+ r"""
66
+ Output class for zero-shot text-to-video pipeline.
67
+
68
+ Args:
69
+ frames (`[List[PIL.Image.Image]`, `np.ndarray`]):
70
+ List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width,
71
+ num_channels)`.
72
+ """
73
+
74
+ frames: Union[List[PIL.Image.Image], np.ndarray]
75
+
76
+
77
+ class StableVideoDiffusionPipeline(DiffusionPipeline):
78
+ r"""
79
+ Pipeline to generate video from an input image using Stable Video Diffusion.
80
+
81
+ This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods
82
+ implemented for all pipelines (downloading, saving, running on a particular device, etc.).
83
+
84
+ Args:
85
+ vae ([`AutoencoderKL`]):
86
+ Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations.
87
+ image_encoder ([`~transformers.CLIPVisionModelWithProjection`]):
88
+ Frozen CLIP image-encoder ([laion/CLIP-ViT-H-14-laion2B-s32B-b79K](https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K)).
89
+ unet ([`UNetSpatioTemporalConditionModel`]):
90
+ A `UNetSpatioTemporalConditionModel` to denoise the encoded image latents.
91
+ scheduler ([`EulerDiscreteScheduler`]):
92
+ A scheduler to be used in combination with `unet` to denoise the encoded image latents.
93
+ feature_extractor ([`~transformers.CLIPImageProcessor`]):
94
+ A `CLIPImageProcessor` to extract features from generated images.
95
+ """
96
+
97
+ model_cpu_offload_seq = "image_encoder->unet->vae"
98
+ _callback_tensor_inputs = ["latents"]
99
+
100
+ def __init__(
101
+ self,
102
+ vae: AutoencoderKLTemporalDecoder,
103
+ image_encoder: CLIPVisionModelWithProjection,
104
+ unet: UNetSpatioTemporalConditionModel,
105
+ scheduler: EulerDiscreteScheduler,
106
+ feature_extractor: CLIPImageProcessor,
107
+ ):
108
+ super().__init__()
109
+
110
+ self.register_modules(
111
+ vae=vae,
112
+ image_encoder=image_encoder,
113
+ unet=unet,
114
+ scheduler=scheduler,
115
+ feature_extractor=feature_extractor,
116
+ )
117
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
118
+ self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
119
+
120
+ def _encode_image(
121
+ self, image, device, num_videos_per_prompt, do_classifier_free_guidance
122
+ ):
123
+ dtype = next(self.image_encoder.parameters()).dtype
124
+
125
+ if not isinstance(image, torch.Tensor):
126
+ image = self.image_processor.pil_to_numpy(image)
127
+ image = self.image_processor.numpy_to_pt(image)
128
+
129
+ # We normalize the image before resizing to match with the original implementation.
130
+ # Then we unnormalize it after resizing.
131
+ image = image * 2.0 - 1.0
132
+ image = _resize_with_antialiasing(image, (224, 224))
133
+ image = (image + 1.0) / 2.0
134
+
135
+ # Normalize the image with for CLIP input
136
+ image = self.feature_extractor(
137
+ images=image,
138
+ do_normalize=True,
139
+ do_center_crop=False,
140
+ do_resize=False,
141
+ do_rescale=False,
142
+ return_tensors="pt",
143
+ ).pixel_values
144
+
145
+ image = image.to(device=device, dtype=dtype)
146
+ image_embeddings = self.image_encoder(image).image_embeds
147
+ image_embeddings = image_embeddings.unsqueeze(1)
148
+
149
+ # duplicate image embeddings for each generation per prompt, using mps friendly method
150
+ bs_embed, seq_len, _ = image_embeddings.shape
151
+ image_embeddings = image_embeddings.repeat(1, num_videos_per_prompt, 1)
152
+ image_embeddings = image_embeddings.view(
153
+ bs_embed * num_videos_per_prompt, seq_len, -1
154
+ )
155
+
156
+ if do_classifier_free_guidance:
157
+ negative_image_embeddings = torch.zeros_like(image_embeddings)
158
+
159
+ # For classifier free guidance, we need to do two forward passes.
160
+ # Here we concatenate the unconditional and text embeddings into a single batch
161
+ # to avoid doing two forward passes
162
+ image_embeddings = torch.cat([negative_image_embeddings, image_embeddings])
163
+
164
+ return image_embeddings
165
+
166
+ def _encode_vae_image(
167
+ self,
168
+ image: torch.Tensor,
169
+ device,
170
+ num_videos_per_prompt,
171
+ do_classifier_free_guidance,
172
+ ):
173
+ image = image.to(device=device)
174
+ image_latents = self.vae.encode(image).latent_dist.mode()
175
+
176
+ if do_classifier_free_guidance:
177
+ negative_image_latents = torch.zeros_like(image_latents)
178
+
179
+ # For classifier free guidance, we need to do two forward passes.
180
+ # Here we concatenate the unconditional and text embeddings into a single batch
181
+ # to avoid doing two forward passes
182
+ image_latents = torch.cat([negative_image_latents, image_latents])
183
+
184
+ # duplicate image_latents for each generation per prompt, using mps friendly method
185
+ image_latents = image_latents.repeat(num_videos_per_prompt, 1, 1, 1)
186
+
187
+ return image_latents
188
+
189
+ def _get_add_time_ids(
190
+ self,
191
+ fps,
192
+ motion_bucket_id,
193
+ noise_aug_strength,
194
+ dtype,
195
+ batch_size,
196
+ num_videos_per_prompt,
197
+ do_classifier_free_guidance,
198
+ ):
199
+ add_time_ids = [fps, motion_bucket_id, noise_aug_strength]
200
+
201
+ passed_add_embed_dim = self.unet.config.addition_time_embed_dim * len(
202
+ add_time_ids
203
+ )
204
+ expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features
205
+
206
+ if expected_add_embed_dim != passed_add_embed_dim:
207
+ raise ValueError(
208
+ f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`."
209
+ )
210
+
211
+ add_time_ids = torch.tensor([add_time_ids], dtype=dtype)
212
+ add_time_ids = add_time_ids.repeat(batch_size * num_videos_per_prompt, 1)
213
+
214
+ if do_classifier_free_guidance:
215
+ add_time_ids = torch.cat([add_time_ids, add_time_ids])
216
+
217
+ return add_time_ids
218
+
219
+ def decode_latents(self, latents, num_frames, decode_chunk_size=14):
220
+ # [batch, frames, channels, height, width] -> [batch*frames, channels, height, width]
221
+ latents = latents.flatten(0, 1)
222
+
223
+ latents = 1 / self.vae.config.scaling_factor * latents
224
+
225
+ forward_vae_fn = (
226
+ self.vae._orig_mod.forward
227
+ if is_compiled_module(self.vae)
228
+ else self.vae.forward
229
+ )
230
+ accepts_num_frames = "num_frames" in set(
231
+ inspect.signature(forward_vae_fn).parameters.keys()
232
+ )
233
+
234
+ # decode decode_chunk_size frames at a time to avoid OOM
235
+ frames = []
236
+ for i in range(0, latents.shape[0], decode_chunk_size):
237
+ num_frames_in = latents[i : i + decode_chunk_size].shape[0]
238
+ decode_kwargs = {}
239
+ if accepts_num_frames:
240
+ # we only pass num_frames_in if it's expected
241
+ decode_kwargs["num_frames"] = num_frames_in
242
+
243
+ frame = self.vae.decode(
244
+ latents[i : i + decode_chunk_size], **decode_kwargs
245
+ ).sample
246
+ frames.append(frame)
247
+ frames = torch.cat(frames, dim=0)
248
+
249
+ # [batch*frames, channels, height, width] -> [batch, channels, frames, height, width]
250
+ frames = frames.reshape(-1, num_frames, *frames.shape[1:]).permute(
251
+ 0, 2, 1, 3, 4
252
+ )
253
+
254
+ # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16
255
+ frames = frames.float()
256
+ return frames
257
+
258
+ def check_inputs(self, image, height, width):
259
+ if (
260
+ not isinstance(image, torch.Tensor)
261
+ and not isinstance(image, PIL.Image.Image)
262
+ and not isinstance(image, list)
263
+ ):
264
+ raise ValueError(
265
+ "`image` has to be of type `torch.FloatTensor` or `PIL.Image.Image` or `List[PIL.Image.Image]` but is"
266
+ f" {type(image)}"
267
+ )
268
+
269
+ if height % 8 != 0 or width % 8 != 0:
270
+ raise ValueError(
271
+ f"`height` and `width` have to be divisible by 8 but are {height} and {width}."
272
+ )
273
+
274
+ def prepare_latents(
275
+ self,
276
+ batch_size,
277
+ num_frames,
278
+ num_channels_latents,
279
+ height,
280
+ width,
281
+ dtype,
282
+ device,
283
+ generator,
284
+ latents=None,
285
+ ):
286
+ shape = (
287
+ batch_size,
288
+ num_frames,
289
+ num_channels_latents // 2,
290
+ height // self.vae_scale_factor,
291
+ width // self.vae_scale_factor,
292
+ )
293
+ if isinstance(generator, list) and len(generator) != batch_size:
294
+ raise ValueError(
295
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
296
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
297
+ )
298
+
299
+ if latents is None:
300
+ latents = randn_tensor(
301
+ shape, generator=generator, device=device, dtype=dtype
302
+ )
303
+ else:
304
+ latents = latents.to(device)
305
+
306
+ # scale the initial noise by the standard deviation required by the scheduler
307
+ latents = latents * self.scheduler.init_noise_sigma
308
+ return latents
309
+
310
+ @property
311
+ def guidance_scale(self):
312
+ return self._guidance_scale
313
+
314
+ # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
315
+ # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
316
+ # corresponds to doing no classifier free guidance.
317
+ @property
318
+ def do_classifier_free_guidance(self):
319
+ if isinstance(self.guidance_scale, (int, float)):
320
+ return self.guidance_scale > 1
321
+ return self.guidance_scale.max() > 1
322
+
323
+ @property
324
+ def num_timesteps(self):
325
+ return self._num_timesteps
326
+
327
+ @torch.no_grad()
328
+ def __call__(
329
+ self,
330
+ image: Union[PIL.Image.Image, List[PIL.Image.Image], torch.FloatTensor],
331
+ height: int = 576,
332
+ width: int = 1024,
333
+ num_frames: Optional[int] = None,
334
+ num_inference_steps: int = 25,
335
+ min_guidance_scale: float = 1.0,
336
+ max_guidance_scale: float = 3.0,
337
+ fps: int = 7,
338
+ motion_bucket_id: int = 127,
339
+ noise_aug_strength: int = 0.02,
340
+ decode_chunk_size: Optional[int] = None,
341
+ num_videos_per_prompt: Optional[int] = 1,
342
+ generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
343
+ latents: Optional[torch.FloatTensor] = None,
344
+ output_type: Optional[str] = "pil",
345
+ callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,
346
+ callback_on_step_end_tensor_inputs: List[str] = ["latents"],
347
+ return_dict: bool = True,
348
+ ):
349
+ r"""
350
+ The call function to the pipeline for generation.
351
+
352
+ Args:
353
+ image (`PIL.Image.Image` or `List[PIL.Image.Image]` or `torch.FloatTensor`):
354
+ Image or images to guide image generation. If you provide a tensor, it needs to be compatible with
355
+ [`CLIPImageProcessor`](https://huggingface.co/lambdalabs/sd-image-variations-diffusers/blob/main/feature_extractor/preprocessor_config.json).
356
+ height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):
357
+ The height in pixels of the generated image.
358
+ width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`):
359
+ The width in pixels of the generated image.
360
+ num_frames (`int`, *optional*):
361
+ The number of video frames to generate. Defaults to 14 for `stable-video-diffusion-img2vid` and to 25 for `stable-video-diffusion-img2vid-xt`
362
+ num_inference_steps (`int`, *optional*, defaults to 25):
363
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
364
+ expense of slower inference. This parameter is modulated by `strength`.
365
+ min_guidance_scale (`float`, *optional*, defaults to 1.0):
366
+ The minimum guidance scale. Used for the classifier free guidance with first frame.
367
+ max_guidance_scale (`float`, *optional*, defaults to 3.0):
368
+ The maximum guidance scale. Used for the classifier free guidance with last frame.
369
+ fps (`int`, *optional*, defaults to 7):
370
+ Frames per second. The rate at which the generated images shall be exported to a video after generation.
371
+ Note that Stable Diffusion Video's UNet was micro-conditioned on fps-1 during training.
372
+ motion_bucket_id (`int`, *optional*, defaults to 127):
373
+ The motion bucket ID. Used as conditioning for the generation. The higher the number the more motion will be in the video.
374
+ noise_aug_strength (`int`, *optional*, defaults to 0.02):
375
+ The amount of noise added to the init image, the higher it is the less the video will look like the init image. Increase it for more motion.
376
+ decode_chunk_size (`int`, *optional*):
377
+ The number of frames to decode at a time. The higher the chunk size, the higher the temporal consistency
378
+ between frames, but also the higher the memory consumption. By default, the decoder will decode all frames at once
379
+ for maximal quality. Reduce `decode_chunk_size` to reduce memory usage.
380
+ num_videos_per_prompt (`int`, *optional*, defaults to 1):
381
+ The number of images to generate per prompt.
382
+ generator (`torch.Generator` or `List[torch.Generator]`, *optional*):
383
+ A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make
384
+ generation deterministic.
385
+ latents (`torch.FloatTensor`, *optional*):
386
+ Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image
387
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
388
+ tensor is generated by sampling using the supplied random `generator`.
389
+ output_type (`str`, *optional*, defaults to `"pil"`):
390
+ The output format of the generated image. Choose between `PIL.Image` or `np.array`.
391
+ callback_on_step_end (`Callable`, *optional*):
392
+ A function that calls at the end of each denoising steps during the inference. The function is called
393
+ with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
394
+ callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
395
+ `callback_on_step_end_tensor_inputs`.
396
+ callback_on_step_end_tensor_inputs (`List`, *optional*):
397
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
398
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
399
+ `._callback_tensor_inputs` attribute of your pipeline class.
400
+ return_dict (`bool`, *optional*, defaults to `True`):
401
+ Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
402
+ plain tuple.
403
+
404
+ Returns:
405
+ [`~pipelines.stable_diffusion.StableVideoDiffusionPipelineOutput`] or `tuple`:
406
+ If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableVideoDiffusionPipelineOutput`] is returned,
407
+ otherwise a `tuple` is returned where the first element is a list of list with the generated frames.
408
+
409
+ Examples:
410
+
411
+ ```py
412
+ from diffusers import StableVideoDiffusionPipeline
413
+ from diffusers.utils import load_image, export_to_video
414
+
415
+ pipe = StableVideoDiffusionPipeline.from_pretrained("stabilityai/stable-video-diffusion-img2vid-xt", torch_dtype=torch.float16, variant="fp16")
416
+ pipe.to("cuda")
417
+
418
+ image = load_image("https://lh3.googleusercontent.com/y-iFOHfLTwkuQSUegpwDdgKmOjRSTvPxat63dQLB25xkTs4lhIbRUFeNBWZzYf370g=s1200")
419
+ image = image.resize((1024, 576))
420
+
421
+ frames = pipe(image, num_frames=25, decode_chunk_size=8).frames[0]
422
+ export_to_video(frames, "generated.mp4", fps=7)
423
+ ```
424
+ """
425
+ # 0. Default height and width to unet
426
+ height = height or self.unet.config.sample_size * self.vae_scale_factor
427
+ width = width or self.unet.config.sample_size * self.vae_scale_factor
428
+
429
+ num_frames = (
430
+ num_frames if num_frames is not None else self.unet.config.num_frames
431
+ )
432
+ decode_chunk_size = (
433
+ decode_chunk_size if decode_chunk_size is not None else num_frames
434
+ )
435
+
436
+ # 1. Check inputs. Raise error if not correct
437
+ self.check_inputs(image, height, width)
438
+
439
+ # 2. Define call parameters
440
+ if isinstance(image, PIL.Image.Image):
441
+ batch_size = 1
442
+ elif isinstance(image, list):
443
+ batch_size = len(image)
444
+ else:
445
+ batch_size = image.shape[0]
446
+ device = self._execution_device
447
+ # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
448
+ # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
449
+ # corresponds to doing no classifier free guidance.
450
+ self._guidance_scale = max_guidance_scale
451
+
452
+ # 3. Encode input image
453
+ image_embeddings = self._encode_image(
454
+ image, device, num_videos_per_prompt, self.do_classifier_free_guidance
455
+ )
456
+
457
+ # NOTE: Stable Diffusion Video was conditioned on fps - 1, which
458
+ # is why it is reduced here.
459
+ # See: https://github.com/Stability-AI/generative-models/blob/ed0997173f98eaf8f4edf7ba5fe8f15c6b877fd3/scripts/sampling/simple_video_sample.py#L188
460
+ fps = fps - 1
461
+
462
+ # 4. Encode input image using VAE
463
+ image = self.image_processor.preprocess(image, height=height, width=width)
464
+ noise = randn_tensor(
465
+ image.shape, generator=generator, device=image.device, dtype=image.dtype
466
+ )
467
+ image = image + noise_aug_strength * noise
468
+
469
+ needs_upcasting = (
470
+ self.vae.dtype == torch.float16 and self.vae.config.force_upcast
471
+ )
472
+ if needs_upcasting:
473
+ self.vae.to(dtype=torch.float32)
474
+
475
+ image_latents = self._encode_vae_image(
476
+ image, device, num_videos_per_prompt, self.do_classifier_free_guidance
477
+ )
478
+ image_latents = image_latents.to(image_embeddings.dtype)
479
+
480
+ # cast back to fp16 if needed
481
+ if needs_upcasting:
482
+ self.vae.to(dtype=torch.float16)
483
+
484
+ # Repeat the image latents for each frame so we can concatenate them with the noise
485
+ # image_latents [batch, channels, height, width] ->[batch, num_frames, channels, height, width]
486
+ image_latents = image_latents.unsqueeze(1).repeat(1, num_frames, 1, 1, 1)
487
+
488
+ # 5. Get Added Time IDs
489
+ added_time_ids = self._get_add_time_ids(
490
+ fps,
491
+ motion_bucket_id,
492
+ noise_aug_strength,
493
+ image_embeddings.dtype,
494
+ batch_size,
495
+ num_videos_per_prompt,
496
+ self.do_classifier_free_guidance,
497
+ )
498
+ added_time_ids = added_time_ids.to(device)
499
+
500
+ # 4. Prepare timesteps
501
+ self.scheduler.set_timesteps(num_inference_steps, device=device)
502
+ print("converted after karras", self.scheduler.sigmas)
503
+ timesteps = self.scheduler.timesteps
504
+
505
+ # 5. Prepare latent variables
506
+ num_channels_latents = self.unet.config.in_channels
507
+ latents = self.prepare_latents(
508
+ batch_size * num_videos_per_prompt,
509
+ num_frames,
510
+ num_channels_latents,
511
+ height,
512
+ width,
513
+ image_embeddings.dtype,
514
+ device,
515
+ generator,
516
+ latents,
517
+ )
518
+
519
+ # 7. Prepare guidance scale
520
+ guidance_scale = torch.linspace(
521
+ min_guidance_scale, max_guidance_scale, num_frames
522
+ ).unsqueeze(0)
523
+ guidance_scale = guidance_scale.to(device, latents.dtype)
524
+ guidance_scale = guidance_scale.repeat(batch_size * num_videos_per_prompt, 1)
525
+ guidance_scale = _append_dims(guidance_scale, latents.ndim)
526
+
527
+ self._guidance_scale = guidance_scale
528
+
529
+ # 8. Denoising loop
530
+ num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order
531
+ self._num_timesteps = len(timesteps)
532
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
533
+ for i, t in enumerate(timesteps):
534
+ # expand the latents if we are doing classifier free guidance
535
+ latent_model_input = (
536
+ torch.cat([latents] * 2)
537
+ if self.do_classifier_free_guidance
538
+ else latents
539
+ )
540
+ latent_model_input = self.scheduler.scale_model_input(
541
+ latent_model_input, t
542
+ )
543
+
544
+ # Concatenate image_latents over channels dimention
545
+ latent_model_input = torch.cat(
546
+ [latent_model_input, image_latents], dim=2
547
+ )
548
+
549
+ # predict the noise residual
550
+ noise_pred = self.unet(
551
+ latent_model_input,
552
+ t,
553
+ encoder_hidden_states=image_embeddings,
554
+ added_time_ids=added_time_ids,
555
+ return_dict=False,
556
+ )[0]
557
+
558
+ # perform guidance
559
+ if self.do_classifier_free_guidance:
560
+ noise_pred_uncond, noise_pred_cond = noise_pred.chunk(2)
561
+ noise_pred = noise_pred_uncond + self.guidance_scale * (
562
+ noise_pred_cond - noise_pred_uncond
563
+ )
564
+
565
+ # compute the previous noisy sample x_t -> x_t-1
566
+ latents = self.scheduler.step(noise_pred, t, latents).prev_sample
567
+
568
+ if callback_on_step_end is not None:
569
+ callback_kwargs = {}
570
+ for k in callback_on_step_end_tensor_inputs:
571
+ callback_kwargs[k] = locals()[k]
572
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
573
+
574
+ latents = callback_outputs.pop("latents", latents)
575
+
576
+ if i == len(timesteps) - 1 or (
577
+ (i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0
578
+ ):
579
+ progress_bar.update()
580
+
581
+ if not output_type == "latent":
582
+ # cast back to fp16 if needed
583
+ if needs_upcasting:
584
+ self.vae.to(dtype=torch.float16)
585
+ frames = self.decode_latents(latents, num_frames, decode_chunk_size)
586
+ frames = tensor2vid(frames, self.image_processor, output_type=output_type)
587
+ else:
588
+ frames = latents
589
+
590
+ self.maybe_free_model_hooks()
591
+
592
+ if not return_dict:
593
+ return frames
594
+
595
+ return StableVideoDiffusionPipelineOutput(frames=frames)
596
+
597
+
598
+ # resizing utils
599
+ # TODO: clean up later
600
+ def _resize_with_antialiasing(input, size, interpolation="bicubic", align_corners=True):
601
+ h, w = input.shape[-2:]
602
+ factors = (h / size[0], w / size[1])
603
+
604
+ # First, we have to determine sigma
605
+ # Taken from skimage: https://github.com/scikit-image/scikit-image/blob/v0.19.2/skimage/transform/_warps.py#L171
606
+ sigmas = (
607
+ max((factors[0] - 1.0) / 2.0, 0.001),
608
+ max((factors[1] - 1.0) / 2.0, 0.001),
609
+ )
610
+
611
+ # Now kernel size. Good results are for 3 sigma, but that is kind of slow. Pillow uses 1 sigma
612
+ # https://github.com/python-pillow/Pillow/blob/master/src/libImaging/Resample.c#L206
613
+ # But they do it in the 2 passes, which gives better results. Let's try 2 sigmas for now
614
+ ks = int(max(2.0 * 2 * sigmas[0], 3)), int(max(2.0 * 2 * sigmas[1], 3))
615
+
616
+ # Make sure it is odd
617
+ if (ks[0] % 2) == 0:
618
+ ks = ks[0] + 1, ks[1]
619
+
620
+ if (ks[1] % 2) == 0:
621
+ ks = ks[0], ks[1] + 1
622
+
623
+ input = _gaussian_blur2d(input, ks, sigmas)
624
+
625
+ output = torch.nn.functional.interpolate(
626
+ input, size=size, mode=interpolation, align_corners=align_corners
627
+ )
628
+ return output
629
+
630
+
631
+ def _compute_padding(kernel_size):
632
+ """Compute padding tuple."""
633
+ # 4 or 6 ints: (padding_left, padding_right,padding_top,padding_bottom)
634
+ # https://pytorch.org/docs/stable/nn.html#torch.nn.functional.pad
635
+ if len(kernel_size) < 2:
636
+ raise AssertionError(kernel_size)
637
+ computed = [k - 1 for k in kernel_size]
638
+
639
+ # for even kernels we need to do asymmetric padding :(
640
+ out_padding = 2 * len(kernel_size) * [0]
641
+
642
+ for i in range(len(kernel_size)):
643
+ computed_tmp = computed[-(i + 1)]
644
+
645
+ pad_front = computed_tmp // 2
646
+ pad_rear = computed_tmp - pad_front
647
+
648
+ out_padding[2 * i + 0] = pad_front
649
+ out_padding[2 * i + 1] = pad_rear
650
+
651
+ return out_padding
652
+
653
+
654
+ def _filter2d(input, kernel):
655
+ # prepare kernel
656
+ b, c, h, w = input.shape
657
+ tmp_kernel = kernel[:, None, ...].to(device=input.device, dtype=input.dtype)
658
+
659
+ tmp_kernel = tmp_kernel.expand(-1, c, -1, -1)
660
+
661
+ height, width = tmp_kernel.shape[-2:]
662
+
663
+ padding_shape: list[int] = _compute_padding([height, width])
664
+ input = torch.nn.functional.pad(input, padding_shape, mode="reflect")
665
+
666
+ # kernel and input tensor reshape to align element-wise or batch-wise params
667
+ tmp_kernel = tmp_kernel.reshape(-1, 1, height, width)
668
+ input = input.view(-1, tmp_kernel.size(0), input.size(-2), input.size(-1))
669
+
670
+ # convolve the tensor with the kernel.
671
+ output = torch.nn.functional.conv2d(
672
+ input, tmp_kernel, groups=tmp_kernel.size(0), padding=0, stride=1
673
+ )
674
+
675
+ out = output.view(b, c, h, w)
676
+ return out
677
+
678
+
679
+ def _gaussian(window_size: int, sigma):
680
+ if isinstance(sigma, float):
681
+ sigma = torch.tensor([[sigma]])
682
+
683
+ batch_size = sigma.shape[0]
684
+
685
+ x = (
686
+ torch.arange(window_size, device=sigma.device, dtype=sigma.dtype)
687
+ - window_size // 2
688
+ ).expand(batch_size, -1)
689
+
690
+ if window_size % 2 == 0:
691
+ x = x + 0.5
692
+
693
+ gauss = torch.exp(-x.pow(2.0) / (2 * sigma.pow(2.0)))
694
+
695
+ return gauss / gauss.sum(-1, keepdim=True)
696
+
697
+
698
+ def _gaussian_blur2d(input, kernel_size, sigma):
699
+ if isinstance(sigma, tuple):
700
+ sigma = torch.tensor([sigma], dtype=input.dtype)
701
+ else:
702
+ sigma = sigma.to(dtype=input.dtype)
703
+
704
+ ky, kx = int(kernel_size[0]), int(kernel_size[1])
705
+ bs = sigma.shape[0]
706
+ kernel_x = _gaussian(kx, sigma[:, 1].view(bs, 1))
707
+ kernel_y = _gaussian(ky, sigma[:, 0].view(bs, 1))
708
+ out_x = _filter2d(input, kernel_x[..., None, :])
709
+ out = _filter2d(out_x, kernel_y[..., None])
710
+
711
+ return out
requirements.txt ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ diffusers==0.25.1
2
+ gradio==4.19.2
3
+ Pillow==10.2.0
4
+ torch==2.2.0
safetensors/AnimateLCM-SVD-xt-1.1.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:94b2da1fcca8d03458ef0b07b94b4c55f94117cb0d90265c6c8452239ecc166e
3
+ size 6098682464
safetensors/AnimateLCM-SVD-xt.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6ca55e35f29437e8a65e8a1a9ce75262d5bab3d4fe137bdc3f3a94512c54b377
3
+ size 6098682464
test_imgs/ai-generated-8255456_1280.png ADDED

Git LFS Details

  • SHA256: 4e36d82e13141d8fc8ead89d57f209a9d27d5768cd1b92323b0bf51e465a7968
  • Pointer size: 132 Bytes
  • Size of remote file: 1.44 MB
test_imgs/ai-generated-8411866_1280.jpg ADDED
test_imgs/ai-generated-8463496_1280.jpg ADDED
test_imgs/ai-generated-8476858_1280.png ADDED
test_imgs/ai-generated-8479572_1280.jpg ADDED
test_imgs/ai-generated-8481641_1280.jpg ADDED
test_imgs/ai-generated-8489879_1280.png ADDED

Git LFS Details

  • SHA256: 4a0ecd8306216a74ae6e19b63753284ee4c58f8dba4c09d8b3091c1f8353745c
  • Pointer size: 132 Bytes
  • Size of remote file: 1.53 MB
test_imgs/ai-generated-8496135_1280.jpg ADDED
test_imgs/ai-generated-8496952_1280.jpg ADDED
test_imgs/ai-generated-8498844_1280.jpg ADDED
test_imgs/bird-7411270_1280.jpg ADDED
test_imgs/bird-7586857_1280.jpg ADDED
test_imgs/bird-8014191_1280.jpg ADDED
test_imgs/couple-8019370_1280.jpg ADDED
test_imgs/cupcakes-380178_1280.jpg ADDED
test_imgs/dog-7330712_1280.jpg ADDED
test_imgs/dog-7396912_1280.jpg ADDED
test_imgs/girl-4898696_1280.jpg ADDED
test_imgs/grey-capped-flycatcher-8071233_1280.jpg ADDED
test_imgs/halloween-4585684_1280.jpg ADDED
test_imgs/leaf-7260246_1280.jpg ADDED
test_imgs/meerkat-7465819_1280.jpg ADDED
test_imgs/mobile-phone-1875813_1280.jpg ADDED
test_imgs/mother-8097324_1280.jpg ADDED
test_imgs/plane-8145957_1280.jpg ADDED
test_imgs/power-station-6579092_1280.jpg ADDED
test_imgs/ship-7833921_1280.jpg ADDED
test_imgs/sleep-7871915_1280.jpg ADDED
test_imgs/squirrel-7985502_1280.jpg ADDED
test_imgs/squirrel-8211238_1280.jpg ADDED
test_imgs/training-8122941_1280.jpg ADDED