First motion control: fixed camera (#40)

- First motion control: fixed camera (82ee1a35bd10490d96510e2f219120185c0afe5e)


Co-authored-by: Fabrice TIERCELIN <Fabrice-TIERCELIN@users.noreply.huggingface.co>

app.py

@@ -42,7 +42,8 @@ def animate(
     frame_format: str = "webp",
     version: str = "auto",
     width: int = 1024,
-    height: int = 576
+    height: int = 576,
+    motion_control: bool = False
 ):
     start = time.time()
 
@@ -50,9 +51,12 @@ def animate(
         raise gr.Error("Please provide an image to animate.")
 
     output_folder = "outputs"
-    image = resize_image(image, output_size=(width, height))
-    if image.mode == "RGBA":
-        image = image.convert("RGB")
+    image_data = resize_image(image, output_size=(width, height))
+    if image_data.mode == "RGBA":
+        image_data = image_data.convert("RGB")
+
+    if motion_control:
+        image_data = [image_data] * 25
         
     if randomize_seed:
         seed = random.randint(0, max_64_bit_int)
@@ -64,7 +68,7 @@ def animate(
             version = "svd"
 
     frames = animate_on_gpu(
-        image,
+        image_data,
         seed,
         motion_bucket_id,
         fps_id,
@@ -121,7 +125,7 @@ def animate(
 @torch.no_grad()
 @spaces.GPU(duration=180)
 def animate_on_gpu(
-    image: Image,
+    image_data: Union[Image, List[Image]],
     seed: Optional[int] = 42,
     motion_bucket_id: int = 127,
     fps_id: int = 6,
@@ -134,11 +138,11 @@ def animate_on_gpu(
     generator = torch.manual_seed(seed)
 
     if version == "dragnuwa":
-        return dragnuwaPipe(image, width=width, height=height, decode_chunk_size=decoding_t, generator=generator, motion_bucket_id=motion_bucket_id, noise_aug_strength=noise_aug_strength, num_frames=25).frames[0]
+        return dragnuwaPipe(image_data, width=width, height=height, decode_chunk_size=decoding_t, generator=generator, motion_bucket_id=motion_bucket_id, noise_aug_strength=noise_aug_strength, num_frames=25).frames[0]
     elif version == "svdxt":
-        return fps25Pipe(image, width=width, height=height, decode_chunk_size=decoding_t, generator=generator, motion_bucket_id=motion_bucket_id, noise_aug_strength=noise_aug_strength, num_frames=25).frames[0]
+        return fps25Pipe(image_data, width=width, height=height, decode_chunk_size=decoding_t, generator=generator, motion_bucket_id=motion_bucket_id, noise_aug_strength=noise_aug_strength, num_frames=25).frames[0]
     else:
-        return fps14Pipe(image, width=width, height=height, decode_chunk_size=decoding_t, generator=generator, motion_bucket_id=motion_bucket_id, noise_aug_strength=noise_aug_strength, num_frames=25).frames[0]
+        return fps14Pipe(image_data, width=width, height=height, decode_chunk_size=decoding_t, generator=generator, motion_bucket_id=motion_bucket_id, noise_aug_strength=noise_aug_strength, num_frames=25).frames[0]
 
 
 def resize_image(image, output_size=(1024, 576)):
@@ -188,7 +192,8 @@ def reset():
         "webp",
         "auto",
         1024,
-        576
+        576,
+        False
     ]
 
 with gr.Blocks() as demo:
@@ -210,12 +215,13 @@ with gr.Blocks() as demo:
           with gr.Accordion("Advanced options", open=False):
               width = gr.Slider(label="Width", info="Width of the video", value=1024, minimum=256, maximum=1024, step=8)
               height = gr.Slider(label="Height", info="Height of the video", value=576, minimum=256, maximum=576, step=8)
+              motion_control = gr.Checkbox(label="Motion control (fixed camera)", info="Fix the camera", value=False)
+              video_format = gr.Radio([["*.mp4", "mp4"], ["*.avi", "avi"], ["*.wmv", "wmv"], ["*.mkv", "mkv"], ["*.mov", "mov"], ["*.gif", "gif"]], label="Video format for result", info="File extention", value="mp4", interactive=True)
+              frame_format = gr.Radio([["*.webp", "webp"], ["*.png", "png"], ["*.jpeg", "jpeg"], ["*.gif (unanimated)", "gif"], ["*.bmp", "bmp"]], label="Image format for frames", info="File extention", value="webp", interactive=True)
               fps_id = gr.Slider(label="Frames per second", info="The length of your video in seconds will be 25/fps", value=25, minimum=5, maximum=30)
               motion_bucket_id = gr.Slider(label="Motion bucket id", info="Controls how much motion to add/remove from the image", value=127, minimum=1, maximum=255)
               noise_aug_strength = gr.Slider(label="Noise strength", info="The noise to add", value=0.1, minimum=0, maximum=1, step=0.1)
               decoding_t = gr.Slider(label="Decoding", info="Number of frames decoded at a time; this eats more VRAM; reduce if necessary", value=3, minimum=1, maximum=5, step=1)
-              video_format = gr.Radio([["*.mp4", "mp4"], ["*.avi", "avi"], ["*.wmv", "wmv"], ["*.mkv", "mkv"], ["*.mov", "mov"], ["*.gif", "gif"]], label="Video format for result", info="File extention", value="mp4", interactive=True)
-              frame_format = gr.Radio([["*.webp", "webp"], ["*.png", "png"], ["*.jpeg", "jpeg"], ["*.gif (unanimated)", "gif"], ["*.bmp", "bmp"]], label="Image format for frames", info="File extention", value="webp", interactive=True)
               version = gr.Radio([["Auto", "auto"], ["🏃🏻‍♀️ SVD (trained on 14 f/s)", "svd"], ["🏃🏻‍♀️💨 SVD-XT (trained on 25 f/s)", "svdxt"], ["DragNUWA (unstable)", "dragnuwa"]], label="Model", info="Trained model", value="auto", interactive=True)
               seed = gr.Slider(label="Seed", value=42, randomize=True, minimum=0, maximum=max_64_bit_int, step=1)
               randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
@@ -242,7 +248,8 @@ with gr.Blocks() as demo:
       frame_format,
       version,
       width,
-      height
+      height,
+      motion_control
   ], outputs=[
       video_output,
       gif_output,
@@ -265,16 +272,17 @@ with gr.Blocks() as demo:
       frame_format,
       version,
       width,
-      height
+      height,
+      motion_control
   ], queue = False, show_progress = False)
     
   gr.Examples(
     examples=[
-        ["Examples/Fire.webp", 42, True, 127, 25, 0.1, 3, "mp4", "png", "auto", 1024, 576],
-        ["Examples/Water.png", 42, True, 127, 25, 0.1, 3, "mp4", "png", "auto", 1024, 576],
-        ["Examples/Town.jpeg", 42, True, 127, 25, 0.1, 3, "mp4", "png", "auto", 1024, 576]
+        ["Examples/Fire.webp", 42, True, 127, 25, 0.1, 3, "mp4", "png", "auto", 1024, 576, False],
+        ["Examples/Water.png", 42, True, 127, 25, 0.1, 3, "mp4", "png", "auto", 1024, 576, False],
+        ["Examples/Town.jpeg", 42, True, 127, 25, 0.1, 3, "mp4", "png", "auto", 1024, 576, False]
     ],
-    inputs=[image, seed, randomize_seed, motion_bucket_id, fps_id, noise_aug_strength, decoding_t, video_format, frame_format, version, width, height],
+    inputs=[image, seed, randomize_seed, motion_bucket_id, fps_id, noise_aug_strength, decoding_t, video_format, frame_format, version, width, height, motion_control],
     outputs=[video_output, gif_output, download_button, gallery, seed, information_msg, reset_btn],
     fn=animate,
     run_on_click=True,