This PR makes the space work again

README.md

@@ -3,46 +3,7 @@ title: 📺RTV🖼️ - Real Time Video AI
 emoji: 🖼️📺
 colorFrom: purple
 colorTo: yellow
-sdk: gradio
-sdk_version: 4.4.0
-app_file: app.py
-pinned: false
+short_description: Animate Your Pictures With Stable VIdeo DIffusion
 license: other
----
-
-1. Process Images in real time with prompts:
-2. Example:
-  - Elves Toy Factories North Pole Christmas Magic Elf Happy Magical Toy Robots Polar Bears Creatures Winter Streets with Holiday Festivals Christmas Present Lists Toys Candy Books Christmas Fun Facts
-Happy New Years
-3. Add prompt here and download images: https://huggingface.co/spaces/awacke1/RealTimeImageGen
-4. Load images to 01.png thru 09.png below
-5. Rebuild 
-
-
-One of the greatest new generative AI models.  This model is being tested to generate Christmas themed videos including:
-1. Reindeer
-2. Sunset and Sunrise views of Christmas Eve and Christmas Day
-3. Saint Nicholas
-4. Elves and Toy Factories
-5. Polar Bears, and Other Winter Creatures
-6. Winter Streets with Holiday Festivals
-7. Christmas Present Lists Toys and Candy
-8. Books of Christmas with Fun Facts
-9. Happy New Years!
-
--In app.py this is implemented here and will cache the examples and process while loading creating 4 second videos for each image:
-
-```
-gr.Examples(
-    examples=[
-        "images/01.png",
-        "images/02.png",
-        "images/03.png",
-        "images/04.png",
-        "images/05.png",
-        "images/06.png",
-        "images/07.png",
-        "images/08.png",
-        "images/09.png"
-    ],
-```
+sdk: gradio
+---

app.py

@@ -1,114 +1,300 @@
 import gradio as gr
 import torch
 import os
-import uuid
 import random
+import time
+import math
+import spaces
 from glob import glob
 from pathlib import Path
-from typing import Optional
+from typing import Optional, List, Union
+
 from diffusers import StableVideoDiffusionPipeline
-from diffusers.utils import load_image, export_to_video
+from diffusers.utils import export_to_video, export_to_gif
 from PIL import Image
-from huggingface_hub import hf_hub_download
 
+fps25Pipe = StableVideoDiffusionPipeline.from_pretrained(
+    "vdo/stable-video-diffusion-img2vid-xt-1-1", torch_dtype=torch.float16, variant="fp16"
+)
+fps25Pipe.to("cuda")
 
-pipe = StableVideoDiffusionPipeline.from_pretrained(
-    "stabilityai/stable-video-diffusion-img2vid-xt", torch_dtype=torch.float16, variant="fp16"
+fps14Pipe = StableVideoDiffusionPipeline.from_pretrained(
+    "stabilityai/stable-video-diffusion-img2vid", torch_dtype=torch.float16, variant="fp16"
 )
-pipe.to("cuda")
-pipe.unet = torch.compile(pipe.unet, mode="reduce-overhead", fullgraph=True)
-max_64_bit_int = 2**63 - 1
+fps14Pipe.to("cuda")
+
+dragnuwaPipe = StableVideoDiffusionPipeline.from_pretrained(
+    "a-r-r-o-w/dragnuwa-svd", torch_dtype=torch.float16, variant="fp16", low_cpu_mem_usage=False, device_map=None
+)
+dragnuwaPipe.to("cuda")
 
+max_64_bit_int = 2**63 - 1
 
-def sample(
+def animate(
     image: Image,
     seed: Optional[int] = 42,
     randomize_seed: bool = True,
     motion_bucket_id: int = 127,
-    fps_id: int = 6,
-    version: str = "svd_xt",
-    cond_aug: float = 0.02,
-    decoding_t: int = 3,  # Number of frames decoded at a time! This eats most VRAM. Reduce if necessary.
-    device: str = "cuda",
-    output_folder: str = "outputs",
+    fps_id: int = 25,
+    noise_aug_strength: float = 0.1,
+    decoding_t: int = 3,
+    video_format: str = "mp4",
+    frame_format: str = "webp",
+    version: str = "auto",
+    width: int = 1024,
+    height: int = 576,
+    motion_control: bool = False,
+    num_inference_steps: int = 25
 ):
-    
-    if image.mode == "RGBA":
-        image = image.convert("RGB")
-    if(randomize_seed):
-        seed = random.randint(0, max_64_bit_int)
+    start = time.time()
+
+    if image is None:
+        raise gr.Error("Please provide an image to animate.")
+
+    output_folder = "outputs"
+    image_data = resize_image(image, output_size=(width, height))
+    if image_data.mode == "RGBA":
+        image_data = image_data.convert("RGB")
+
+    if motion_control:
+        image_data = [image_data] * 2
         
-    generator = torch.manual_seed(seed)
+    if randomize_seed:
+        seed = random.randint(0, max_64_bit_int)
+    
+    if version == "auto":
+        if 14 < fps_id:
+            version = "svdxt"
+        else:
+            version = "svd"
 
+    frames = animate_on_gpu(
+        image_data,
+        seed,
+        motion_bucket_id,
+        fps_id,
+        noise_aug_strength,
+        decoding_t,
+        version,
+        width,
+        height,
+        num_inference_steps
+    )
+    
     os.makedirs(output_folder, exist_ok=True)
-    base_count = len(glob(os.path.join(output_folder, "*.mp4")))
-    video_path = os.path.join(output_folder, f"{base_count:06d}.mp4")
-    frames = pipe(image, decode_chunk_size=decoding_t, generator=generator, motion_bucket_id=motion_bucket_id, noise_aug_strength=0.1, num_frames=25).frames[0]
-    export_to_video(frames, video_path, fps=fps_id)
-    torch.manual_seed(seed)
-    return video_path, seed
+    base_count = len(glob(os.path.join(output_folder, "*." + video_format)))
+    result_path = os.path.join(output_folder, f"{base_count:06d}." + video_format)
+
+    if video_format == "gif":
+        video_path = None
+        gif_path = result_path
+        export_to_gif(image=frames, output_gif_path=gif_path, fps=fps_id)
+    else:
+        video_path = result_path
+        gif_path = None
+        export_to_video(frames, video_path, fps=fps_id)
+    
+    end = time.time()
+    secondes = int(end - start)
+    minutes = math.floor(secondes / 60)
+    secondes = secondes - (minutes * 60)
+    hours = math.floor(minutes / 60)
+    minutes = minutes - (hours * 60)
+    information = ("Start the process again if you want a different result. " if randomize_seed else "") + \
+    "Wait 2 min before a new run to avoid quota penalty or use another computer. " + \
+    "The video has been generated in " + \
+    ((str(hours) + " h, ") if hours != 0 else "") + \
+    ((str(minutes) + " min, ") if hours != 0 or minutes != 0 else "") + \
+    str(secondes) + " sec."
+    
+    return [
+        # Display for video
+        gr.update(value = video_path, visible = video_format != "gif"),
+        # Display for gif
+        gr.update(value = gif_path, visible = video_format == "gif"),
+        # Download button
+        gr.update(label = "💾 Download animation in *." + video_format + " format", value=result_path, visible=True),
+        # Frames
+        gr.update(label = "Generated frames in *." + frame_format + " format", format = frame_format, value = frames, visible = True),
+        # Used seed
+        seed,
+        # Information
+        gr.update(value = information, visible = True),
+        # Reset button
+        gr.update(visible = True)
+    ]
+
+@torch.no_grad()
+@spaces.GPU(duration=180)
+def animate_on_gpu(
+    image_data: Union[Image.Image, List[Image.Image]],
+    seed: Optional[int] = 42,
+    motion_bucket_id: int = 127,
+    fps_id: int = 6,
+    noise_aug_strength: float = 0.1,
+    decoding_t: int = 3,
+    version: str = "svdxt",
+    width: int = 1024,
+    height: int = 576,
+    num_inference_steps: int = 25
+):
+    generator = torch.manual_seed(seed)
+
+    if version == "dragnuwa":
+        return dragnuwaPipe(image_data, width=width, height=height, decode_chunk_size=decoding_t, generator=generator, motion_bucket_id=motion_bucket_id, noise_aug_strength=noise_aug_strength, num_frames=25, num_inference_steps=num_inference_steps).frames[0]
+    elif version == "svdxt":
+        return fps25Pipe(image_data, width=width, height=height, decode_chunk_size=decoding_t, generator=generator, motion_bucket_id=motion_bucket_id, noise_aug_strength=noise_aug_strength, num_frames=25, num_inference_steps=num_inference_steps).frames[0]
+    else:
+        return fps14Pipe(image_data, width=width, height=height, decode_chunk_size=decoding_t, generator=generator, motion_bucket_id=motion_bucket_id, noise_aug_strength=noise_aug_strength, num_frames=25, num_inference_steps=num_inference_steps).frames[0]
+
 
 def resize_image(image, output_size=(1024, 576)):
+    # Do not touch the image if the size is good
+    if image.width == output_size[0] and image.height == output_size[1]:
+        return image
+
+    # Calculate aspect ratios
     target_aspect = output_size[0] / output_size[1]  # Aspect ratio of the desired size
     image_aspect = image.width / image.height  # Aspect ratio of the original image
 
+    # Resize if the original image is larger
     if image_aspect > target_aspect:
+        # Resize the image to match the target height, maintaining aspect ratio
         new_height = output_size[1]
         new_width = int(new_height * image_aspect)
         resized_image = image.resize((new_width, new_height), Image.LANCZOS)
+        # Calculate coordinates for cropping
         left = (new_width - output_size[0]) / 2
         top = 0
         right = (new_width + output_size[0]) / 2
         bottom = output_size[1]
     else:
+        # Resize the image to match the target width, maintaining aspect ratio
         new_width = output_size[0]
         new_height = int(new_width / image_aspect)
         resized_image = image.resize((new_width, new_height), Image.LANCZOS)
+        # Calculate coordinates for cropping
         left = 0
         top = (new_height - output_size[1]) / 2
         right = output_size[0]
         bottom = (new_height + output_size[1]) / 2
 
-    cropped_image = resized_image.crop((left, top, right, bottom))
-    return cropped_image
+    # Crop the image
+    return resized_image.crop((left, top, right, bottom))
+
+def reset():
+    return [
+        None,
+        random.randint(0, max_64_bit_int),
+        True,
+        127,
+        6,
+        0.1,
+        3,
+        "mp4",
+        "webp",
+        "auto",
+        1024,
+        576,
+        False,
+        25
+    ]
 
 with gr.Blocks() as demo:
+  gr.HTML("""
+    <h1><center>Image-to-Video</center></h1>
+    <big><center>Animate your image into 25 frames of 1024x576 pixels freely, without account, without watermark and download the video</center></big>
+    <br/>
     
-  gr.Markdown('''# Stable Video Diffusion using Image 2 Video XT ([model](https://huggingface.co/stabilityai/stable-video-diffusion-img2vid-xt), 
-      [paper](https://stability.ai/research/stable-video-diffusion-scaling-latent-video-diffusion-models-to-large-datasets), 
-      [stability's ui waitlist](https://stability.ai/contact))
-      #### Research release ([_non-commercial_](https://huggingface.co/stabilityai/stable-video-diffusion-img2vid-xt/blob/main/LICENSE)): generate `4s` vid from a single image at (`25 frames` at `6 fps`). this demo uses [🧨 diffusers for low VRAM and fast generation](https://huggingface.co/docs/diffusers/main/en/using-diffusers/svd).
-  ''')
-    
+    <p>
+    This demo is based on <i>Stable Video Diffusion</i> artificial intelligence.
+    No prompt or camera control is handled here.
+    To control motions, rather use <i><a href="https://huggingface.co/spaces/TencentARC/MotionCtrl_SVD">MotionCtrl SVD</a></i>.
+    If you need 128 frames, rather use <i><a href="https://huggingface.co/spaces/modelscope/ExVideo-SVD-128f-v1">ExVideo</a></i>.
+    </p>
+    """)
   with gr.Row():
+      with gr.Column():
+          image = gr.Image(label="Upload your image", type="pil")
+          with gr.Accordion("Advanced options", open=False):
+              width = gr.Slider(label="Width", info="Width of the video", value=1024, minimum=256, maximum=1024, step=8)
+              height = gr.Slider(label="Height", info="Height of the video", value=576, minimum=256, maximum=576, step=8)
+              motion_control = gr.Checkbox(label="Motion control (experimental)", info="Fix the camera", value=False)
+              video_format = gr.Radio([["*.mp4", "mp4"], ["*.avi", "avi"], ["*.wmv", "wmv"], ["*.mkv", "mkv"], ["*.mov", "mov"], ["*.gif", "gif"]], label="Video format for result", info="File extention", value="mp4", interactive=True)
+              frame_format = gr.Radio([["*.webp", "webp"], ["*.png", "png"], ["*.jpeg", "jpeg"], ["*.gif (unanimated)", "gif"], ["*.bmp", "bmp"]], label="Image format for frames", info="File extention", value="webp", interactive=True)
+              fps_id = gr.Slider(label="Frames per second", info="The length of your video in seconds will be 25/fps", value=25, minimum=5, maximum=30)
+              motion_bucket_id = gr.Slider(label="Motion bucket id", info="Controls how much motion to add/remove from the image", value=127, minimum=1, maximum=255)
+              noise_aug_strength = gr.Slider(label="Noise strength", info="The noise to add", value=0.1, minimum=0, maximum=1, step=0.1)
+              num_inference_steps = gr.Slider(label="Number inference steps", info="More denoising steps usually lead to a higher quality video at the expense of slower inference", value=25, minimum=1, maximum=100, step=1)
+              decoding_t = gr.Slider(label="Decoding", info="Number of frames decoded at a time; this eats more VRAM; reduce if necessary", value=3, minimum=1, maximum=5, step=1)
+              version = gr.Radio([["Auto", "auto"], ["🏃🏻‍♀️ SVD (trained on 14 f/s)", "svd"], ["🏃🏻‍♀️💨 SVD-XT (trained on 25 f/s)", "svdxt"]], label="Model", info="Trained model", value="auto", interactive=True)
+              seed = gr.Slider(label="Seed", value=42, randomize=True, minimum=0, maximum=max_64_bit_int, step=1)
+              randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
+
+          generate_btn = gr.Button(value="🚀 Animate", variant="primary")
+          reset_btn = gr.Button(value="🧹 Reinit page", variant="stop", elem_id="reset_button", visible = False)
+
+      with gr.Column():
+          video_output = gr.Video(label="Generated video", format="mp4", autoplay=True, show_download_button=False)
+          gif_output = gr.Image(label="Generated video", format="gif", show_download_button=False, visible=False)
+          download_button = gr.DownloadButton(label="💾 Download video", visible=False)
+          information_msg = gr.HTML(visible=False)
+          gallery = gr.Gallery(label="Generated frames", visible=False)
       
-    with gr.Column():
-        image = gr.Image(label="Upload your image", type="pil")
-        generate_btn = gr.Button("Generate")
-        
-    video = gr.Video()
-      
-  with gr.Accordion("Advanced options", open=False):
-      
-      seed = gr.Slider(label="Seed", value=42, randomize=True, minimum=0, maximum=max_64_bit_int, step=1)
-      randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
-      motion_bucket_id = gr.Slider(label="Motion bucket id", info="Controls how much motion to add/remove from the image", value=127, minimum=1, maximum=255)
-      fps_id = gr.Slider(label="Frames per second", info="The length of your video in seconds will be 25/fps", value=6, minimum=5, maximum=30)
-      
-  image.upload(fn=resize_image, inputs=image, outputs=image, queue=False)
-  generate_btn.click(fn=sample, inputs=[image, seed, randomize_seed, motion_bucket_id, fps_id], outputs=[video, seed], api_name="video")
+  generate_btn.click(fn=animate, inputs=[
+      image,
+      seed,
+      randomize_seed,
+      motion_bucket_id,
+      fps_id,
+      noise_aug_strength,
+      decoding_t,
+      video_format,
+      frame_format,
+      version,
+      width,
+      height,
+      motion_control,
+      num_inference_steps
+  ], outputs=[
+      video_output,
+      gif_output,
+      download_button,
+      gallery,
+      seed,
+      information_msg,
+      reset_btn
+  ], api_name="video")
+
+  reset_btn.click(fn = reset, inputs = [], outputs = [
+      image,
+      seed,
+      randomize_seed,
+      motion_bucket_id,
+      fps_id,
+      noise_aug_strength,
+      decoding_t,
+      video_format,
+      frame_format,
+      version,
+      width,
+      height,
+      motion_control,
+      num_inference_steps
+  ], queue = False, show_progress = False)
+    
   gr.Examples(
     examples=[
-        "images/01.png",
-        "images/02.png",
-        "images/03.png",
+        ["images/01.png", 42, True, 127, 25, 0.1, 3, "mp4", "png", "auto", 1024, 576, False, 25],
+        ["images/02.png", 42, True, 127, 25, 0.1, 3, "mp4", "png", "auto", 1024, 576, False, 25],
+        ["images/03.png", 42, True, 127, 25, 0.1, 3, "mp4", "png", "auto", 1024, 576, False, 25]
     ],
-    inputs=image,
-    outputs=[video, seed],
-    fn=sample,
-    cache_examples=True,
+    inputs=[image, seed, randomize_seed, motion_bucket_id, fps_id, noise_aug_strength, decoding_t, video_format, frame_format, version, width, height, motion_control, num_inference_steps],
+    outputs=[video_output, gif_output, download_button, gallery, seed, information_msg, reset_btn],
+    fn=animate,
+    run_on_click=True,
+    cache_examples=False,
   )
 
 if __name__ == "__main__":
-    demo.queue(max_size=20)
-    demo.launch(share=True)
+    demo.launch(share=True, show_api=False)

requirements.txt

@@ -4,4 +4,5 @@ transformers
 accelerate
 safetensors
 opencv-python
-uuid
+uuid
+torch