Create app.py

app.py

@@ -0,0 +1,144 @@
+import gradio as gr
+import torch
+import os
+import spaces
+import uuid
+
+from diffusers import AnimateDiffPipeline, MotionAdapter, EulerDiscreteScheduler
+from diffusers.utils import export_to_video
+from huggingface_hub import hf_hub_download
+from safetensors.torch import load_file
+from PIL import Image
+
+# Constants
+bases = {
+    "ToonYou": "frankjoshua/toonyou_beta6",
+    "epiCRealism": "emilianJR/epiCRealism"
+}
+step_loaded = None
+base_loaded = "ToonYou"
+motion_loaded = None
+
+# Ensure model and scheduler are initialized in GPU-enabled function
+if not torch.cuda.is_available():
+    raise NotImplementedError("No GPU detected!")
+
+device = "cuda"
+dtype = torch.float16
+pipe = AnimateDiffPipeline.from_pretrained(bases[base_loaded], torch_dtype=dtype).to(device)
+pipe.scheduler = EulerDiscreteScheduler.from_config(pipe.scheduler.config, timestep_spacing="trailing", beta_schedule="linear")
+
+# Safety checkers
+
+from transformers import CLIPFeatureExtractor
+
+
+feature_extractor = CLIPFeatureExtractor.from_pretrained("openai/clip-vit-base-patch32")
+
+
+
+# Function 
+@spaces.GPU(enable_queue=True)
+def generate_image(prompt, base, motion, step, progress=gr.Progress()):
+    global step_loaded
+    global base_loaded
+    global motion_loaded
+    print(prompt, base, step)
+
+    
+
+    if base_loaded != base:
+        pipe.unet.load_state_dict(torch.load(hf_hub_download(bases[base], "unet/diffusion_pytorch_model.bin"), map_location=device), strict=False)
+        base_loaded = base
+
+    if motion_loaded != motion:
+        pipe.unload_lora_weights()
+        if motion != "":
+            pipe.load_lora_weights(motion, adapter_name="motion")
+            pipe.set_adapters(["motion"], [0.7])
+        motion_loaded = motion
+
+    
+    output = pipe(prompt=prompt, guidance_scale=1.0, num_inference_steps=int(step))
+
+
+    name = str(uuid.uuid4()).replace("-", "")
+    path = f"/tmp/{name}.mp4"
+    export_to_video(output.frames[0], path, fps=10)
+    return path
+
+
+# Gradio Interface
+with gr.Blocks(css="style.css") as demo:
+    gr.HTML(
+        "<h1><center>Text To Video ⚡</center></h1>" +
+        "<p><center>SDXL text-to-video generation</center></p>" +
+        "<p><center><a href='https://github/ByteDance/AnimateDiff-Lightning'>https://huggingface.co/ByteDance/AnimateDiff-Lightning</a></center></p>"
+    )
+    with gr.Group():
+        with gr.Row():
+            prompt = gr.Textbox(
+                label='Prompt (Eng)'
+            )
+        with gr.Row():
+            select_base = gr.Dropdown(
+                label='Base model',
+v                choices=[
+                    "ToonYou", 
+                    "epiCRealism",
+                ],
+                value=base_loaded,
+                interactive=True
+            )
+            select_motion = gr.Dropdown(
+                label='Motion',
+                choices=[
+                    ("Default", ""),
+                    ("Zoom in", "guoyww/animatediff-motion-lora-zoom-in"),
+                    ("Zoom out", "guoyww/animatediff-motion-lora-zoom-out"),
+                    ("Tilt up", "guoyww/animatediff-motion-lora-tilt-up"),
+                    ("Tilt down", "guoyww/animatediff-motion-lora-tilt-down"),
+                    ("Pan left", "guoyww/animatediff-motion-lora-pan-left"),
+                    ("Pan right", "guoyww/animatediff-motion-lora-pan-right"),
+                    ("Roll left", "guoyww/animatediff-motion-lora-rolling-anticlockwise"),
+                    ("Roll right", "guoyww/animatediff-motion-lora-rolling-clockwise"),
+                ],
+                value="",
+                interactive=True
+            )
+            select_step = gr.Dropdown(
+                label='Inference steps',
+                choices=[
+                    ('1-Step', 1), 
+                    ('2-Step', 2),
+                    ('4-Step', 4),
+                    ('8-Step', 8),
+                    ('16-Step', 16),
+                    ('25-Step', 25)],
+                value=16,
+                interactive=True
+            )
+            submit = gr.Button(
+                scale=1,
+                variant='primary'
+            )
+    video = gr.Video(
+        label='Text To Video',
+        autoplay=True,
+        height=512,
+        width=512,
+        elem_id="video_output"
+    )
+
+    prompt.submit(
+        fn=generate_image,
+        inputs=[prompt, select_base, select_motion, select_step],
+        outputs=video,
+    )
+    submit.click(
+        fn=generate_image,
+        inputs=[prompt, select_base, select_motion, select_step],
+        outputs=video,
+    )
+
+demo.queue().launch()