Latent audio lipsync

README.md

@@ -4,7 +4,7 @@ emoji: 👄
 colorFrom: blue
 colorTo: blue
 sdk: gradio
-sdk_version: 5.33.2
+sdk_version: 5.12.0
 app_file: app.py
 pinned: false
 short_description: Audio Conditioned LipSync with Latent Diffusion Models

app.py

@@ -1,5 +1,4 @@
 import gradio as gr
-import spaces
 import os
 import sys
 import shutil
@@ -80,26 +79,7 @@ from accelerate.utils import set_seed
 from latentsync.whisper.audio2feature import Audio2Feature
 
 
-@spaces.GPU(duration=180)
 def main(video_path, audio_path, progress=gr.Progress(track_tqdm=True)):
-    """
-    Perform lip-sync video generation using an input video and a separate audio track.
-    
-    This function takes an input video (usually a person speaking) and an audio file,
-    and synchronizes the video frames so that the lips of the speaker match the audio content.
-    It uses a latent diffusion model-based pipeline (LatentSync) for audio-conditioned lip synchronization.
-    
-    Args:
-        video_path (str): File path to the input video in MP4 format.
-        audio_path (str): File path to the input audio file (e.g., WAV or MP3).
-        progress (gr.Progress, optional): Gradio progress tracker for UI feedback (auto-injected).
-        
-    Returns:
-        str: File path to the generated output video with lip synchronization applied.
-    """
-
-    gr.Info("180 seconds will be used from your daily ZeroGPU time credits.")
-    
     inference_ckpt_path = "checkpoints/latentsync_unet.pt"
     unet_config_path = "configs/unet/second_stage.yaml"
     config = OmegaConf.load(unet_config_path)
@@ -143,12 +123,9 @@ def main(video_path, audio_path, progress=gr.Progress(track_tqdm=True)):
 
     unet = unet.to(dtype=torch.float16)
 
-    """
     # set xformers
-    
     if is_xformers_available():
         unet.enable_xformers_memory_efficient_attention()
-    """
 
     pipeline = LipsyncPipeline(
         vae=vae,
@@ -239,4 +216,4 @@ with gr.Blocks(css=css) as demo:
         outputs = [video_result]
     )
 
-demo.queue().launch(show_api=True, show_error=True, ssr_mode=False, mcp_server=True)
+demo.queue().launch(show_api=False, show_error=True)

latentsync/models/attention.py

@@ -9,10 +9,10 @@ import torch.nn.functional as F
 from torch import nn
 
 from diffusers.configuration_utils import ConfigMixin, register_to_config
-from diffusers.models.modeling_utils import ModelMixin
+from diffusers.modeling_utils import ModelMixin
 from diffusers.utils import BaseOutput
 from diffusers.utils.import_utils import is_xformers_available
-from diffusers.models.attention import Attention as CrossAttention, FeedForward, AdaLayerNorm
+from diffusers.models.attention import CrossAttention, FeedForward, AdaLayerNorm
 
 from einops import rearrange, repeat
 from .utils import zero_module

latentsync/models/motion_module.py

@@ -11,10 +11,10 @@ import torch.nn.functional as F
 from torch import nn
 
 from diffusers.configuration_utils import ConfigMixin, register_to_config
-from diffusers.models.modeling_utils import ModelMixin
+from diffusers.modeling_utils import ModelMixin
 from diffusers.utils import BaseOutput
 from diffusers.utils.import_utils import is_xformers_available
-from diffusers.models.attention import Attention as CrossAttention, FeedForward
+from diffusers.models.attention import CrossAttention, FeedForward
 
 from einops import rearrange, repeat
 import math

latentsync/models/unet.py

@@ -9,7 +9,7 @@ import torch.nn as nn
 import torch.utils.checkpoint
 
 from diffusers.configuration_utils import ConfigMixin, register_to_config
-from diffusers.models.modeling_utils import ModelMixin
+from diffusers.modeling_utils import ModelMixin
 from diffusers import UNet2DConditionModel
 from diffusers.utils import BaseOutput, logging
 from diffusers.models.embeddings import TimestepEmbedding, Timesteps

latentsync/pipelines/lipsync_pipeline.py

@@ -15,7 +15,7 @@ from packaging import version
 
 from diffusers.configuration_utils import FrozenDict
 from diffusers.models import AutoencoderKL
-from diffusers.pipelines.pipeline_utils import DiffusionPipeline
+from diffusers.pipeline_utils import DiffusionPipeline
 from diffusers.schedulers import (
     DDIMScheduler,
     DPMSolverMultistepScheduler,

requirements.txt

@@ -1,18 +1,18 @@
-torch==2.5.1
-torchvision==0.20.1
+torch==2.2.2
+torchvision==0.17.2
 --extra-index-url https://download.pytorch.org/whl/cu121
-xformers==0.0.29.post1
-triton==3.1.0
+xformers==0.0.26
+triton==2.2.0
 
-diffusers==0.33.1
-transformers==4.52.3
-huggingface-hub==0.32.2
+diffusers==0.11.1
+transformers==4.38.0
+huggingface-hub==0.25.2
 imageio==2.27.0
 decord==0.6.0
 accelerate==0.26.1
 einops==0.7.0
 omegaconf==2.3.0
-safetensors>=0.4.3
+safetensors==0.4.2
 opencv-python==4.9.0.80
 mediapipe==0.10.11
 av==11.0.0
@@ -27,7 +27,5 @@ face-alignment==1.4.1
 ninja==1.11.1.1
 pandas==2.0.3
 numpy==1.24.4
-pydub==0.25.1
-moviepy==1.0.3
-spaces
-gradio[mcp]
+pydub
+moviepy==1.0.3