Upload 3 files

Dockerfile

@@ -0,0 +1,43 @@
+FROM pytorch/pytorch:2.0.1-cuda11.7-cudnn8-runtime
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Set the MKL_THREADING_LAYER environment variable to GNU
+ENV MKL_THREADING_LAYER=GNU
+
+RUN apt-get update && apt-get install -y git wget libgl1-mesa-glx libglib2.0-0 ffmpeg libx264-dev build-essential cmake
+
+RUN useradd -m -u 1000 user
+
+USER user
+
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH \
+    PYTHONPATH=$HOME/app \
+	PYTHONUNBUFFERED=1 \
+	GRADIO_ALLOW_FLAGGING=never \
+	GRADIO_NUM_PORTS=1 \
+	GRADIO_SERVER_NAME=0.0.0.0 \
+	GRADIO_THEME=huggingface \
+    GRADIO_SHARE=False \
+	SYSTEM=spaces
+
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+RUN git clone -b dev https://github.com/fffiloni/dreamtalk $HOME/app 
+    
+RUN wget https://huggingface.co/camenduru/dreamtalk/resolve/main/damo/dreamtalk/checkpoints/denoising_network.pth -O $HOME/app/checkpoints/denoising_network.pth
+RUN wget https://huggingface.co/camenduru/dreamtalk/resolve/main/damo/dreamtalk/checkpoints/renderer.pt -O $HOME/app/checkpoints/renderer.pt
+    
+# Install dependencies
+RUN pip install --no-cache-dir urllib3==1.26.6 transformers==4.28.1 dlib yacs scipy scikit-image scikit-learn PyYAML Pillow numpy opencv-python imageio ffmpeg-python av moviepy gradio
+
+COPY app.py .
+
+# Set the environment variable to specify the GPU device
+ENV CUDA_DEVICE_ORDER=PCI_BUS_ID
+ENV CUDA_VISIBLE_DEVICES=0
+
+# Run your app.py script
+CMD ["python", "app.py"]

app (14).py

@@ -0,0 +1,181 @@
+import gradio as gr
+import subprocess
+from moviepy.editor import VideoFileClip
+import datetime
+
+def convert_to_mp4_with_aac(input_path, output_path):
+    # Load the video
+    video = VideoFileClip(input_path)
+    
+    # Set the output format to mp4 with AAC codec
+    video.write_videofile(output_path, codec="libx264", audio_codec="aac")
+
+    return output_path
+
+
+# Function to check if the audio file path exists in the list
+def check_file_exists(file_path, audio_list):
+    return file_path in audio_list
+
+def load_audio(audio_listed):
+    if audio_listed is None:
+        return None
+    else:
+        return f"data/audio/{audio_listed}"
+
+def execute_command(command: str) -> None:
+    subprocess.run(command, check=True)
+
+def infer(audio_input, image_path, emotional_style):
+    # Get the current timestamp
+    timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
+
+    output_name = f"lipsynced_result_{timestamp}"
+
+    command = [
+        f"python",
+        f"inference_for_demo_video.py",
+        f"--wav_path={audio_input}",
+        f"--style_clip_path=data/style_clip/3DMM/{emotional_style}",
+        f"--pose_path=data/pose/RichardShelby_front_neutral_level1_001.mat",
+        f"--image_path={image_path}",
+        f"--cfg_scale=1.0",
+        f"--max_gen_len=30",
+        f"--output_name={output_name}"
+    ]
+
+    execute_command(command)
+
+    # Convert video to compatible codecs
+    input_file = f"output_video/{output_name}.mp4"
+    output_file = f"{output_name}.mp4"
+    
+    result = convert_to_mp4_with_aac(input_file, output_file)
+    
+    return result
+
+css="""
+#col-container{
+    margin: 0 auto;
+    max-width: 940px;
+}
+#project-links{
+    margin: 0 0 12px !important;
+    column-gap: 8px;
+    display: flex;
+    justify-content: center;
+    flex-wrap: nowrap;
+    flex-direction: row;
+    align-items: center;
+}
+#run-btn{
+    border: var(--button-border-width) solid var(--button-primary-border-color);
+    background: var(--button-primary-background-fill);
+    color: var(--button-primary-text-color);
+}
+#run-btn:hover{
+    border-color: var(--button-primary-border-color-hover);
+    background: var(--button-primary-background-fill-hover);
+    color: var(--button-primary-text-color-hover);
+}
+"""
+with gr.Blocks(css=css) as demo:
+    with gr.Column(elem_id="col-container"):
+        gr.HTML("""
+        <h2 style="text-align: center;">DreamTalk</h2>
+        <p style="text-align: center;">When Expressive Talking Head Generation Meets Diffusion Probabilistic Models</p>
+        <p style="margin:12px auto;display: flex;justify-content: center;">
+            <a href="https://huggingface.co/spaces/fffiloni/dreamtalk?duplicate=true"><img src="https://huggingface.co/datasets/huggingface/badges/resolve/main/duplicate-this-space-lg.svg" alt="Duplicate this Space"></a>
+        </p>
+        
+        """)
+        with gr.Row():
+            with gr.Column():
+                image_path = gr.Image(label="Image", type="filepath", sources=["upload"])
+                audio_input = gr.Audio(label="Audio input", type="filepath", sources=["upload"], value="data/audio/acknowledgement_english.m4a")
+                with gr.Row():
+                    audio_list = gr.Dropdown(
+                        label="Choose an audio (optional)",
+                        choices=[
+                            "German1.wav", "German2.wav", "German3.wav", "German4.wav",
+                            "acknowledgement_chinese.m4a", "acknowledgement_english.m4a",
+                            "chinese1_haierlizhi.wav", "chinese2_guanyu.wav",
+                            "french1.wav", "french2.wav", "french3.wav",
+                            "italian1.wav", "italian2.wav", "italian3.wav",
+                            "japan1.wav", "japan2.wav", "japan3.wav",
+                            "korean1.wav", "korean2.wav", "korean3.wav",
+                            "noisy_audio_cafeter_snr_0.wav", "noisy_audio_meeting_snr_0.wav", "noisy_audio_meeting_snr_10.wav", "noisy_audio_meeting_snr_20.wav", "noisy_audio_narrative.wav", "noisy_audio_office_snr_0.wav", "out_of_domain_narrative.wav",
+                            "spanish1.wav", "spanish2.wav", "spanish3.wav"
+                            ],
+                        value = "acknowledgement_english.m4a"
+                    )
+                    audio_list.change(
+                        fn = load_audio,
+                        inputs = [audio_list],
+                        outputs = [audio_input]
+                    )
+                    emotional_style = gr.Dropdown(
+                        label = "emotional style",
+                        choices = [
+                            "M030_front_angry_level3_001.mat",
+                            "M030_front_contempt_level3_001.mat",
+                            "M030_front_disgusted_level3_001.mat",
+                            "M030_front_fear_level3_001.mat",
+                            "M030_front_happy_level3_001.mat",
+                            "M030_front_neutral_level1_001.mat",
+                            "M030_front_sad_level3_001.mat",
+                            "M030_front_surprised_level3_001.mat",
+                            "W009_front_angry_level3_001.mat",
+                            "W009_front_contempt_level3_001.mat",
+                            "W009_front_disgusted_level3_001.mat",
+                            "W009_front_fear_level3_001.mat",
+                            "W009_front_happy_level3_001.mat",
+                            "W009_front_neutral_level1_001.mat",
+                            "W009_front_sad_level3_001.mat",
+                            "W009_front_surprised_level3_001.mat",
+                            "W011_front_angry_level3_001.mat",
+                            "W011_front_contempt_level3_001.mat",
+                            "W011_front_disgusted_level3_001.mat",
+                            "W011_front_fear_level3_001.mat",
+                            "W011_front_happy_level3_001.mat",
+                            "W011_front_neutral_level1_001.mat",
+                            "W011_front_sad_level3_001.mat",
+                            "W011_front_surprised_level3_001.mat"
+                        ],
+                        value = "M030_front_neutral_level1_001.mat"
+                    )
+                gr.Examples(
+                    examples = [
+                        "data/src_img/uncropped/face3.png",
+                        "data/src_img/uncropped/male_face.png",
+                        "data/src_img/uncropped/uncut_src_img.jpg",
+                        "data/src_img/cropped/chpa5.png",
+                        "data/src_img/cropped/cut_img.png",
+                        "data/src_img/cropped/f30.png",
+                        "data/src_img/cropped/menglu2.png",
+                        "data/src_img/cropped/nscu2.png",
+                        "data/src_img/cropped/zp1.png",
+                        "data/src_img/cropped/zt12.png"
+                    ],
+                    inputs=[image_path],
+                    examples_per_page=5
+                )
+                with gr.Row():
+                    gr.ClearButton([audio_input, image_path, audio_list])
+                    run_btn = gr.Button("Run", elem_id="run-btn")
+            with gr.Column():
+                output_video = gr.Video(format="mp4")
+                gr.HTML("""
+                <p id="project-links" align="center">
+                  <a href='https://dreamtalk-project.github.io/'><img src='https://img.shields.io/badge/Project-Page-Green'></a> <a href='https://arxiv.org/abs/2312.09767'><img src='https://img.shields.io/badge/Paper-Arxiv-red'></a> <a href='https://youtu.be/VF4vlE6ZqWQ'><img src='https://badges.aleen42.com/src/youtube.svg'></a>
+                </p>
+                <img src="https://github.com/ali-vilab/dreamtalk/raw/main/media/teaser.gif" style="margin: 0 auto;border-radius: 10px;" />    
+                """)
+    
+    run_btn.click(
+        fn = infer,
+        inputs = [audio_input, image_path, emotional_style],
+        outputs = [output_video]
+    )
+
+demo.queue().launch()

gitattributes (1)

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text