File size: 8,770 Bytes
8d0320b
6877edc
75517c0
8d0320b
6d490a9
af6eab9
923fdda
8d0320b
 
ae3f094
8d0320b
 
d20794a
6d490a9
8d0320b
fc61926
8d0320b
fc61926
8d0320b
 
 
aa9bb98
8d0320b
 
af6eab9
b3e0151
 
af6eab9
6e0e19c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6d490a9
 
 
 
 
 
 
 
 
 
fbd6bad
6d490a9
 
 
 
8d0320b
6d490a9
 
 
 
 
 
 
 
 
 
8d0320b
6d490a9
 
 
 
 
 
 
 
 
 
8d0320b
6d490a9
 
8d0320b
6d490a9
 
 
8d0320b
6d490a9
 
 
 
 
 
8d0320b
 
6d490a9
8d0320b
6d490a9
 
 
75517c0
 
 
 
7ac5c7d
49f95b1
23be978
 
 
6877edc
23be978
 
 
6877edc
23be978
 
 
 
b361117
23be978
 
6877edc
8d0320b
 
 
6877edc
23be978
6877edc
8d0320b
23be978
8d0320b
 
23be978
75517c0
23be978
6d490a9
4fe6158
23be978
75517c0
23be978
49f95b1
23be978
8d0320b
4fe6158
 
49f95b1
4fe6158
23be978
4fe6158
23be978
 
8d0320b
 
23be978
8d0320b
23be978
 
 
 
8d0320b
23be978
8d0320b
23be978
 
 
8d0320b
6877edc
8d0320b
49f95b1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
590ffd9
 
49f95b1
 
590ffd9
49f95b1
 
4bc5468
49f95b1
 
 
 
 
4bc5468
6d490a9
 
a47bd89
cde7b9e
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
import os
import uuid
import asyncio
import subprocess
import json
from zipfile import ZipFile
import stat
import gradio as gr
import ffmpeg
import cv2
import edge_tts
from googletrans import Translator
from huggingface_hub import HfApi
import moviepy.editor as mp
import spaces

# Constants and initialization
HF_TOKEN = os.environ.get("HF_TOKEN")
REPO_ID = "artificialguybr/video-dubbing"
MAX_VIDEO_DURATION = 60  # seconds

api = HfApi(token=HF_TOKEN)

# Extract and set permissions for ffmpeg
ZipFile("ffmpeg.zip").extractall()
st = os.stat('ffmpeg')
os.chmod('ffmpeg', st.st_mode | stat.S_IEXEC)

language_mapping = {
    'English': ('en', 'en-US-EricNeural'),
    'Spanish': ('es', 'es-ES-AlvaroNeural'),
    'French': ('fr', 'fr-FR-HenriNeural'),
    'German': ('de', 'de-DE-ConradNeural'),
    'Italian': ('it', 'it-IT-DiegoNeural'),
    'Portuguese': ('pt', 'pt-PT-DuarteNeural'),
    'Polish': ('pl', 'pl-PL-MarekNeural'),
    'Turkish': ('tr', 'tr-TR-AhmetNeural'),
    'Russian': ('ru', 'ru-RU-DmitryNeural'),
    'Dutch': ('nl', 'nl-NL-MaartenNeural'),
    'Czech': ('cs', 'cs-CZ-AntoninNeural'),
    'Arabic': ('ar', 'ar-SA-HamedNeural'),
    'Chinese (Simplified)': ('zh-CN', 'zh-CN-YunxiNeural'),
    'Japanese': ('ja', 'ja-JP-KeitaNeural'),
    'Korean': ('ko', 'ko-KR-InJoonNeural'),
    'Hindi': ('hi', 'hi-IN-MadhurNeural'),
    'Swedish': ('sv', 'sv-SE-MattiasNeural'),
    'Danish': ('da', 'da-DK-JeppeNeural'),
    'Finnish': ('fi', 'fi-FI-HarriNeural'),
    'Greek': ('el', 'el-GR-NestorasNeural')
}

print("Starting the program...")

def generate_unique_filename(extension):
    return f"{uuid.uuid4()}{extension}"

def cleanup_files(*files):
    for file in files:
        if file and os.path.exists(file):
            os.remove(file)
            print(f"Removed file: {file}")

@spaces.GPU(duration=90)
def transcribe_audio(file_path):
    print(f"Starting transcription of file: {file_path}")
    temp_audio = None
    
    if file_path.endswith(('.mp4', '.avi', '.mov', '.flv')):
        print("Video file detected. Extracting audio...")
        try:
            video = mp.VideoFileClip(file_path)
            temp_audio = generate_unique_filename(".wav")
            video.audio.write_audiofile(temp_audio)
            file_path = temp_audio
        except Exception as e:
            print(f"Error extracting audio from video: {e}")
            raise

    output_file = generate_unique_filename(".json")
    command = [
        "insanely-fast-whisper",
        "--file-name", file_path,
        "--device-id", "0",
        "--model-name", "openai/whisper-large-v3",
        "--task", "transcribe",
        "--timestamp", "chunk",
        "--transcript-path", output_file
    ]
    
    try:
        result = subprocess.run(command, check=True, capture_output=True, text=True)
        print(f"Transcription output: {result.stdout}")
    except subprocess.CalledProcessError as e:
        print(f"Error running insanely-fast-whisper: {e}")
        raise

    try:
        with open(output_file, "r") as f:
            transcription = json.load(f)
    except json.JSONDecodeError as e:
        print(f"Error decoding JSON: {e}")
        raise

    result = transcription.get("text", " ".join([chunk["text"] for chunk in transcription.get("chunks", [])]))
    
    cleanup_files(output_file, temp_audio)
    
    return result

async def text_to_speech(text, voice, output_file):
    communicate = edge_tts.Communicate(text, voice)
    await communicate.save(output_file)

@spaces.GPU
def process_video(video, target_language, use_wav2lip):
    try:
        if target_language is None:
            raise ValueError("Please select a Target Language for Dubbing.")
        
        run_uuid = uuid.uuid4().hex[:6]
        output_filename = f"{run_uuid}_resized_video.mp4"
        ffmpeg.input(video).output(output_filename, vf='scale=-2:720').run()

        video_path = output_filename
        
        if not os.path.exists(video_path):
            raise FileNotFoundError(f"Error: {video_path} does not exist.")

        video_info = ffmpeg.probe(video_path)
        video_duration = float(video_info['streams'][0]['duration'])

        if video_duration > MAX_VIDEO_DURATION:
            cleanup_files(video_path)
            raise ValueError(f"Video duration exceeds {MAX_VIDEO_DURATION} seconds. Please upload a shorter video.")

        ffmpeg.input(video_path).output(f"{run_uuid}_output_audio.wav", acodec='pcm_s24le', ar=48000, map='a').run()

        subprocess.run(f"ffmpeg -y -i {run_uuid}_output_audio.wav -af lowpass=3000,highpass=100 {run_uuid}_output_audio_final.wav", shell=True, check=True)
        
        whisper_text = transcribe_audio(f"{run_uuid}_output_audio_final.wav")
        print(f"Transcription successful: {whisper_text}")
                
        target_language_code, voice = language_mapping[target_language]
        translator = Translator()
        translated_text = translator.translate(whisper_text, dest=target_language_code).text
        print(f"Translated text: {translated_text}")

        asyncio.run(text_to_speech(translated_text, voice, f"{run_uuid}_output_synth.wav"))
        
        if use_wav2lip:
            try:
                subprocess.run(f"python Wav2Lip/inference.py --checkpoint_path 'Wav2Lip/checkpoints/wav2lip_gan.pth' --face '{video_path}' --audio '{run_uuid}_output_synth.wav' --pads 0 15 0 0 --resize_factor 1 --nosmooth --outfile '{run_uuid}_output_video.mp4'", shell=True, check=True)
            except subprocess.CalledProcessError as e:
                print(f"Wav2Lip error: {str(e)}")
                gr.Warning("Wav2lip encountered an error. Falling back to simple audio replacement.")
                subprocess.run(f"ffmpeg -i {video_path} -i {run_uuid}_output_synth.wav -c:v copy -c:a aac -strict experimental -map 0:v:0 -map 1:a:0 {run_uuid}_output_video.mp4", shell=True, check=True)
        else:
            subprocess.run(f"ffmpeg -i {video_path} -i {run_uuid}_output_synth.wav -c:v copy -c:a aac -strict experimental -map 0:v:0 -map 1:a:0 {run_uuid}_output_video.mp4", shell=True, check=True)

        output_video_path = f"{run_uuid}_output_video.mp4"
        if not os.path.exists(output_video_path):
            raise FileNotFoundError(f"Error: {output_video_path} was not generated.")

        cleanup_files(
            f"{run_uuid}_resized_video.mp4",
            f"{run_uuid}_output_audio.wav",
            f"{run_uuid}_output_audio_final.wav",
            f"{run_uuid}_output_synth.wav"
        )

        return output_video_path, ""

    except Exception as e:
        print(f"Error in process_video: {str(e)}")
        return None, f"Error: {str(e)}"

# Gradio interface setup
with gr.Blocks(theme=gr.themes.Soft()) as demo:
    gr.Markdown("# AI Video Dubbing")
    gr.Markdown("This tool uses AI to dub videos into different languages. Upload a video, choose a target language, and get a dubbed version!")
    
    with gr.Row():
        with gr.Column(scale=2):
            video_input = gr.Video(label="Upload Video")
            target_language = gr.Dropdown(
                choices=list(language_mapping.keys()), 
                label="Target Language for Dubbing", 
                value="Spanish"
            )
            use_wav2lip = gr.Checkbox(
                label="Use Wav2Lip for lip sync", 
                value=False, 
                info="Enable this if the video has close-up faces. May not work for all videos."
            )
            submit_button = gr.Button("Process Video", variant="primary")
        
        with gr.Column(scale=2):
            output_video = gr.Video(label="Processed Video")
            error_message = gr.Textbox(label="Status/Error Message")

    submit_button.click(
        process_video, 
        inputs=[video_input, target_language, use_wav2lip], 
        outputs=[output_video, error_message]
    )

    gr.Markdown("""
    ## Notes:
    - Video limit is 10 minute. The tool will dub all speakers using a single voice.
    - Processing may take up to 50 minutes.
    - This is an alpha version using open-source models.
    - Quality vs. speed trade-off was made for scalability and hardware limitations.
    - For videos longer than 10 minute, please duplicate this Space and adjust the limit in the code.
    """)

    gr.Markdown("""
    ---
    Developed by [@artificialguybr](https://twitter.com/artificialguybr) using open-source tools. 
    Special thanks to Hugging Face for GPU support and [@yeswondwer](https://twitter.com/@yeswondwerr) for the original code.
    
    Try our [Video Transcription and Translation](https://huggingface.co/spaces/artificialguybr/VIDEO-TRANSLATION-TRANSCRIPTION) tool!
    """)

print("Launching Gradio interface...")
demo.queue()
demo.launch()