Update app.py

app.py

@@ -22,24 +22,18 @@ from tqdm import tqdm
 from numba import jit
 from huggingface_hub import HfApi
 
-# Environment setup
 HF_TOKEN = os.environ.get("HF_TOKEN")
 os.environ["COQUI_TOS_AGREED"] = "1"
 api = HfApi(token=HF_TOKEN)
 repo_id = "artificialguybr/video-dubbing"
-
-# Extract ffmpeg
 ZipFile("ffmpeg.zip").extractall()
 st = os.stat('ffmpeg')
 os.chmod('ffmpeg', st.st_mode | stat.S_IEXEC)
 
-# Initialize Whisper model
+#Whisper
 model_size = "small"
 model = WhisperModel(model_size, device="cpu", compute_type="int8")
 
-# Initialize TTS model
-tts = TTS("tts_models/multilingual/multi-dataset/xtts_v2")
-
 def check_for_faces(video_path):
     face_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')
     cap = cv2.VideoCapture(video_path)
@@ -57,17 +51,6 @@ def check_for_faces(video_path):
 
     return False
 
-@spaces.GPU
-def transcribe_audio(audio_path):
-    segments, info = model.transcribe(audio_path, beam_size=5)
-    whisper_text = " ".join(segment.text for segment in segments)
-    whisper_language = info.language
-    return whisper_text, whisper_language
-
-@spaces.GPU
-def generate_tts(text, speaker_wav, language_code):
-    tts.tts_to_file(text, speaker_wav=speaker_wav, file_path="output_synth.wav", language=language_code)
-
 @spaces.GPU
 def process_video(radio, video, target_language, has_closeup_face):
     if target_language is None:
@@ -96,12 +79,14 @@ def process_video(radio, video, target_language, has_closeup_face):
     
     print("Attempting to transcribe with Whisper...")
     try:
-        whisper_text, whisper_language = transcribe_audio(f"{run_uuid}_output_audio_final.wav")
+        segments, info = model.transcribe(f"{run_uuid}_output_audio_final.wav", beam_size=5)
+        whisper_text = " ".join(segment.text for segment in segments)
+        whisper_language = info.language
         print(f"Transcription successful: {whisper_text}")
     except RuntimeError as e:
         print(f"RuntimeError encountered: {str(e)}")
         if "CUDA failed with error device-side assert triggered" in str(e):
-            gr.Warning("Error. Space needs to restart. Please retry in a minute")
+            gr.Warning("Error. Space need to restart. Please retry in a minute")
             api.restart_space(repo_id=repo_id)
             
     language_mapping = {'English': 'en', 'Spanish': 'es', 'French': 'fr', 'German': 'de', 'Italian': 'it', 'Portuguese': 'pt', 'Polish': 'pl', 'Turkish': 'tr', 'Russian': 'ru', 'Dutch': 'nl', 'Czech': 'cs', 'Arabic': 'ar', 'Chinese (Simplified)': 'zh-cn'}
@@ -110,19 +95,34 @@ def process_video(radio, video, target_language, has_closeup_face):
     translated_text = translator.translate(whisper_text, src=whisper_language, dest=target_language_code).text
     print(translated_text)
 
-    generate_tts(translated_text, f"{run_uuid}_output_audio_final.wav", target_language_code)
+    tts = TTS("tts_models/multilingual/multi-dataset/xtts_v2")
+    tts.to('cuda')
+    tts.tts_to_file(translated_text, speaker_wav=f"{run_uuid}_output_audio_final.wav", file_path=f"{run_uuid}_output_synth.wav", language=target_language_code)
     
+    pad_top = 0
+    pad_bottom = 15
+    pad_left = 0
+    pad_right = 0
+    rescaleFactor = 1
+
+    video_path_fix = video_path
+
+    if has_closeup_face:
+        has_face = True
+    else:
+        has_face = check_for_faces(video_path)
+
     if has_closeup_face:
         try:
-            cmd = f"python Wav2Lip/inference.py --checkpoint_path 'Wav2Lip/checkpoints/wav2lip_gan.pth' --face {shlex.quote(video_path)} --audio 'output_synth.wav' --pads 0 15 0 0 --resize_factor 1 --nosmooth --outfile '{run_uuid}_output_video.mp4'"
+            cmd = f"python Wav2Lip/inference.py --checkpoint_path 'Wav2Lip/checkpoints/wav2lip_gan.pth' --face {shlex.quote(video_path)} --audio '{run_uuid}_output_synth.wav' --pads {pad_top} {pad_bottom} {pad_left} {pad_right} --resize_factor {rescaleFactor} --nosmooth --outfile '{run_uuid}_output_video.mp4'"
             subprocess.run(cmd, shell=True, check=True)
         except subprocess.CalledProcessError as e:
             if "Face not detected! Ensure the video contains a face in all the frames." in str(e.stderr):
                 gr.Warning("Wav2lip didn't detect a face. Please try again with the option disabled.")
-                cmd = f"ffmpeg -i {video_path} -i output_synth.wav -c:v copy -c:a aac -strict experimental -map 0:v:0 -map 1:a:0 {run_uuid}_output_video.mp4"
+                cmd = f"ffmpeg -i {video_path} -i {run_uuid}_output_synth.wav -c:v copy -c:a aac -strict experimental -map 0:v:0 -map 1:a:0 {run_uuid}_output_video.mp4"
                 subprocess.run(cmd, shell=True)
     else:
-        cmd = f"ffmpeg -i {video_path} -i output_synth.wav -c:v copy -c:a aac -strict experimental -map 0:v:0 -map 1:a:0 {run_uuid}_output_video.mp4"
+        cmd = f"ffmpeg -i {video_path} -i {run_uuid}_output_synth.wav -c:v copy -c:a aac -strict experimental -map 0:v:0 -map 1:a:0 {run_uuid}_output_video.mp4"
         subprocess.run(cmd, shell=True)
 
     if not os.path.exists(f"{run_uuid}_output_video.mp4"):
@@ -130,12 +130,11 @@ def process_video(radio, video, target_language, has_closeup_face):
 
     output_video_path = f"{run_uuid}_output_video.mp4"
 
-    # Cleanup
     files_to_delete = [
         f"{run_uuid}_resized_video.mp4",
         f"{run_uuid}_output_audio.wav",
         f"{run_uuid}_output_audio_final.wav",
-        "output_synth.wav"
+        f"{run_uuid}_output_synth.wav"
     ]
     for file in files_to_delete:
         try:
@@ -144,11 +143,13 @@ def process_video(radio, video, target_language, has_closeup_face):
             print(f"File {file} not found for deletion.")
 
     return output_video_path
-
+    
 def swap(radio):
-    return gr.update(source="upload" if radio == "Upload" else "webcam")
-
-# Gradio interface setup
+    if(radio == "Upload"):
+        return gr.update(source="upload")
+    else:
+        return gr.update(source="webcam")
+        
 video = gr.Video()
 radio = gr.Radio(["Upload", "Record"], value="Upload", show_label=False)
 iface = gr.Interface(
@@ -157,7 +158,10 @@ iface = gr.Interface(
         radio,
         video,
         gr.Dropdown(choices=["English", "Spanish", "French", "German", "Italian", "Portuguese", "Polish", "Turkish", "Russian", "Dutch", "Czech", "Arabic", "Chinese (Simplified)"], label="Target Language for Dubbing", value="Spanish"),
-        gr.Checkbox(label="Video has a close-up face. Use Wav2lip.", value=False, info="Say if video have close-up face. For Wav2lip. Will not work if checked wrongly.")
+        gr.Checkbox(
+                    label="Video has a close-up face. Use Wav2lip.",
+                    value=False,
+                    info="Say if video have close-up face. For Wav2lip. Will not work if checked wrongly.")
     ],
     outputs=gr.Video(),
     live=False,
@@ -165,20 +169,18 @@ iface = gr.Interface(
     description="""This tool was developed by [@artificialguybr](https://twitter.com/artificialguybr) using entirely open-source tools. Special thanks to Hugging Face for the GPU support. Thanks [@yeswondwer](https://twitter.com/@yeswondwerr) for original code. Test the [Video Transcription and Translate](https://huggingface.co/spaces/artificialguybr/VIDEO-TRANSLATION-TRANSCRIPTION) space!""",
     allow_flagging=False
 )
-
 with gr.Blocks() as demo:
     iface.render()
     radio.change(swap, inputs=[radio], outputs=video)
     gr.Markdown("""
     **Note:**
-    - Video limit is 1 minute. It will dubbing all people using just one voice.
+    - Video limit is 1 minute. It will dubbling all people using just one voice.
     - Generation may take up to 5 minutes.
     - By using this demo you agree to the terms of the Coqui Public Model License at https://coqui.ai/cpml
-    - The tool uses open-source models for all models. It's an alpha version.
+    - The tool uses open-source models for all models. It's a alpha version.
     - Quality can be improved but would require more processing time per video. For scalability and hardware limitations, speed was chosen, not just quality.
     - If you need more than 1 minute, duplicate the Space and change the limit on app.py.
     - If you incorrectly mark the 'Video has a close-up face' checkbox, the dubbing may not work as expected.
     """)
-
 demo.queue(concurrency_count=1, max_size=15)
 demo.launch()