import os import shutil from huggingface_hub import snapshot_download import gradio as gr from gradio_client import Client, handle_file from mutagen.mp3 import MP3 from pydub import AudioSegment from PIL import Image import ffmpeg os.chdir(os.path.dirname(os.path.abspath(__file__))) from scripts.inference import inference_process import argparse import uuid is_shared_ui = True if "fffiloni/tts-hallo-talking-portrait" in os.environ['SPACE_ID'] else False hallo_dir = snapshot_download(repo_id="fudan-generative-ai/hallo", local_dir="pretrained_models") AUDIO_MAX_DURATION = 4000 ############# # UTILITIES # ############# def is_mp3(file_path): try: audio = MP3(file_path) return True except Exception as e: return False def convert_mp3_to_wav(mp3_file_path, wav_file_path): # Load the MP3 file audio = AudioSegment.from_mp3(mp3_file_path) # Export as WAV file audio.export(wav_file_path, format="wav") return wav_file_path def trim_audio(file_path, output_path, max_duration): # Load the audio file audio = AudioSegment.from_wav(file_path) # Check the length of the audio in milliseconds audio_length = len(audio) # If the audio is longer than the maximum duration, trim it if audio_length > max_duration: trimmed_audio = audio[:max_duration] else: trimmed_audio = audio # Export the trimmed audio to a new file trimmed_audio.export(output_path, format="wav") return output_path def add_silence_to_wav(wav_file_path, duration_s=1): # Load the WAV file audio = AudioSegment.from_wav(wav_file_path) # Create 1 second of silence silence = AudioSegment.silent(duration=duration_s * 1000) # duration is in milliseconds # Add silence to the end of the audio file audio_with_silence = audio + silence # Export the modified audio audio_with_silence.export(wav_file_path, format="wav") return wav_file_path def check_mp3(file_path): if is_mp3(file_path): unique_id = uuid.uuid4() wav_file_path = f"{os.path.splitext(file_path)[0]}-{unique_id}.wav" converted_audio = convert_mp3_to_wav(file_path, wav_file_path) print(f"File converted to {wav_file_path}") return converted_audio, gr.update(value=converted_audio, visible=True) else: print("The file is not an MP3 file.") return file_path, gr.update(value=file_path, visible=True) def check_and_convert_webp_to_png(input_path, output_path): try: # Open the image file with Image.open(input_path) as img: # Check if the image is in WebP format if img.format == 'WEBP': # Convert and save as PNG img.save(output_path, 'PNG') print(f"Converted {input_path} to {output_path}") return output_path else: print(f"The file {input_path} is not in WebP format.") return input_path except IOError: print(f"Cannot open {input_path}. The file might not exist or is not an image.") def convert_user_uploded_webp(input_path): # convert to png if necessary input_file = input_path unique_id = uuid.uuid4() output_file = f"converted_to_png_portrait-{unique_id}.png" ready_png = check_and_convert_webp_to_png(input_file, output_file) print(f"PORTRAIT PNG FILE: {ready_png}") return ready_png def clear_audio_elms(): return gr.update(value=None, visible=False) def change_video_codec(input_file, output_file, codec='libx264', audio_codec='aac'): try: ( ffmpeg .input(input_file) .output(output_file, vcodec=codec, acodec=audio_codec) .run(overwrite_output=True) ) print(f'Successfully changed codec of {input_file} and saved as {output_file}') except ffmpeg.Error as e: print(f'Error occurred: {e.stderr.decode()}') ####################################################### # Gradio APIs for optional image and voice generation # ####################################################### def generate_portrait(prompt_image): if prompt_image is None or prompt_image == "": raise gr.Error("Can't generate a portrait without a prompt !") try: client = Client("ByteDance/SDXL-Lightning") except: raise gr.Error(f"ByteDance/SDXL-Lightning space's api might not be ready, please wait, or upload an image instead.") result = client.predict( prompt = prompt_image, ckpt = "4-Step", api_name = "/generate_image" ) print(result) # convert to png if necessary input_file = result unique_id = uuid.uuid4() output_file = f"converted_to_png_portrait-{unique_id}.png" ready_png = check_and_convert_webp_to_png(input_file, output_file) print(f"PORTRAIT PNG FILE: {ready_png}") return ready_png def generate_voice_with_parler(prompt_audio, voice_description): if prompt_audio is None or prompt_audio == "" : raise gr.Error(f"Can't generate a voice without text to synthetize !") if voice_description is None or voice_description == "": gr.Info( "For better control, You may want to provide a voice character description next time.", duration = 10, visible = True ) try: client = Client("parler-tts/parler_tts_mini") except: raise gr.Error(f"parler-tts/parler_tts_mini space's api might not be ready, please wait, or upload an audio instead.") result = client.predict( text = prompt_audio, description = voice_description, api_name = "/gen_tts" ) print(result) return result, gr.update(value=result, visible=True) def get_whisperspeech(prompt_audio_whisperspeech, audio_to_clone): try: client = Client("collabora/WhisperSpeech") except: raise gr.Error(f"collabora/WhisperSpeech space's api might not be ready, please wait, or upload an audio instead.") result = client.predict( multilingual_text = prompt_audio_whisperspeech, speaker_audio = handle_file(audio_to_clone), speaker_url = "", cps = 14, api_name = "/whisper_speech_demo" ) print(result) return result, gr.update(value=result, visible=True) ######################## # TALKING PORTRAIT GEN # ######################## def run_hallo(source_image, driving_audio, progress=gr.Progress(track_tqdm=True)): unique_id = uuid.uuid4() args = argparse.Namespace( config = 'configs/inference/default.yaml', source_image = source_image, driving_audio = driving_audio, output = f'output-{unique_id}.mp4', pose_weight = 1.0, face_weight = 1.0, lip_weight = 1.0, face_expand_ratio = 1.2, checkpoint = None ) inference_process(args) return f'output-{unique_id}.mp4' def generate_talking_portrait(portrait, voice, progress=gr.Progress(track_tqdm=True)): if portrait is None: raise gr.Error("Please provide a portrait to animate.") if voice is None: raise gr.Error("Please provide audio (4 seconds max).") if is_shared_ui : # Trim audio to AUDIO_MAX_DURATION for better shared experience with community input_file = voice unique_id = uuid.uuid4() trimmed_output_file = f"-{unique_id}.wav" trimmed_output_file = trim_audio(input_file, trimmed_output_file, AUDIO_MAX_DURATION) voice = trimmed_output_file # Add 1 second of silence at the end to avoid last word being cut by hallo ready_audio = add_silence_to_wav(voice) print(f"1 second of silence added to {voice}") # Call hallo talking_portrait_vid = run_hallo(portrait, ready_audio) # Convert video to readable format final_output_file = f"converted_{talking_portrait_vid}" change_video_codec(talking_portrait_vid, final_output_file) return final_output_file css = ''' #col-container { margin: 0 auto; } #column-names { margin-top: 50px; } #main-group { background-color: none; } .tabs { background-color: unset; } #image-block { flex: 1; } #video-block { flex: 9; } #audio-block, #audio-clone-elm { flex: 1; } div#audio-clone-elm > .audio-container > button { height: 180px!important; } div#audio-clone-elm > .audio-container > button > .wrap { font-size: 0.9em; } #text-synth, #voice-desc{ height: 130px; } #text-synth-wsp { height: 120px; } #audio-column, #result-column { display: flex; } #gen-voice-btn { flex: 1; } #parler-tab, #whisperspeech-tab { padding: 0; } #main-submit{ flex: 1; } #pro-tips { margin-top: 50px; } div#warning-ready { background-color: #ecfdf5; padding: 0 16px 16px; margin: 20px 0; color: #030303!important; } div#warning-ready > .gr-prose > h2, div#warning-ready > .gr-prose > p { color: #057857!important; } div#warning-duplicate { background-color: #ebf5ff; padding: 0 16px 16px; margin: 20px 0; color: #030303!important; } div#warning-duplicate > .gr-prose > h2, div#warning-duplicate > .gr-prose > p { color: #0f4592!important; } div#warning-duplicate strong { color: #0f4592; } p.actions { display: flex; align-items: center; margin: 20px 0; } div#warning-duplicate .actions a { display: inline-block; margin-right: 10px; } .dark #warning-duplicate { background-color: #0c0c0c !important; border: 1px solid white !important; } ''' with gr.Blocks(css=css) as demo: with gr.Column(elem_id="col-container"): gr.Markdown(""" # TTS x Hallo Talking Portrait Generator This demo allows you to generate a talking portrait with the help of several open-source projects: SDXL Lightning | Parler TTS | WhisperSpeech | Hallo To let the community try and enjoy this demo, video length is limited to 4 seconds audio maximum. Duplicate this space to skip the queue and get unlimited video duration. 4-5 seconds of audio will take ~5 minutes per inference, please be patient. """) with gr.Row(elem_id="column-names"): gr.Markdown("## 1. Load Portrait") gr.Markdown("## 2. Load Voice") gr.Markdown("## 3. Result") with gr.Group(elem_id="main-group"): with gr.Row(): with gr.Column(): portrait = gr.Image( sources = ["upload"], type = "filepath", format = "png", elem_id = "image-block" ) prompt_image = gr.Textbox( label = "Generate image", lines = 2, max_lines = 2 ) gen_image_btn = gr.Button("Generate portrait (optional)") with gr.Column(elem_id="audio-column"): voice = gr.Audio( type = "filepath", elem_id = "audio-block" ) preprocess_audio_file = gr.File(visible=False) with gr.Tab("Parler TTS", elem_id="parler-tab"): prompt_audio = gr.Textbox( label = "Text to synthetize", lines = 3, max_lines = 3, elem_id = "text-synth" ) voice_description = gr.Textbox( label = "Voice description", lines = 3, max_lines = 3, elem_id = "voice-desc" ) gen_voice_btn = gr.Button("Generate voice (optional)") with gr.Tab("WhisperSpeech", elem_id="whisperspeech-tab"): prompt_audio_whisperspeech = gr.Textbox( label = "Text to synthetize", lines = 2, max_lines = 2, elem_id = "text-synth-wsp" ) audio_to_clone = gr.Audio( label = "Voice to clone", type = "filepath", elem_id = "audio-clone-elm" ) gen_wsp_voice_btn = gr.Button("Generate voice clone (optional)") with gr.Column(elem_id="result-column"): result = gr.Video( elem_id="video-block" ) submit_btn = gr.Button("Go talking Portrait !", elem_id="main-submit") with gr.Row(elem_id="pro-tips"): gr.Markdown(""" # Hallo Pro Tips: Hallo has a few simple requirements for input data: For the source image: 1. It should be cropped into squares. 2. The face should be the main focus, making up 50%-70% of the image. 3. The face should be facing forward, with a rotation angle of less than 30° (no side profiles). For the driving audio: 1. It must be in WAV format. 2. It must be in English since our training datasets are only in this language. 3. Ensure the vocals are clear; background music is acceptable. """) gr.Markdown(""" # TTS Pro Tips: For Parler TTS: - Include the term "very clear audio" to generate the highest quality audio, and "very noisy audio" for high levels of background noise - Punctuation can be used to control the prosody of the generations, e.g. use commas to add small breaks in speech - The remaining speech features (gender, speaking rate, pitch and reverberation) can be controlled directly through the prompt For WhisperSpeech: WhisperSpeech is able to quickly clone a voice from an audio sample. - Upload a voice sample in the WhisperSpeech tab - Add text to synthetize, hit Generate voice clone button """) portrait.upload( fn = convert_user_uploded_webp, inputs = [portrait], outputs = [portrait], queue = False, show_api = False ) voice.upload( fn = check_mp3, inputs = [voice], outputs = [voice, preprocess_audio_file], queue = False, show_api = False ) voice.clear( fn = clear_audio_elms, inputs = None, outputs = [preprocess_audio_file], queue = False, show_api = False ) gen_image_btn.click( fn = generate_portrait, inputs = [prompt_image], outputs = [portrait], queue = False, show_api = False ) gen_voice_btn.click( fn = generate_voice_with_parler, inputs = [prompt_audio, voice_description], outputs = [voice, preprocess_audio_file], queue = False, show_api = False ) gen_wsp_voice_btn.click( fn = get_whisperspeech, inputs = [prompt_audio_whisperspeech, audio_to_clone], outputs = [voice, preprocess_audio_file], queue = False, show_api = False ) submit_btn.click( fn = generate_talking_portrait, inputs = [portrait, voice], outputs = [result], show_api = False ) demo.queue(max_size=2).launch(show_error=True, show_api=False)