import concurrent import os import tempfile from typing import Optional, Tuple import numpy as np import spaces from transformers import pipeline import gradio as gr import torch import torchaudio from resemble_enhance.enhancer.inference import denoise, enhance from flore200_codes import flores_codes from tts import BambaraTTS # Check if CUDA is available device = "cuda" if torch.cuda.is_available() else "cpu" # Translation pipeline translation_model = "oza75/nllb-600M-mt-french-bambara" translator = pipeline("translation", model=translation_model, max_length=512) # Text-to-Speech pipeline tts_model = "oza75/bambara-tts" tts = BambaraTTS(tts_model) # Function to translate text to Bambara def translate_to_bambara(text, src_lang): translation = translator(text, src_lang=src_lang, tgt_lang="bam_Latn") return str(translation[0]['translation_text']) # Function to convert text to speech def text_to_speech(bambara_text, reference_speaker: str, reference_audio: Optional[Tuple] = None): if reference_audio is not None: ref_sr, ref_audio = reference_audio ref_audio = torch.from_numpy(ref_audio) # Add a channel dimension if the audio is 1D if ref_audio.ndim == 1: ref_audio = ref_audio.unsqueeze(0) # Save the reference audio to a temporary file if it's not None with tempfile.NamedTemporaryFile(delete=False, suffix='.wav') as tmp: torchaudio.save(tmp.name, ref_audio, ref_sr) tmp_path = tmp.name # Use the temporary file as the speaker reference sr, audio = tts.text_to_speech(bambara_text, speaker_reference_wav_path=tmp_path) # Clean up the temporary file os.unlink(tmp_path) else: # If no reference audio provided, proceed with the reference_speaker sr, audio = tts.text_to_speech(bambara_text, speaker_reference_wav_path=reference_speaker) audio = audio.mean(dim=0) return audio, sr # Function to enhance speech def enhance_speech(audio_array, sampling_rate, solver, nfe, tau, denoise_before_enhancement): solver = solver.lower() nfe = int(nfe) lambd = 0.9 if denoise_before_enhancement else 0.1 def denoise_audio(): try: return denoise(audio_array, sampling_rate, device) except Exception as e: print("> Error while denoising : ", str(e)) return audio_array, sampling_rate def enhance_audio(): try: return enhance(audio_array, sampling_rate, device, nfe=nfe, solver=solver, lambd=lambd, tau=tau) except Exception as e: print("> Error while enhancement : ", str(e)) return audio_array, sampling_rate with concurrent.futures.ThreadPoolExecutor() as executor: future_denoise = executor.submit(denoise_audio) future_enhance = executor.submit(enhance_audio) denoised_audio, new_sr1 = future_denoise.result() enhanced_audio, new_sr2 = future_enhance.result() # Convert to numpy and return return (new_sr1, denoised_audio.cpu().numpy()), (new_sr2, enhanced_audio.cpu().numpy()) # Define the Gradio interface @spaces.GPU def _fn( src_lang, text, reference_speaker, reference_audio=None, solver="Midpoint", nfe=64, prior_temp=0.5, denoise_before_enhancement=False ): source_lang = flores_codes[src_lang] reference_speaker = os.path.join("./audios", reference_speaker) # Step 1: Translate the text to Bambara bambara_text = translate_to_bambara(text, source_lang) yield bambara_text, None, None, None # Step 2: Convert the translated text to speech with reference audio if reference_audio is not None: audio_array, sampling_rate = text_to_speech(bambara_text, reference_audio) else: audio_array, sampling_rate = text_to_speech(bambara_text, reference_speaker=reference_speaker) yield bambara_text, (sampling_rate, audio_array.numpy()), None, None # Step 3: Enhance the audio denoised_audio, enhanced_audio = enhance_speech( audio_array, sampling_rate, solver, nfe, prior_temp, denoise_before_enhancement ) yield bambara_text, (sampling_rate, audio_array.numpy()), denoised_audio, enhanced_audio def main(): lang_codes = list(flores_codes.keys()) # List all files in the ./audios directory for the dropdown audio_files = [f for f in os.listdir('./audios') if os.path.isfile(os.path.join('./audios', f))] # Build Gradio app app = gr.Interface( fn=_fn, inputs=[ gr.Dropdown(label="Source Language", choices=lang_codes, value='French'), gr.Textbox(label="Text to Translate", lines=3, value="Vous pouvez entrer un texte ici et il sera traduite en Bambara."), gr.Dropdown(label="Voice", choices=audio_files, value="male_2.wav"), gr.Audio(label="Clone your voice (optional)", type="numpy", format="wav", elem_id="clone_voice_input"), # gr.Dropdown( # choices=["Midpoint", "RK4", "Euler"], value="Midpoint", # label="ODE Solver (Midpoint is recommended)" # ), # gr.Slider(minimum=1, maximum=128, value=64, step=1, label="Number of Function Evaluations"), # gr.Slider(minimum=0.1, maximum=1, value=0.5, step=0.01, label="Prior Temperature"), # gr.Checkbox(value=False, label="Denoise Before Enhancement") ], outputs=[ gr.Textbox(label="Translated Text"), gr.Audio(label="Original TTS Audio", format='wav'), gr.Audio(label="Denoised Audio", format='wav'), gr.Audio(label="Enhanced Audio", format='wav') ], examples=[ ['English', "Thomas Isidore Noël Sankara is the son of a Peul father — originally from the village of Sitoèga in the department of Bokin in the province of Passoré — and a Mossi mother, and grew up between military values and religiosity.", "male_2.wav", None], ['French', "Vous ne pouviez subvenir à ses besoins. Il lui fallait une garde.", "male_3.wav", None], ['French', "Vous pouvez entrer un texte ici et il sera traduite en Bambara.", "male_2.wav", None], ], css="#clone_voice_input .audio-container button.boundedheight { height: 147px !important; }", title="Bambara Translation and Text to Speech with Audio Enhancement", description="Translate text to Bambara and convert it to speech with options to enhance audio quality." ) app.launch(share=False) if __name__ == "__main__": main()