Spaces:
Runtime error
Runtime error
| from __future__ import annotations | |
| import os | |
| import pathlib | |
| import gradio as gr | |
| import numpy as np | |
| import torch | |
| import torchaudio | |
| from fairseq2.assets import InProcAssetMetadataProvider, asset_store | |
| from huggingface_hub import snapshot_download | |
| from seamless_communication.inference import Translator | |
| from lang_list import ( | |
| ASR_TARGET_LANGUAGE_NAMES, | |
| LANGUAGE_NAME_TO_CODE, | |
| S2ST_TARGET_LANGUAGE_NAMES, | |
| S2TT_TARGET_LANGUAGE_NAMES, | |
| T2ST_TARGET_LANGUAGE_NAMES, | |
| T2TT_TARGET_LANGUAGE_NAMES, | |
| TEXT_SOURCE_LANGUAGE_NAMES, | |
| ) | |
| CHECKPOINTS_PATH = pathlib.Path(os.getenv("CHECKPOINTS_PATH", "/home/user/app/models")) | |
| if not CHECKPOINTS_PATH.exists(): | |
| snapshot_download(repo_id="facebook/seamless-m4t-v2-large", repo_type="model", local_dir=CHECKPOINTS_PATH) | |
| asset_store.env_resolvers.clear() | |
| asset_store.env_resolvers.append(lambda: "demo") | |
| demo_metadata = [ | |
| { | |
| "name": "seamlessM4T_v2_large@demo", | |
| "checkpoint": f"file://{CHECKPOINTS_PATH}/seamlessM4T_v2_large.pt", | |
| "char_tokenizer": f"file://{CHECKPOINTS_PATH}/spm_char_lang38_tc.model", | |
| }, | |
| { | |
| "name": "vocoder_v2@demo", | |
| "checkpoint": f"file://{CHECKPOINTS_PATH}/vocoder_v2.pt", | |
| }, | |
| ] | |
| asset_store.metadata_providers.append(InProcAssetMetadataProvider(demo_metadata)) | |
| DESCRIPTION = """\ | |
| # SeamlessM4T | |
| [SeamlessM4T](https://github.com/facebookresearch/seamless_communication) is designed to provide high-quality | |
| translation, allowing people from different linguistic communities to communicate effortlessly through speech and text. | |
| This unified model enables multiple tasks like Speech-to-Speech (S2ST), Speech-to-Text (S2TT), Text-to-Speech (T2ST) | |
| translation and more, without relying on multiple separate models. The model is also in use on the | |
| [SeamlessM4T demo website](https://seamless.metademolab.com/m4t?utm_source=huggingface&utm_medium=web&utm_campaign=seamless&utm_content=m4tspace). | |
| """ | |
| CACHE_EXAMPLES = os.getenv("CACHE_EXAMPLES") == "1" and torch.cuda.is_available() | |
| AUDIO_SAMPLE_RATE = 16000.0 | |
| MAX_INPUT_AUDIO_LENGTH = 60 # in seconds | |
| DEFAULT_TARGET_LANGUAGE = "French" | |
| if torch.cuda.is_available(): | |
| device = torch.device("cuda:0") | |
| dtype = torch.float16 | |
| else: | |
| device = torch.device("cpu") | |
| dtype = torch.float32 | |
| translator = Translator( | |
| model_name_or_card="seamlessM4T_v2_large", | |
| vocoder_name_or_card="vocoder_v2", | |
| device=device, | |
| dtype=dtype, | |
| apply_mintox=True, | |
| ) | |
| def preprocess_audio(input_audio: str) -> None: | |
| arr, org_sr = torchaudio.load(input_audio) | |
| new_arr = torchaudio.functional.resample(arr, orig_freq=org_sr, new_freq=AUDIO_SAMPLE_RATE) | |
| max_length = int(MAX_INPUT_AUDIO_LENGTH * AUDIO_SAMPLE_RATE) | |
| if new_arr.shape[1] > max_length: | |
| new_arr = new_arr[:, :max_length] | |
| gr.Warning(f"Input audio is too long. Only the first {MAX_INPUT_AUDIO_LENGTH} seconds is used.") | |
| torchaudio.save(input_audio, new_arr, sample_rate=int(AUDIO_SAMPLE_RATE)) | |
| def run_s2st( | |
| input_audio: str, source_language: str, target_language: str | |
| ) -> tuple[tuple[int, np.ndarray] | None, str]: | |
| preprocess_audio(input_audio) | |
| source_language_code = LANGUAGE_NAME_TO_CODE[source_language] | |
| target_language_code = LANGUAGE_NAME_TO_CODE[target_language] | |
| out_texts, out_audios = translator.predict( | |
| input=input_audio, | |
| task_str="S2ST", | |
| src_lang=source_language_code, | |
| tgt_lang=target_language_code, | |
| ) | |
| out_text = str(out_texts[0]) | |
| out_wav = out_audios.audio_wavs[0].cpu().detach().numpy() | |
| return (int(AUDIO_SAMPLE_RATE), out_wav), out_text | |
| def run_s2tt(input_audio: str, source_language: str, target_language: str) -> str: | |
| preprocess_audio(input_audio) | |
| source_language_code = LANGUAGE_NAME_TO_CODE[source_language] | |
| target_language_code = LANGUAGE_NAME_TO_CODE[target_language] | |
| out_texts, _ = translator.predict( | |
| input=input_audio, | |
| task_str="S2TT", | |
| src_lang=source_language_code, | |
| tgt_lang=target_language_code, | |
| ) | |
| return str(out_texts[0]) | |
| def run_t2st(input_text: str, source_language: str, target_language: str) -> tuple[tuple[int, np.ndarray] | None, str]: | |
| source_language_code = LANGUAGE_NAME_TO_CODE[source_language] | |
| target_language_code = LANGUAGE_NAME_TO_CODE[target_language] | |
| out_texts, out_audios = translator.predict( | |
| input=input_text, | |
| task_str="T2ST", | |
| src_lang=source_language_code, | |
| tgt_lang=target_language_code, | |
| ) | |
| out_text = str(out_texts[0]) | |
| out_wav = out_audios.audio_wavs[0].cpu().detach().numpy() | |
| return (int(AUDIO_SAMPLE_RATE), out_wav), out_text | |
| def run_t2tt(input_text: str, source_language: str, target_language: str) -> str: | |
| source_language_code = LANGUAGE_NAME_TO_CODE[source_language] | |
| target_language_code = LANGUAGE_NAME_TO_CODE[target_language] | |
| out_texts, _ = translator.predict( | |
| input=input_text, | |
| task_str="T2TT", | |
| src_lang=source_language_code, | |
| tgt_lang=target_language_code, | |
| ) | |
| return str(out_texts[0]) | |
| def run_asr(input_audio: str, target_language: str) -> str: | |
| preprocess_audio(input_audio) | |
| target_language_code = LANGUAGE_NAME_TO_CODE[target_language] | |
| out_texts, _ = translator.predict( | |
| input=input_audio, | |
| task_str="ASR", | |
| src_lang=target_language_code, | |
| tgt_lang=target_language_code, | |
| ) | |
| return str(out_texts[0]) | |
| with gr.Blocks() as demo_s2st: | |
| with gr.Row(): | |
| with gr.Column(): | |
| with gr.Group(): | |
| input_audio = gr.Audio(label="Input speech", type="filepath") | |
| source_language = gr.Dropdown( | |
| label="Source language", | |
| choices=ASR_TARGET_LANGUAGE_NAMES, | |
| value="English", | |
| ) | |
| target_language = gr.Dropdown( | |
| label="Target language", | |
| choices=S2ST_TARGET_LANGUAGE_NAMES, | |
| value=DEFAULT_TARGET_LANGUAGE, | |
| ) | |
| btn = gr.Button("Translate") | |
| with gr.Column(): | |
| with gr.Group(): | |
| output_audio = gr.Audio( | |
| label="Translated speech", | |
| autoplay=False, | |
| streaming=False, | |
| type="numpy", | |
| ) | |
| output_text = gr.Textbox(label="Translated text") | |
| gr.Examples( | |
| examples=[ | |
| ["assets/sample_input.mp3", "English", "French"], | |
| ["assets/sample_input.mp3", "English", "Mandarin Chinese"], | |
| ["assets/sample_input_2.mp3", "English", "Hindi"], | |
| ["assets/sample_input_2.mp3", "English", "Spanish"], | |
| ], | |
| inputs=[input_audio, source_language, target_language], | |
| outputs=[output_audio, output_text], | |
| fn=run_s2st, | |
| cache_examples=CACHE_EXAMPLES, | |
| api_name=False, | |
| ) | |
| btn.click( | |
| fn=run_s2st, | |
| inputs=[input_audio, source_language, target_language], | |
| outputs=[output_audio, output_text], | |
| api_name="s2st", | |
| ) | |
| with gr.Blocks() as demo_s2tt: | |
| with gr.Row(): | |
| with gr.Column(): | |
| with gr.Group(): | |
| input_audio = gr.Audio(label="Input speech", type="filepath") | |
| source_language = gr.Dropdown( | |
| label="Source language", | |
| choices=ASR_TARGET_LANGUAGE_NAMES, | |
| value="English", | |
| ) | |
| target_language = gr.Dropdown( | |
| label="Target language", | |
| choices=S2TT_TARGET_LANGUAGE_NAMES, | |
| value=DEFAULT_TARGET_LANGUAGE, | |
| ) | |
| btn = gr.Button("Translate") | |
| with gr.Column(): | |
| output_text = gr.Textbox(label="Translated text") | |
| gr.Examples( | |
| examples=[ | |
| ["assets/sample_input.mp3", "English", "French"], | |
| ["assets/sample_input.mp3", "English", "Mandarin Chinese"], | |
| ["assets/sample_input_2.mp3", "English", "Hindi"], | |
| ["assets/sample_input_2.mp3", "English", "Spanish"], | |
| ], | |
| inputs=[input_audio, source_language, target_language], | |
| outputs=output_text, | |
| fn=run_s2tt, | |
| cache_examples=CACHE_EXAMPLES, | |
| api_name=False, | |
| ) | |
| btn.click( | |
| fn=run_s2tt, | |
| inputs=[input_audio, source_language, target_language], | |
| outputs=output_text, | |
| api_name="s2tt", | |
| ) | |
| with gr.Blocks() as demo_t2st: | |
| with gr.Row(): | |
| with gr.Column(): | |
| with gr.Group(): | |
| input_text = gr.Textbox(label="Input text") | |
| with gr.Row(): | |
| source_language = gr.Dropdown( | |
| label="Source language", | |
| choices=TEXT_SOURCE_LANGUAGE_NAMES, | |
| value="English", | |
| ) | |
| target_language = gr.Dropdown( | |
| label="Target language", | |
| choices=T2ST_TARGET_LANGUAGE_NAMES, | |
| value=DEFAULT_TARGET_LANGUAGE, | |
| ) | |
| btn = gr.Button("Translate") | |
| with gr.Column(): | |
| with gr.Group(): | |
| output_audio = gr.Audio( | |
| label="Translated speech", | |
| autoplay=False, | |
| streaming=False, | |
| type="numpy", | |
| ) | |
| output_text = gr.Textbox(label="Translated text") | |
| gr.Examples( | |
| examples=[ | |
| [ | |
| "My favorite animal is the elephant.", | |
| "English", | |
| "French", | |
| ], | |
| [ | |
| "My favorite animal is the elephant.", | |
| "English", | |
| "Mandarin Chinese", | |
| ], | |
| [ | |
| "Meta AI's Seamless M4T model is democratising spoken communication across language barriers", | |
| "English", | |
| "Hindi", | |
| ], | |
| [ | |
| "Meta AI's Seamless M4T model is democratising spoken communication across language barriers", | |
| "English", | |
| "Spanish", | |
| ], | |
| ], | |
| inputs=[input_text, source_language, target_language], | |
| outputs=[output_audio, output_text], | |
| fn=run_t2st, | |
| cache_examples=CACHE_EXAMPLES, | |
| api_name=False, | |
| ) | |
| gr.on( | |
| triggers=[input_text.submit, btn.click], | |
| fn=run_t2st, | |
| inputs=[input_text, source_language, target_language], | |
| outputs=[output_audio, output_text], | |
| api_name="t2st", | |
| ) | |
| with gr.Blocks() as demo_t2tt: | |
| with gr.Row(): | |
| with gr.Column(): | |
| with gr.Group(): | |
| input_text = gr.Textbox(label="Input text") | |
| with gr.Row(): | |
| source_language = gr.Dropdown( | |
| label="Source language", | |
| choices=TEXT_SOURCE_LANGUAGE_NAMES, | |
| value="English", | |
| ) | |
| target_language = gr.Dropdown( | |
| label="Target language", | |
| choices=T2TT_TARGET_LANGUAGE_NAMES, | |
| value=DEFAULT_TARGET_LANGUAGE, | |
| ) | |
| btn = gr.Button("Translate") | |
| with gr.Column(): | |
| output_text = gr.Textbox(label="Translated text") | |
| gr.Examples( | |
| examples=[ | |
| [ | |
| "My favorite animal is the elephant.", | |
| "English", | |
| "French", | |
| ], | |
| [ | |
| "My favorite animal is the elephant.", | |
| "English", | |
| "Mandarin Chinese", | |
| ], | |
| [ | |
| "Meta AI's Seamless M4T model is democratising spoken communication across language barriers", | |
| "English", | |
| "Hindi", | |
| ], | |
| [ | |
| "Meta AI's Seamless M4T model is democratising spoken communication across language barriers", | |
| "English", | |
| "Spanish", | |
| ], | |
| ], | |
| inputs=[input_text, source_language, target_language], | |
| outputs=output_text, | |
| fn=run_t2tt, | |
| cache_examples=CACHE_EXAMPLES, | |
| api_name=False, | |
| ) | |
| gr.on( | |
| triggers=[input_text.submit, btn.click], | |
| fn=run_t2tt, | |
| inputs=[input_text, source_language, target_language], | |
| outputs=output_text, | |
| api_name="t2tt", | |
| ) | |
| with gr.Blocks() as demo_asr: | |
| with gr.Row(): | |
| with gr.Column(): | |
| with gr.Group(): | |
| input_audio = gr.Audio(label="Input speech", type="filepath") | |
| target_language = gr.Dropdown( | |
| label="Target language", | |
| choices=ASR_TARGET_LANGUAGE_NAMES, | |
| value=DEFAULT_TARGET_LANGUAGE, | |
| ) | |
| btn = gr.Button("Translate") | |
| with gr.Column(): | |
| output_text = gr.Textbox(label="Translated text") | |
| gr.Examples( | |
| examples=[ | |
| ["assets/sample_input.mp3", "English"], | |
| ["assets/sample_input_2.mp3", "English"], | |
| ], | |
| inputs=[input_audio, target_language], | |
| outputs=output_text, | |
| fn=run_asr, | |
| cache_examples=CACHE_EXAMPLES, | |
| api_name=False, | |
| ) | |
| btn.click( | |
| fn=run_asr, | |
| inputs=[input_audio, target_language], | |
| outputs=output_text, | |
| api_name="asr", | |
| ) | |
| with gr.Blocks(css="style.css") as demo: | |
| gr.Markdown(DESCRIPTION) | |
| gr.DuplicateButton( | |
| value="Duplicate Space for private use", | |
| elem_id="duplicate-button", | |
| visible=os.getenv("SHOW_DUPLICATE_BUTTON") == "1", | |
| ) | |
| with gr.Tabs(): | |
| with gr.Tab(label="S2ST"): | |
| demo_s2st.render() | |
| with gr.Tab(label="S2TT"): | |
| demo_s2tt.render() | |
| with gr.Tab(label="T2ST"): | |
| demo_t2st.render() | |
| with gr.Tab(label="T2TT"): | |
| demo_t2tt.render() | |
| with gr.Tab(label="ASR"): | |
| demo_asr.render() | |
| if __name__ == "__main__": | |
| demo.queue(max_size=50).launch() | |