Spaces:
Runtime error
Runtime error
| import json | |
| import os | |
| from pathlib import Path | |
| from typing import List, Tuple | |
| import tempfile | |
| import soundfile as sf | |
| import gradio as gr | |
| import numpy as np | |
| import torch | |
| import torchaudio | |
| # from app.pipelines import Pipeline | |
| from fairseq import hub_utils | |
| from fairseq.checkpoint_utils import load_model_ensemble_and_task_from_hf_hub | |
| from fairseq.models.speech_to_speech.hub_interface import S2SHubInterface | |
| from fairseq.models.speech_to_text.hub_interface import S2THubInterface | |
| from fairseq.models.text_to_speech import CodeHiFiGANVocoder | |
| from fairseq.models.text_to_speech.hub_interface import ( | |
| TTSHubInterface, | |
| VocoderHubInterface, | |
| ) | |
| from huggingface_hub import snapshot_download | |
| ARG_OVERRIDES_MAP = { | |
| "facebook/xm_transformer_s2ut_800m-es-en-st-asr-bt_h1_2022": { | |
| "config_yaml": "config.yaml", | |
| "task": "speech_to_text", | |
| } | |
| } | |
| class SpeechToSpeechPipeline(): | |
| def __init__(self, model_id: str): | |
| arg_overrides = ARG_OVERRIDES_MAP.get( | |
| model_id, {} | |
| ) # Model specific override. TODO: Update on checkpoint side in the future | |
| arg_overrides["config_yaml"] = "config.yaml" # common override | |
| models, cfg, task = load_model_ensemble_and_task_from_hf_hub( | |
| model_id, | |
| arg_overrides=arg_overrides, | |
| cache_dir=os.getenv("HUGGINGFACE_HUB_CACHE"), | |
| ) | |
| self.cfg = cfg | |
| self.model = models[0].cpu() | |
| self.model.eval() | |
| self.task = task | |
| self.sampling_rate = getattr(self.task, "sr", None) or 16_000 | |
| tgt_lang = self.task.data_cfg.hub.get("tgt_lang", None) | |
| pfx = f"{tgt_lang}_" if self.task.data_cfg.prepend_tgt_lang_tag else "" | |
| generation_args = self.task.data_cfg.hub.get(f"{pfx}generation_args", None) | |
| if generation_args is not None: | |
| for key in generation_args: | |
| setattr(cfg.generation, key, generation_args[key]) | |
| self.generator = task.build_generator([self.model], cfg.generation) | |
| tts_model_id = self.task.data_cfg.hub.get(f"{pfx}tts_model_id", None) | |
| self.unit_vocoder = self.task.data_cfg.hub.get(f"{pfx}unit_vocoder", None) | |
| self.tts_model, self.tts_task, self.tts_generator = None, None, None | |
| if tts_model_id is not None: | |
| _id = tts_model_id.split(":")[-1] | |
| cache_dir = os.getenv("HUGGINGFACE_HUB_CACHE") | |
| if self.unit_vocoder is not None: | |
| library_name = "fairseq" | |
| cache_dir = ( | |
| cache_dir or (Path.home() / ".cache" / library_name).as_posix() | |
| ) | |
| cache_dir = snapshot_download( | |
| f"facebook/{_id}", cache_dir=cache_dir, library_name=library_name | |
| ) | |
| x = hub_utils.from_pretrained( | |
| cache_dir, | |
| "model.pt", | |
| ".", | |
| archive_map=CodeHiFiGANVocoder.hub_models(), | |
| config_yaml="config.json", | |
| fp16=False, | |
| is_vocoder=True, | |
| ) | |
| with open(f"{x['args']['data']}/config.json") as f: | |
| vocoder_cfg = json.load(f) | |
| assert ( | |
| len(x["args"]["model_path"]) == 1 | |
| ), "Too many vocoder models in the input" | |
| vocoder = CodeHiFiGANVocoder(x["args"]["model_path"][0], vocoder_cfg) | |
| self.tts_model = VocoderHubInterface(vocoder_cfg, vocoder) | |
| else: | |
| ( | |
| tts_models, | |
| tts_cfg, | |
| self.tts_task, | |
| ) = load_model_ensemble_and_task_from_hf_hub( | |
| f"facebook/{_id}", | |
| arg_overrides={"vocoder": "griffin_lim", "fp16": False}, | |
| cache_dir=cache_dir, | |
| ) | |
| self.tts_model = tts_models[0].cpu() | |
| self.tts_model.eval() | |
| tts_cfg["task"].cpu = True | |
| TTSHubInterface.update_cfg_with_data_cfg( | |
| tts_cfg, self.tts_task.data_cfg | |
| ) | |
| self.tts_generator = self.tts_task.build_generator( | |
| [self.tts_model], tts_cfg | |
| ) | |
| def __call__(self, inputs: str) -> Tuple[np.array, int, List[str]]: | |
| """ | |
| Args: | |
| inputs (:obj:`np.array`): | |
| The raw waveform of audio received. By default sampled at `self.sampling_rate`. | |
| The shape of this array is `T`, where `T` is the time axis | |
| Return: | |
| A :obj:`tuple` containing: | |
| - :obj:`np.array`: | |
| The return shape of the array must be `C'`x`T'` | |
| - a :obj:`int`: the sampling rate as an int in Hz. | |
| - a :obj:`List[str]`: the annotation for each out channel. | |
| This can be the name of the instruments for audio source separation | |
| or some annotation for speech enhancement. The length must be `C'`. | |
| """ | |
| # _inputs = torch.from_numpy(inputs).unsqueeze(0) | |
| # print(f"input: {inputs}") | |
| # _inputs = torchaudio.load(inputs) | |
| _inputs = inputs | |
| sample, text = None, None | |
| if self.cfg.task._name in ["speech_to_text", "speech_to_text_sharded"]: | |
| sample = S2THubInterface.get_model_input(self.task, _inputs) | |
| text = S2THubInterface.get_prediction( | |
| self.task, self.model, self.generator, sample | |
| ) | |
| elif self.cfg.task._name in ["speech_to_speech"]: | |
| s2shubinerface = S2SHubInterface(self.cfg, self.task, self.model) | |
| sample = s2shubinerface.get_model_input(self.task, _inputs) | |
| text = S2SHubInterface.get_prediction( | |
| self.task, self.model, self.generator, sample | |
| ) | |
| wav, sr = np.zeros((0,)), self.sampling_rate | |
| if self.unit_vocoder is not None: | |
| tts_sample = self.tts_model.get_model_input(text) | |
| wav, sr = self.tts_model.get_prediction(tts_sample) | |
| text = "" | |
| else: | |
| tts_sample = TTSHubInterface.get_model_input(self.tts_task, text) | |
| wav, sr = TTSHubInterface.get_prediction( | |
| self.tts_task, self.tts_model, self.tts_generator, tts_sample | |
| ) | |
| temp_file = "" | |
| with tempfile.NamedTemporaryFile(suffix=".wav") as tmp_output_file: | |
| sf.write(tmp_output_file, wav.detach().cpu().numpy(), sr) | |
| tmp_output_file.seek(0) | |
| temp_file = gr.Audio(tmp_output_file.name) | |
| # return wav, sr, [text] | |
| return temp_file |