lpw commited on
Commit
665ffa8
1 Parent(s): 73f764f

Create README.md

Browse files
Files changed (1) hide show
  1. README.md +107 -0
README.md ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: cc-by-nc-4.0
3
+ library_name: fairseq
4
+ task: audio-to-audio
5
+ tags:
6
+ - fairseq
7
+ - audio
8
+ - audio-to-audio
9
+ - speech-to-speech-translation
10
+
11
+ datasets:
12
+ - mtedx
13
+ - covost2
14
+ - europarl_st
15
+ - voxpopuli
16
+ widget:
17
+ - example_title: Common Voice sample 1
18
+ src: https://huggingface.co/facebook/xm_transformer_600m-es_en-multi_domain/resolve/main/common_voice_es_19966634.flac
19
+ ---
20
+ ## xm_transformer_s2ut_800m-es-en-st-asr-bt_h1_2022
21
+
22
+ Speech-to-speech translation model from fairseq S2UT ([paper](https://arxiv.org/abs/2204.02967)/[code](https://github.com/facebookresearch/fairseq/blob/main/examples/speech_to_speech/docs/enhanced_direct_s2st_discrete_units.md)):
23
+ - Spanish-English
24
+ - Trained on mTEDx, CoVoST 2, Europarl-ST and VoxPopuli
25
+ - Speech synthesis with [facebook/unit_hifigan_mhubert_vp_en_es_fr_it3_400k_layer11_km1000_lj_dur](https://huggingface.co/facebook/unit_hifigan_mhubert_vp_en_es_fr_it3_400k_layer11_km1000_lj_dur)
26
+
27
+ ## Usage
28
+ ```python
29
+ import json
30
+ import os
31
+ from pathlib import Path
32
+
33
+ import IPython.display as ipd
34
+ from fairseq import hub_utils
35
+ from fairseq.checkpoint_utils import load_model_ensemble_and_task_from_hf_hub
36
+ from fairseq.models.speech_to_text.hub_interface import S2THubInterface
37
+ from fairseq.models.text_to_speech import CodeHiFiGANVocoder
38
+ from fairseq.models.text_to_speech.hub_interface import VocoderHubInterface
39
+
40
+ from huggingface_hub import snapshot_download
41
+ import torchaudio
42
+
43
+ cache_dir = os.getenv("HUGGINGFACE_HUB_CACHE")
44
+
45
+ models, cfg, task = load_model_ensemble_and_task_from_hf_hub(
46
+ "facebook/xm_transformer_s2ut_800m-es-en-st-asr-bt_h1_2022",
47
+ arg_overrides={"config_yaml": "config.yaml", "task": "speech_to_text"},
48
+ cache_dir=cache_dir,
49
+ )
50
+ #model = models[0].cpu()
51
+ #cfg["task"].cpu = True
52
+ generator = task.build_generator([model], cfg)
53
+
54
+
55
+ # requires 16000Hz mono channel audio
56
+ audio, _ = torchaudio.load("/path/to/an/audio/file")
57
+
58
+ sample = S2THubInterface.get_model_input(task, audio)
59
+ unit = S2THubInterface.get_prediction(task, model, generator, sample)
60
+
61
+ # speech synthesis
62
+ library_name = "fairseq"
63
+ cache_dir = (
64
+ cache_dir or (Path.home() / ".cache" / library_name).as_posix()
65
+ )
66
+ cache_dir = snapshot_download(
67
+ f"facebook/unit_hifigan_mhubert_vp_en_es_fr_it3_400k_layer11_km1000_lj_dur", cache_dir=cache_dir, library_name=library_name
68
+ )
69
+
70
+ x = hub_utils.from_pretrained(
71
+ cache_dir,
72
+ "model.pt",
73
+ ".",
74
+ archive_map=CodeHiFiGANVocoder.hub_models(),
75
+ config_yaml="config.json",
76
+ fp16=False,
77
+ is_vocoder=True,
78
+ )
79
+
80
+ with open(f"{x['args']['data']}/config.json") as f:
81
+ vocoder_cfg = json.load(f)
82
+ assert (
83
+ len(x["args"]["model_path"]) == 1
84
+ ), "Too many vocoder models in the input"
85
+
86
+ vocoder = CodeHiFiGANVocoder(x["args"]["model_path"][0], vocoder_cfg)
87
+ tts_model = VocoderHubInterface(vocoder_cfg, vocoder)
88
+
89
+ tts_sample = tts_model.get_model_input(unit)
90
+ wav, sr = tts_model.get_prediction(tts_sample)
91
+
92
+ ipd.Audio(wav, rate=sr)
93
+ ```
94
+
95
+ ## Citation
96
+ ```bibtex
97
+ @misc{https://doi.org/10.48550/arxiv.2204.02967,
98
+ doi = {10.48550/ARXIV.2204.02967},
99
+ url = {https://arxiv.org/abs/2204.02967},
100
+ author = {Popuri, Sravya and Chen, Peng-Jen and Wang, Changhan and Pino, Juan and Adi, Yossi and Gu, Jiatao and Hsu, Wei-Ning and Lee, Ann},
101
+ keywords = {Computation and Language (cs.CL), Sound (cs.SD), Audio and Speech Processing (eess.AS), FOS: Computer and information sciences, FOS: Computer and information sciences, FOS: Electrical engineering, electronic engineering, information engineering, FOS: Electrical engineering, electronic engineering, information engineering},
102
+ title = {Enhanced Direct Speech-to-Speech Translation Using Self-supervised Pre-training and Data Augmentation},
103
+ publisher = {arXiv},
104
+ year = {2022},
105
+ copyright = {arXiv.org perpetual, non-exclusive license}
106
+ }
107
+ ```