Spaces:
Runtime error
Runtime error
| import re | |
| import sys | |
| import panphon | |
| import phonemizer | |
| import torch | |
| from Preprocessing.papercup_features import generate_feature_table | |
| class ArticulatoryCombinedTextFrontend: | |
| def __init__(self, | |
| language, | |
| use_word_boundaries=False, # goes together well with | |
| # parallel models and an aligner. Doesn't go together | |
| # well with autoregressive models. | |
| use_explicit_eos=True, | |
| use_prosody=False, # unfortunately the non-segmental | |
| # nature of prosodic markers mixed with the sequential | |
| # phonemes hurts the performance of end-to-end models a | |
| # lot, even though one might think enriching the input | |
| # with such information would help. | |
| use_lexical_stress=False, | |
| silent=True, | |
| allow_unknown=False, | |
| add_silence_to_end=True, | |
| strip_silence=True): | |
| """ | |
| Mostly preparing ID lookups | |
| """ | |
| self.strip_silence = strip_silence | |
| self.use_word_boundaries = use_word_boundaries | |
| self.allow_unknown = allow_unknown | |
| self.use_explicit_eos = use_explicit_eos | |
| self.use_prosody = use_prosody | |
| self.use_stress = use_lexical_stress | |
| self.add_silence_to_end = add_silence_to_end | |
| self.feature_table = panphon.FeatureTable() | |
| if language == "en": | |
| self.g2p_lang = "en-us" | |
| self.expand_abbreviations = english_text_expansion | |
| if not silent: | |
| print("Created an English Text-Frontend") | |
| elif language == "de": | |
| self.g2p_lang = "de" | |
| self.expand_abbreviations = lambda x: x | |
| if not silent: | |
| print("Created a German Text-Frontend") | |
| elif language == "el": | |
| self.g2p_lang = "el" | |
| self.expand_abbreviations = lambda x: x | |
| if not silent: | |
| print("Created a Greek Text-Frontend") | |
| elif language == "es": | |
| self.g2p_lang = "es" | |
| self.expand_abbreviations = lambda x: x | |
| if not silent: | |
| print("Created a Spanish Text-Frontend") | |
| elif language == "fi": | |
| self.g2p_lang = "fi" | |
| self.expand_abbreviations = lambda x: x | |
| if not silent: | |
| print("Created a Finnish Text-Frontend") | |
| elif language == "ru": | |
| self.g2p_lang = "ru" | |
| self.expand_abbreviations = lambda x: x | |
| if not silent: | |
| print("Created a Russian Text-Frontend") | |
| elif language == "hu": | |
| self.g2p_lang = "hu" | |
| self.expand_abbreviations = lambda x: x | |
| if not silent: | |
| print("Created a Hungarian Text-Frontend") | |
| elif language == "nl": | |
| self.g2p_lang = "nl" | |
| self.expand_abbreviations = lambda x: x | |
| if not silent: | |
| print("Created a Dutch Text-Frontend") | |
| elif language == "fr": | |
| self.g2p_lang = "fr-fr" | |
| self.expand_abbreviations = lambda x: x | |
| if not silent: | |
| print("Created a French Text-Frontend") | |
| elif language == "it": | |
| self.g2p_lang = "it" | |
| self.expand_abbreviations = lambda x: x | |
| if not silent: | |
| print("Created a Italian Text-Frontend") | |
| elif language == "pt": | |
| self.g2p_lang = "pt" | |
| self.expand_abbreviations = lambda x: x | |
| if not silent: | |
| print("Created a Portuguese Text-Frontend") | |
| elif language == "pl": | |
| self.g2p_lang = "pl" | |
| self.expand_abbreviations = lambda x: x | |
| if not silent: | |
| print("Created a Polish Text-Frontend") | |
| # remember to also update get_language_id() when adding something here | |
| else: | |
| print("Language not supported yet") | |
| sys.exit() | |
| self.phone_to_vector_papercup = generate_feature_table() | |
| self.phone_to_vector = dict() | |
| for phone in self.phone_to_vector_papercup: | |
| panphon_features = self.feature_table.word_to_vector_list(phone, numeric=True) | |
| if panphon_features == []: | |
| panphon_features = [[0] * 24] | |
| papercup_features = self.phone_to_vector_papercup[phone] | |
| self.phone_to_vector[phone] = papercup_features + panphon_features[0] | |
| self.phone_to_id = { # this lookup must be updated manually, because the only | |
| # other way would be extracting them from a set, which can be non-deterministic | |
| '~': 0, | |
| '#': 1, | |
| '?': 2, | |
| '!': 3, | |
| '.': 4, | |
| 'ɜ': 5, | |
| 'ɫ': 6, | |
| 'ə': 7, | |
| 'ɚ': 8, | |
| 'a': 9, | |
| 'ð': 10, | |
| 'ɛ': 11, | |
| 'ɪ': 12, | |
| 'ᵻ': 13, | |
| 'ŋ': 14, | |
| 'ɔ': 15, | |
| 'ɒ': 16, | |
| 'ɾ': 17, | |
| 'ʃ': 18, | |
| 'θ': 19, | |
| 'ʊ': 20, | |
| 'ʌ': 21, | |
| 'ʒ': 22, | |
| 'æ': 23, | |
| 'b': 24, | |
| 'ʔ': 25, | |
| 'd': 26, | |
| 'e': 27, | |
| 'f': 28, | |
| 'g': 29, | |
| 'h': 30, | |
| 'i': 31, | |
| 'j': 32, | |
| 'k': 33, | |
| 'l': 34, | |
| 'm': 35, | |
| 'n': 36, | |
| 'ɳ': 37, | |
| 'o': 38, | |
| 'p': 39, | |
| 'ɡ': 40, | |
| 'ɹ': 41, | |
| 'r': 42, | |
| 's': 43, | |
| 't': 44, | |
| 'u': 45, | |
| 'v': 46, | |
| 'w': 47, | |
| 'x': 48, | |
| 'z': 49, | |
| 'ʀ': 50, | |
| 'ø': 51, | |
| 'ç': 52, | |
| 'ɐ': 53, | |
| 'œ': 54, | |
| 'y': 55, | |
| 'ʏ': 56, | |
| 'ɑ': 57, | |
| 'c': 58, | |
| 'ɲ': 59, | |
| 'ɣ': 60, | |
| 'ʎ': 61, | |
| 'β': 62, | |
| 'ʝ': 63, | |
| 'ɟ': 64, | |
| 'q': 65, | |
| 'ɕ': 66, | |
| 'ʲ': 67, | |
| 'ɭ': 68, | |
| 'ɵ': 69, | |
| 'ʑ': 70, | |
| 'ʋ': 71, | |
| 'ʁ': 72, | |
| 'ɨ': 73, | |
| 'ʂ': 74, | |
| 'ɬ': 75, | |
| } # for the states of the ctc loss and dijkstra/mas in the aligner | |
| self.id_to_phone = {v: k for k, v in self.phone_to_id.items()} | |
| def string_to_tensor(self, text, view=False, device="cpu", handle_missing=True, input_phonemes=False): | |
| """ | |
| Fixes unicode errors, expands some abbreviations, | |
| turns graphemes into phonemes and then vectorizes | |
| the sequence as articulatory features | |
| """ | |
| if input_phonemes: | |
| phones = text | |
| else: | |
| phones = self.get_phone_string(text=text, include_eos_symbol=True) | |
| if view: | |
| print("Phonemes: \n{}\n".format(phones)) | |
| phones_vector = list() | |
| # turn into numeric vectors | |
| for char in phones: | |
| if handle_missing: | |
| try: | |
| phones_vector.append(self.phone_to_vector[char]) | |
| except KeyError: | |
| print("unknown phoneme: {}".format(char)) | |
| else: | |
| phones_vector.append(self.phone_to_vector[char]) # leave error handling to elsewhere | |
| return torch.Tensor(phones_vector, device=device) | |
| def get_phone_string(self, text, include_eos_symbol=True): | |
| # expand abbreviations | |
| utt = self.expand_abbreviations(text) | |
| # phonemize | |
| phones = phonemizer.phonemize(utt, | |
| language_switch='remove-flags', | |
| backend="espeak", | |
| language=self.g2p_lang, | |
| preserve_punctuation=True, | |
| strip=True, | |
| punctuation_marks=';:,.!?¡¿—…"«»“”~/', | |
| with_stress=self.use_stress).replace(";", ",").replace("/", " ").replace("—", "") \ | |
| .replace(":", ",").replace('"', ",").replace("-", ",").replace("...", ",").replace("-", ",").replace("\n", " ") \ | |
| .replace("\t", " ").replace("¡", "").replace("¿", "").replace(",", "~").replace(" ̃", "").replace('̩', "").replace("̃", "").replace("̪", "") | |
| # less than 1 wide characters hidden here | |
| phones = re.sub("~+", "~", phones) | |
| if not self.use_prosody: | |
| # retain ~ as heuristic pause marker, even though all other symbols are removed with this option. | |
| # also retain . ? and ! since they can be indicators for the stop token | |
| phones = phones.replace("ˌ", "").replace("ː", "").replace("ˑ", "") \ | |
| .replace("˘", "").replace("|", "").replace("‖", "") | |
| if not self.use_word_boundaries: | |
| phones = phones.replace(" ", "") | |
| else: | |
| phones = re.sub(r"\s+", " ", phones) | |
| phones = re.sub(" ", "~", phones) | |
| if self.strip_silence: | |
| phones = phones.lstrip("~").rstrip("~") | |
| if self.add_silence_to_end: | |
| phones += "~" # adding a silence in the end during add_silence_to_end produces more natural sounding prosody | |
| if include_eos_symbol: | |
| phones += "#" | |
| phones = "~" + phones | |
| phones = re.sub("~+", "~", phones) | |
| return phones | |
| def english_text_expansion(text): | |
| """ | |
| Apply as small part of the tacotron style text cleaning pipeline, suitable for e.g. LJSpeech. | |
| See https://github.com/keithito/tacotron/ | |
| Careful: Only apply to english datasets. Different languages need different cleaners. | |
| """ | |
| _abbreviations = [(re.compile('\\b%s\\.' % x[0], re.IGNORECASE), x[1]) for x in | |
| [('Mrs.', 'misess'), ('Mr.', 'mister'), ('Dr.', 'doctor'), ('St.', 'saint'), ('Co.', 'company'), ('Jr.', 'junior'), ('Maj.', 'major'), | |
| ('Gen.', 'general'), ('Drs.', 'doctors'), ('Rev.', 'reverend'), ('Lt.', 'lieutenant'), ('Hon.', 'honorable'), ('Sgt.', 'sergeant'), | |
| ('Capt.', 'captain'), ('Esq.', 'esquire'), ('Ltd.', 'limited'), ('Col.', 'colonel'), ('Ft.', 'fort')]] | |
| for regex, replacement in _abbreviations: | |
| text = re.sub(regex, replacement, text) | |
| return text | |
| def get_language_id(language): | |
| if language == "en": | |
| return torch.LongTensor([0]) | |
| elif language == "de": | |
| return torch.LongTensor([1]) | |
| elif language == "el": | |
| return torch.LongTensor([2]) | |
| elif language == "es": | |
| return torch.LongTensor([3]) | |
| elif language == "fi": | |
| return torch.LongTensor([4]) | |
| elif language == "ru": | |
| return torch.LongTensor([5]) | |
| elif language == "hu": | |
| return torch.LongTensor([6]) | |
| elif language == "nl": | |
| return torch.LongTensor([7]) | |
| elif language == "fr": | |
| return torch.LongTensor([8]) | |
| elif language == "pt": | |
| return torch.LongTensor([9]) | |
| elif language == "pl": | |
| return torch.LongTensor([10]) | |
| elif language == "it": | |
| return torch.LongTensor([11]) | |
| if __name__ == '__main__': | |
| # test an English utterance | |
| tfr_en = ArticulatoryCombinedTextFrontend(language="en") | |
| print(tfr_en.string_to_tensor("This is a complex sentence, it even has a pause! But can it do this? Nice.", view=True)) | |
| tfr_en = ArticulatoryCombinedTextFrontend(language="de") | |
| print(tfr_en.string_to_tensor("Alles klar, jetzt testen wir einen deutschen Satz. Ich hoffe es gibt nicht mehr viele unspezifizierte Phoneme.", view=True)) | |