|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import re |
|
|
|
from tensorflow_tts.utils.korean import tokenize as ko_tokenize |
|
from tensorflow_tts.utils.number_norm import normalize_numbers |
|
from unidecode import unidecode |
|
|
|
try: |
|
from german_transliterate.core import GermanTransliterate |
|
except: |
|
pass |
|
|
|
|
|
_whitespace_re = re.compile(r"\s+") |
|
|
|
|
|
_abbreviations = [ |
|
(re.compile("\\b%s\\." % x[0], re.IGNORECASE), x[1]) |
|
for x in [ |
|
("mrs", "misess"), |
|
("mr", "mister"), |
|
("dr", "doctor"), |
|
("st", "saint"), |
|
("co", "company"), |
|
("jr", "junior"), |
|
("maj", "major"), |
|
("gen", "general"), |
|
("drs", "doctors"), |
|
("rev", "reverend"), |
|
("lt", "lieutenant"), |
|
("hon", "honorable"), |
|
("sgt", "sergeant"), |
|
("capt", "captain"), |
|
("esq", "esquire"), |
|
("ltd", "limited"), |
|
("col", "colonel"), |
|
("ft", "fort"), |
|
] |
|
] |
|
|
|
|
|
def expand_abbreviations(text): |
|
for regex, replacement in _abbreviations: |
|
text = re.sub(regex, replacement, text) |
|
return text |
|
|
|
|
|
def expand_numbers(text): |
|
return normalize_numbers(text) |
|
|
|
|
|
def lowercase(text): |
|
return text.lower() |
|
|
|
|
|
def collapse_whitespace(text): |
|
return re.sub(_whitespace_re, " ", text) |
|
|
|
|
|
def convert_to_ascii(text): |
|
return unidecode(text) |
|
|
|
|
|
def basic_cleaners(text): |
|
"""Basic pipeline that lowercases and collapses whitespace without transliteration.""" |
|
text = lowercase(text) |
|
text = collapse_whitespace(text) |
|
return text |
|
|
|
|
|
def transliteration_cleaners(text): |
|
"""Pipeline for non-English text that transliterates to ASCII.""" |
|
text = convert_to_ascii(text) |
|
text = lowercase(text) |
|
text = collapse_whitespace(text) |
|
return text |
|
|
|
|
|
def english_cleaners(text): |
|
"""Pipeline for English text, including number and abbreviation expansion.""" |
|
text = convert_to_ascii(text) |
|
text = lowercase(text) |
|
text = expand_numbers(text) |
|
text = expand_abbreviations(text) |
|
text = collapse_whitespace(text) |
|
return text |
|
|
|
|
|
def korean_cleaners(text): |
|
"""Pipeline for Korean text, including number and abbreviation expansion.""" |
|
text = ko_tokenize( |
|
text |
|
) |
|
return text |
|
|
|
def german_cleaners(text): |
|
"""Pipeline for German text, including number and abbreviation expansion.""" |
|
try: |
|
text = GermanTransliterate(replace={';': ',', ':': ' '}, sep_abbreviation=' -- ').transliterate(text) |
|
except NameError: |
|
raise ModuleNotFoundError("Install german_transliterate package to use german_cleaners") |
|
return text |