Spaces:
Running
Running
# modified from https://github.com/CjangCjengh/vits/blob/main/text/japanese.py | |
import re | |
import sys | |
import pyopenjtalk | |
from text import symbols | |
# Regular expression matching Japanese without punctuation marks: | |
_japanese_characters = re.compile( | |
r"[A-Za-z\d\u3005\u3040-\u30ff\u4e00-\u9fff\uff11-\uff19\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d]" | |
) | |
# Regular expression matching non-Japanese characters or punctuation marks: | |
_japanese_marks = re.compile( | |
r"[^A-Za-z\d\u3005\u3040-\u30ff\u4e00-\u9fff\uff11-\uff19\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d]" | |
) | |
# List of (symbol, Japanese) pairs for marks: | |
_symbols_to_japanese = [(re.compile("%s" % x[0]), x[1]) for x in [("%", "パーセント")]] | |
# List of (consonant, sokuon) pairs: | |
_real_sokuon = [ | |
(re.compile("%s" % x[0]), x[1]) | |
for x in [ | |
(r"Q([↑↓]*[kg])", r"k#\1"), | |
(r"Q([↑↓]*[tdjʧ])", r"t#\1"), | |
(r"Q([↑↓]*[sʃ])", r"s\1"), | |
(r"Q([↑↓]*[pb])", r"p#\1"), | |
] | |
] | |
# List of (consonant, hatsuon) pairs: | |
_real_hatsuon = [ | |
(re.compile("%s" % x[0]), x[1]) | |
for x in [ | |
(r"N([↑↓]*[pbm])", r"m\1"), | |
(r"N([↑↓]*[ʧʥj])", r"n^\1"), | |
(r"N([↑↓]*[tdn])", r"n\1"), | |
(r"N([↑↓]*[kg])", r"ŋ\1"), | |
] | |
] | |
def post_replace_ph(ph): | |
rep_map = { | |
":": ",", | |
";": ",", | |
",": ",", | |
"。": ".", | |
"!": "!", | |
"?": "?", | |
"\n": ".", | |
"·": ",", | |
"、": ",", | |
"...": "…", | |
} | |
if ph in rep_map.keys(): | |
ph = rep_map[ph] | |
if ph in symbols: | |
return ph | |
if ph not in symbols: | |
ph = "UNK" | |
return ph | |
def symbols_to_japanese(text): | |
for regex, replacement in _symbols_to_japanese: | |
text = re.sub(regex, replacement, text) | |
return text | |
def preprocess_jap(text): | |
"""Reference https://r9y9.github.io/ttslearn/latest/notebooks/ch10_Recipe-Tacotron.html""" | |
text = symbols_to_japanese(text) | |
sentences = re.split(_japanese_marks, text) | |
marks = re.findall(_japanese_marks, text) | |
text = [] | |
for i, sentence in enumerate(sentences): | |
if re.match(_japanese_characters, sentence): | |
p = pyopenjtalk.g2p(sentence) | |
text += p.split(" ") | |
if i < len(marks): | |
text += [marks[i].replace(" ", "")] | |
return text | |
def text_normalize(text): | |
# todo: jap text normalize | |
return text | |
def g2p(norm_text): | |
phones = preprocess_jap(norm_text) | |
phones = [post_replace_ph(i) for i in phones] | |
# todo: implement tones and word2ph | |
return phones | |
if __name__ == "__main__": | |
for line in open("../../../Downloads/transcript_utf8.txt").readlines(): | |
text = line.split(":")[1] | |
phones = g2p(text) | |
print(phones) | |