Spaces:
Sleeping
Sleeping
Commit
•
693cbce
1
Parent(s):
325232c
Upload 5 files
Browse files- text/LICENSE +19 -0
- text/__init__.py +32 -0
- text/cleaners.py +11 -0
- text/korean.py +210 -0
- text/symbols.py +74 -0
text/LICENSE
ADDED
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
Copyright (c) 2017 Keith Ito
|
2 |
+
|
3 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
4 |
+
of this software and associated documentation files (the "Software"), to deal
|
5 |
+
in the Software without restriction, including without limitation the rights
|
6 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
7 |
+
copies of the Software, and to permit persons to whom the Software is
|
8 |
+
furnished to do so, subject to the following conditions:
|
9 |
+
|
10 |
+
The above copyright notice and this permission notice shall be included in
|
11 |
+
all copies or substantial portions of the Software.
|
12 |
+
|
13 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
14 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
15 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
16 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
17 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
18 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
19 |
+
THE SOFTWARE.
|
text/__init__.py
ADDED
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
""" from https://github.com/keithito/tacotron """
|
2 |
+
from text import cleaners
|
3 |
+
|
4 |
+
|
5 |
+
def text_to_sequence(text, symbols, cleaner_names):
|
6 |
+
'''Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
|
7 |
+
Args:
|
8 |
+
text: string to convert to a sequence
|
9 |
+
cleaner_names: names of the cleaner functions to run the text through
|
10 |
+
Returns:
|
11 |
+
List of integers corresponding to the symbols in the text
|
12 |
+
'''
|
13 |
+
_symbol_to_id = {s: i for i, s in enumerate(symbols)}
|
14 |
+
|
15 |
+
sequence = []
|
16 |
+
|
17 |
+
clean_text = _clean_text(text, cleaner_names)
|
18 |
+
for symbol in clean_text:
|
19 |
+
if symbol not in _symbol_to_id.keys():
|
20 |
+
continue
|
21 |
+
symbol_id = _symbol_to_id[symbol]
|
22 |
+
sequence += [symbol_id]
|
23 |
+
return sequence
|
24 |
+
|
25 |
+
|
26 |
+
def _clean_text(text, cleaner_names):
|
27 |
+
for name in cleaner_names:
|
28 |
+
cleaner = getattr(cleaners, name)
|
29 |
+
if not cleaner:
|
30 |
+
raise Exception('Unknown cleaner: %s' % name)
|
31 |
+
text = cleaner(text)
|
32 |
+
return text
|
text/cleaners.py
ADDED
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import re
|
2 |
+
from text.korean import latin_to_hangul, number_to_hangul, divide_hangul, korean_to_lazy_ipa, korean_to_ipa
|
3 |
+
|
4 |
+
def korean_cleaners(text):
|
5 |
+
'''Pipeline for Korean text'''
|
6 |
+
text = latin_to_hangul(text)
|
7 |
+
text = number_to_hangul(text)
|
8 |
+
text = divide_hangul(text)
|
9 |
+
if re.match('[\u3131-\u3163]', text[-1]):
|
10 |
+
text += '.'
|
11 |
+
return text
|
text/korean.py
ADDED
@@ -0,0 +1,210 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import re
|
2 |
+
from jamo import h2j, j2hcj
|
3 |
+
import ko_pron
|
4 |
+
|
5 |
+
|
6 |
+
# This is a list of Korean classifiers preceded by pure Korean numerals.
|
7 |
+
_korean_classifiers = '군데 권 개 그루 닢 대 두 마리 모 모금 뭇 발 발짝 방 번 벌 보루 살 수 술 시 쌈 움큼 정 짝 채 척 첩 축 켤레 톨 통'
|
8 |
+
|
9 |
+
# List of (hangul, hangul divided) pairs:
|
10 |
+
_hangul_divided = [(re.compile('%s' % x[0]), x[1]) for x in [
|
11 |
+
('ㄳ', 'ㄱㅅ'),
|
12 |
+
('ㄵ', 'ㄴㅈ'),
|
13 |
+
('ㄶ', 'ㄴㅎ'),
|
14 |
+
('ㄺ', 'ㄹㄱ'),
|
15 |
+
('ㄻ', 'ㄹㅁ'),
|
16 |
+
('ㄼ', 'ㄹㅂ'),
|
17 |
+
('ㄽ', 'ㄹㅅ'),
|
18 |
+
('ㄾ', 'ㄹㅌ'),
|
19 |
+
('ㄿ', 'ㄹㅍ'),
|
20 |
+
('ㅀ', 'ㄹㅎ'),
|
21 |
+
('ㅄ', 'ㅂㅅ'),
|
22 |
+
('ㅘ', 'ㅗㅏ'),
|
23 |
+
('ㅙ', 'ㅗㅐ'),
|
24 |
+
('ㅚ', 'ㅗㅣ'),
|
25 |
+
('ㅝ', 'ㅜㅓ'),
|
26 |
+
('ㅞ', 'ㅜㅔ'),
|
27 |
+
('ㅟ', 'ㅜㅣ'),
|
28 |
+
('ㅢ', 'ㅡㅣ'),
|
29 |
+
('ㅑ', 'ㅣㅏ'),
|
30 |
+
('ㅒ', 'ㅣㅐ'),
|
31 |
+
('ㅕ', 'ㅣㅓ'),
|
32 |
+
('ㅖ', 'ㅣㅔ'),
|
33 |
+
('ㅛ', 'ㅣㅗ'),
|
34 |
+
('ㅠ', 'ㅣㅜ')
|
35 |
+
]]
|
36 |
+
|
37 |
+
# List of (Latin alphabet, hangul) pairs:
|
38 |
+
_latin_to_hangul = [(re.compile('%s' % x[0], re.IGNORECASE), x[1]) for x in [
|
39 |
+
('a', '에이'),
|
40 |
+
('b', '비'),
|
41 |
+
('c', '시'),
|
42 |
+
('d', '디'),
|
43 |
+
('e', '이'),
|
44 |
+
('f', '에프'),
|
45 |
+
('g', '지'),
|
46 |
+
('h', '에이치'),
|
47 |
+
('i', '아이'),
|
48 |
+
('j', '제이'),
|
49 |
+
('k', '케이'),
|
50 |
+
('l', '엘'),
|
51 |
+
('m', '엠'),
|
52 |
+
('n', '엔'),
|
53 |
+
('o', '오'),
|
54 |
+
('p', '피'),
|
55 |
+
('q', '큐'),
|
56 |
+
('r', '아르'),
|
57 |
+
('s', '에스'),
|
58 |
+
('t', '티'),
|
59 |
+
('u', '유'),
|
60 |
+
('v', '브이'),
|
61 |
+
('w', '더블유'),
|
62 |
+
('x', '엑스'),
|
63 |
+
('y', '와이'),
|
64 |
+
('z', '제트')
|
65 |
+
]]
|
66 |
+
|
67 |
+
# List of (ipa, lazy ipa) pairs:
|
68 |
+
_ipa_to_lazy_ipa = [(re.compile('%s' % x[0], re.IGNORECASE), x[1]) for x in [
|
69 |
+
('t͡ɕ','ʧ'),
|
70 |
+
('d͡ʑ','ʥ'),
|
71 |
+
('ɲ','n^'),
|
72 |
+
('ɕ','ʃ'),
|
73 |
+
('ʷ','w'),
|
74 |
+
('ɭ','l`'),
|
75 |
+
('ʎ','ɾ'),
|
76 |
+
('ɣ','ŋ'),
|
77 |
+
('ɰ','ɯ'),
|
78 |
+
('ʝ','j'),
|
79 |
+
('ʌ','ə'),
|
80 |
+
('ɡ','g'),
|
81 |
+
('\u031a','#'),
|
82 |
+
('\u0348','='),
|
83 |
+
('\u031e',''),
|
84 |
+
('\u0320',''),
|
85 |
+
('\u0339','')
|
86 |
+
]]
|
87 |
+
|
88 |
+
|
89 |
+
def latin_to_hangul(text):
|
90 |
+
for regex, replacement in _latin_to_hangul:
|
91 |
+
text = re.sub(regex, replacement, text)
|
92 |
+
return text
|
93 |
+
|
94 |
+
|
95 |
+
def divide_hangul(text):
|
96 |
+
text = j2hcj(h2j(text))
|
97 |
+
for regex, replacement in _hangul_divided:
|
98 |
+
text = re.sub(regex, replacement, text)
|
99 |
+
return text
|
100 |
+
|
101 |
+
|
102 |
+
def hangul_number(num, sino=True):
|
103 |
+
'''Reference https://github.com/Kyubyong/g2pK'''
|
104 |
+
num = re.sub(',', '', num)
|
105 |
+
|
106 |
+
if num == '0':
|
107 |
+
return '영'
|
108 |
+
if not sino and num == '20':
|
109 |
+
return '스무'
|
110 |
+
|
111 |
+
digits = '123456789'
|
112 |
+
names = '일이삼사오육칠팔구'
|
113 |
+
digit2name = {d: n for d, n in zip(digits, names)}
|
114 |
+
|
115 |
+
modifiers = '한 두 세 네 다섯 여섯 일곱 여덟 아홉'
|
116 |
+
decimals = '열 스물 서른 마흔 쉰 예순 일흔 여든 아흔'
|
117 |
+
digit2mod = {d: mod for d, mod in zip(digits, modifiers.split())}
|
118 |
+
digit2dec = {d: dec for d, dec in zip(digits, decimals.split())}
|
119 |
+
|
120 |
+
spelledout = []
|
121 |
+
for i, digit in enumerate(num):
|
122 |
+
i = len(num) - i - 1
|
123 |
+
if sino:
|
124 |
+
if i == 0:
|
125 |
+
name = digit2name.get(digit, '')
|
126 |
+
elif i == 1:
|
127 |
+
name = digit2name.get(digit, '') + '십'
|
128 |
+
name = name.replace('일십', '십')
|
129 |
+
else:
|
130 |
+
if i == 0:
|
131 |
+
name = digit2mod.get(digit, '')
|
132 |
+
elif i == 1:
|
133 |
+
name = digit2dec.get(digit, '')
|
134 |
+
if digit == '0':
|
135 |
+
if i % 4 == 0:
|
136 |
+
last_three = spelledout[-min(3, len(spelledout)):]
|
137 |
+
if ''.join(last_three) == '':
|
138 |
+
spelledout.append('')
|
139 |
+
continue
|
140 |
+
else:
|
141 |
+
spelledout.append('')
|
142 |
+
continue
|
143 |
+
if i == 2:
|
144 |
+
name = digit2name.get(digit, '') + '백'
|
145 |
+
name = name.replace('일백', '백')
|
146 |
+
elif i == 3:
|
147 |
+
name = digit2name.get(digit, '') + '천'
|
148 |
+
name = name.replace('일천', '천')
|
149 |
+
elif i == 4:
|
150 |
+
name = digit2name.get(digit, '') + '만'
|
151 |
+
name = name.replace('일만', '만')
|
152 |
+
elif i == 5:
|
153 |
+
name = digit2name.get(digit, '') + '십'
|
154 |
+
name = name.replace('일십', '십')
|
155 |
+
elif i == 6:
|
156 |
+
name = digit2name.get(digit, '') + '백'
|
157 |
+
name = name.replace('일백', '백')
|
158 |
+
elif i == 7:
|
159 |
+
name = digit2name.get(digit, '') + '천'
|
160 |
+
name = name.replace('일천', '천')
|
161 |
+
elif i == 8:
|
162 |
+
name = digit2name.get(digit, '') + '억'
|
163 |
+
elif i == 9:
|
164 |
+
name = digit2name.get(digit, '') + '십'
|
165 |
+
elif i == 10:
|
166 |
+
name = digit2name.get(digit, '') + '백'
|
167 |
+
elif i == 11:
|
168 |
+
name = digit2name.get(digit, '') + '천'
|
169 |
+
elif i == 12:
|
170 |
+
name = digit2name.get(digit, '') + '조'
|
171 |
+
elif i == 13:
|
172 |
+
name = digit2name.get(digit, '') + '십'
|
173 |
+
elif i == 14:
|
174 |
+
name = digit2name.get(digit, '') + '백'
|
175 |
+
elif i == 15:
|
176 |
+
name = digit2name.get(digit, '') + '천'
|
177 |
+
spelledout.append(name)
|
178 |
+
return ''.join(elem for elem in spelledout)
|
179 |
+
|
180 |
+
|
181 |
+
def number_to_hangul(text):
|
182 |
+
'''Reference https://github.com/Kyubyong/g2pK'''
|
183 |
+
tokens = set(re.findall(r'(\d[\d,]*)([\uac00-\ud71f]+)', text))
|
184 |
+
for token in tokens:
|
185 |
+
num, classifier = token
|
186 |
+
if classifier[:2] in _korean_classifiers or classifier[0] in _korean_classifiers:
|
187 |
+
spelledout = hangul_number(num, sino=False)
|
188 |
+
else:
|
189 |
+
spelledout = hangul_number(num, sino=True)
|
190 |
+
text = text.replace(f'{num}{classifier}', f'{spelledout}{classifier}')
|
191 |
+
# digit by digit for remaining digits
|
192 |
+
digits = '0123456789'
|
193 |
+
names = '영일이삼사오육칠팔구'
|
194 |
+
for d, n in zip(digits, names):
|
195 |
+
text = text.replace(d, n)
|
196 |
+
return text
|
197 |
+
|
198 |
+
|
199 |
+
def korean_to_lazy_ipa(text):
|
200 |
+
text = latin_to_hangul(text)
|
201 |
+
text = number_to_hangul(text)
|
202 |
+
text=re.sub('[\uac00-\ud7af]+',lambda x:ko_pron.romanise(x.group(0),'ipa').split('] ~ [')[0],text)
|
203 |
+
for regex, replacement in _ipa_to_lazy_ipa:
|
204 |
+
text = re.sub(regex, replacement, text)
|
205 |
+
return text
|
206 |
+
|
207 |
+
|
208 |
+
def korean_to_ipa(text):
|
209 |
+
text = korean_to_lazy_ipa(text)
|
210 |
+
return text.replace('ʧ','tʃ').replace('ʥ','dʑ')
|
text/symbols.py
ADDED
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
'''
|
2 |
+
Defines the set of symbols used in text input to the model.
|
3 |
+
'''
|
4 |
+
|
5 |
+
'''# japanese_cleaners
|
6 |
+
_pad = '_'
|
7 |
+
_punctuation = ',.!?-'
|
8 |
+
_letters = 'AEINOQUabdefghijkmnoprstuvwyzʃʧ↓↑ '
|
9 |
+
'''
|
10 |
+
|
11 |
+
'''# japanese_cleaners2
|
12 |
+
_pad = '_'
|
13 |
+
_punctuation = ',.!?-~…'
|
14 |
+
_letters = 'AEINOQUabdefghijkmnoprstuvwyzʃʧʦ↓↑ '
|
15 |
+
'''
|
16 |
+
|
17 |
+
# korean_cleaners
|
18 |
+
_pad = '_'
|
19 |
+
_punctuation = ',.!?…~'
|
20 |
+
_letters = 'ㄱㄴㄷㄹㅁㅂㅅㅇㅈㅊㅋㅌㅍㅎㄲㄸㅃㅆㅉㅏㅓㅗㅜㅡㅣㅐㅔ '
|
21 |
+
|
22 |
+
'''# chinese_cleaners
|
23 |
+
_pad = '_'
|
24 |
+
_punctuation = ',。!?—…'
|
25 |
+
_letters = 'ㄅㄆㄇㄈㄉㄊㄋㄌㄍㄎㄏㄐㄑㄒㄓㄔㄕㄖㄗㄘㄙㄚㄛㄜㄝㄞㄟㄠㄡㄢㄣㄤㄥㄦㄧㄨㄩˉˊˇˋ˙ '
|
26 |
+
'''
|
27 |
+
|
28 |
+
'''# zh_ja_mixture_cleaners
|
29 |
+
_pad = '_'
|
30 |
+
_punctuation = ',.!?-~…'
|
31 |
+
_letters = 'AEINOQUabdefghijklmnoprstuvwyzʃʧʦɯɹəɥ⁼ʰ`→↓↑ '
|
32 |
+
'''
|
33 |
+
|
34 |
+
'''# sanskrit_cleaners
|
35 |
+
_pad = '_'
|
36 |
+
_punctuation = '।'
|
37 |
+
_letters = 'ँंःअआइईउऊऋएऐओऔकखगघङचछजझञटठडढणतथदधनपफबभमयरलळवशषसहऽािीुूृॄेैोौ्ॠॢ '
|
38 |
+
'''
|
39 |
+
|
40 |
+
'''# cjks_cleaners
|
41 |
+
_pad = '_'
|
42 |
+
_punctuation = ',.!?-~…'
|
43 |
+
_letters = 'NQabdefghijklmnopstuvwxyzʃʧʥʦɯɹəɥçɸɾβŋɦː⁼ʰ`^#*=→↓↑ '
|
44 |
+
'''
|
45 |
+
|
46 |
+
'''# thai_cleaners
|
47 |
+
_pad = '_'
|
48 |
+
_punctuation = '.!? '
|
49 |
+
_letters = 'กขฃคฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลวศษสหฬอฮฯะัาำิีึืุูเแโใไๅๆ็่้๊๋์'
|
50 |
+
'''
|
51 |
+
|
52 |
+
'''# cjke_cleaners2
|
53 |
+
_pad = '_'
|
54 |
+
_punctuation = ',.!?-~…'
|
55 |
+
_letters = 'NQabdefghijklmnopstuvwxyzɑæʃʑçɯɪɔɛɹðəɫɥɸʊɾʒθβŋɦ⁼ʰ`^#*=ˈˌ→↓↑ '
|
56 |
+
'''
|
57 |
+
|
58 |
+
'''# shanghainese_cleaners
|
59 |
+
_pad = '_'
|
60 |
+
_punctuation = ',.!?…'
|
61 |
+
_letters = 'abdfghiklmnopstuvyzøŋȵɑɔɕəɤɦɪɿʑʔʰ̩̃ᴀᴇ15678 '
|
62 |
+
'''
|
63 |
+
|
64 |
+
'''# chinese_dialect_cleaners
|
65 |
+
_pad = '_'
|
66 |
+
_punctuation = ',.!?~…─'
|
67 |
+
_letters = '#Nabdefghijklmnoprstuvwxyzæçøŋœȵɐɑɒɓɔɕɗɘəɚɛɜɣɤɦɪɭɯɵɷɸɻɾɿʂʅʊʋʌʏʑʔʦʮʰʷˀː˥˦˧˨˩̥̩̃̚αᴀᴇ↑↓∅ⱼ '
|
68 |
+
'''
|
69 |
+
|
70 |
+
# Export all symbols:
|
71 |
+
symbols = [_pad] + list(_punctuation) + list(_letters)
|
72 |
+
|
73 |
+
# Special symbol ids
|
74 |
+
SPACE_ID = symbols.index(" ")
|