ORI-Muchim commited on
Commit
693cbce
1 Parent(s): 325232c

Upload 5 files

Browse files
Files changed (5) hide show
  1. text/LICENSE +19 -0
  2. text/__init__.py +32 -0
  3. text/cleaners.py +11 -0
  4. text/korean.py +210 -0
  5. text/symbols.py +74 -0
text/LICENSE ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Copyright (c) 2017 Keith Ito
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining a copy
4
+ of this software and associated documentation files (the "Software"), to deal
5
+ in the Software without restriction, including without limitation the rights
6
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
+ copies of the Software, and to permit persons to whom the Software is
8
+ furnished to do so, subject to the following conditions:
9
+
10
+ The above copyright notice and this permission notice shall be included in
11
+ all copies or substantial portions of the Software.
12
+
13
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19
+ THE SOFTWARE.
text/__init__.py ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """ from https://github.com/keithito/tacotron """
2
+ from text import cleaners
3
+
4
+
5
+ def text_to_sequence(text, symbols, cleaner_names):
6
+ '''Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
7
+ Args:
8
+ text: string to convert to a sequence
9
+ cleaner_names: names of the cleaner functions to run the text through
10
+ Returns:
11
+ List of integers corresponding to the symbols in the text
12
+ '''
13
+ _symbol_to_id = {s: i for i, s in enumerate(symbols)}
14
+
15
+ sequence = []
16
+
17
+ clean_text = _clean_text(text, cleaner_names)
18
+ for symbol in clean_text:
19
+ if symbol not in _symbol_to_id.keys():
20
+ continue
21
+ symbol_id = _symbol_to_id[symbol]
22
+ sequence += [symbol_id]
23
+ return sequence
24
+
25
+
26
+ def _clean_text(text, cleaner_names):
27
+ for name in cleaner_names:
28
+ cleaner = getattr(cleaners, name)
29
+ if not cleaner:
30
+ raise Exception('Unknown cleaner: %s' % name)
31
+ text = cleaner(text)
32
+ return text
text/cleaners.py ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+ from text.korean import latin_to_hangul, number_to_hangul, divide_hangul, korean_to_lazy_ipa, korean_to_ipa
3
+
4
+ def korean_cleaners(text):
5
+ '''Pipeline for Korean text'''
6
+ text = latin_to_hangul(text)
7
+ text = number_to_hangul(text)
8
+ text = divide_hangul(text)
9
+ if re.match('[\u3131-\u3163]', text[-1]):
10
+ text += '.'
11
+ return text
text/korean.py ADDED
@@ -0,0 +1,210 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+ from jamo import h2j, j2hcj
3
+ import ko_pron
4
+
5
+
6
+ # This is a list of Korean classifiers preceded by pure Korean numerals.
7
+ _korean_classifiers = '군데 권 개 그루 닢 대 두 마리 모 모금 뭇 발 발짝 방 번 벌 보루 살 수 술 시 쌈 움큼 정 짝 채 척 첩 축 켤레 톨 통'
8
+
9
+ # List of (hangul, hangul divided) pairs:
10
+ _hangul_divided = [(re.compile('%s' % x[0]), x[1]) for x in [
11
+ ('ㄳ', 'ㄱㅅ'),
12
+ ('ㄵ', 'ㄴㅈ'),
13
+ ('ㄶ', 'ㄴㅎ'),
14
+ ('ㄺ', 'ㄹㄱ'),
15
+ ('ㄻ', 'ㄹㅁ'),
16
+ ('ㄼ', 'ㄹㅂ'),
17
+ ('ㄽ', 'ㄹㅅ'),
18
+ ('ㄾ', 'ㄹㅌ'),
19
+ ('ㄿ', 'ㄹㅍ'),
20
+ ('ㅀ', 'ㄹㅎ'),
21
+ ('ㅄ', 'ㅂㅅ'),
22
+ ('ㅘ', 'ㅗㅏ'),
23
+ ('ㅙ', 'ㅗㅐ'),
24
+ ('ㅚ', 'ㅗㅣ'),
25
+ ('ㅝ', 'ㅜㅓ'),
26
+ ('ㅞ', 'ㅜㅔ'),
27
+ ('ㅟ', 'ㅜㅣ'),
28
+ ('ㅢ', 'ㅡㅣ'),
29
+ ('ㅑ', 'ㅣㅏ'),
30
+ ('ㅒ', 'ㅣㅐ'),
31
+ ('ㅕ', 'ㅣㅓ'),
32
+ ('ㅖ', 'ㅣㅔ'),
33
+ ('ㅛ', 'ㅣㅗ'),
34
+ ('ㅠ', 'ㅣㅜ')
35
+ ]]
36
+
37
+ # List of (Latin alphabet, hangul) pairs:
38
+ _latin_to_hangul = [(re.compile('%s' % x[0], re.IGNORECASE), x[1]) for x in [
39
+ ('a', '에이'),
40
+ ('b', '비'),
41
+ ('c', '시'),
42
+ ('d', '디'),
43
+ ('e', '이'),
44
+ ('f', '에프'),
45
+ ('g', '지'),
46
+ ('h', '에이치'),
47
+ ('i', '아이'),
48
+ ('j', '제이'),
49
+ ('k', '케이'),
50
+ ('l', '엘'),
51
+ ('m', '엠'),
52
+ ('n', '엔'),
53
+ ('o', '오'),
54
+ ('p', '피'),
55
+ ('q', '큐'),
56
+ ('r', '아르'),
57
+ ('s', '에스'),
58
+ ('t', '티'),
59
+ ('u', '유'),
60
+ ('v', '브이'),
61
+ ('w', '더블유'),
62
+ ('x', '엑스'),
63
+ ('y', '와이'),
64
+ ('z', '제트')
65
+ ]]
66
+
67
+ # List of (ipa, lazy ipa) pairs:
68
+ _ipa_to_lazy_ipa = [(re.compile('%s' % x[0], re.IGNORECASE), x[1]) for x in [
69
+ ('t͡ɕ','ʧ'),
70
+ ('d͡ʑ','ʥ'),
71
+ ('ɲ','n^'),
72
+ ('ɕ','ʃ'),
73
+ ('ʷ','w'),
74
+ ('ɭ','l`'),
75
+ ('ʎ','ɾ'),
76
+ ('ɣ','ŋ'),
77
+ ('ɰ','ɯ'),
78
+ ('ʝ','j'),
79
+ ('ʌ','ə'),
80
+ ('ɡ','g'),
81
+ ('\u031a','#'),
82
+ ('\u0348','='),
83
+ ('\u031e',''),
84
+ ('\u0320',''),
85
+ ('\u0339','')
86
+ ]]
87
+
88
+
89
+ def latin_to_hangul(text):
90
+ for regex, replacement in _latin_to_hangul:
91
+ text = re.sub(regex, replacement, text)
92
+ return text
93
+
94
+
95
+ def divide_hangul(text):
96
+ text = j2hcj(h2j(text))
97
+ for regex, replacement in _hangul_divided:
98
+ text = re.sub(regex, replacement, text)
99
+ return text
100
+
101
+
102
+ def hangul_number(num, sino=True):
103
+ '''Reference https://github.com/Kyubyong/g2pK'''
104
+ num = re.sub(',', '', num)
105
+
106
+ if num == '0':
107
+ return '영'
108
+ if not sino and num == '20':
109
+ return '스무'
110
+
111
+ digits = '123456789'
112
+ names = '일이삼사오육칠팔구'
113
+ digit2name = {d: n for d, n in zip(digits, names)}
114
+
115
+ modifiers = '한 두 세 네 다섯 여섯 일곱 여덟 아홉'
116
+ decimals = '열 스물 서른 마흔 쉰 예순 일흔 여든 아흔'
117
+ digit2mod = {d: mod for d, mod in zip(digits, modifiers.split())}
118
+ digit2dec = {d: dec for d, dec in zip(digits, decimals.split())}
119
+
120
+ spelledout = []
121
+ for i, digit in enumerate(num):
122
+ i = len(num) - i - 1
123
+ if sino:
124
+ if i == 0:
125
+ name = digit2name.get(digit, '')
126
+ elif i == 1:
127
+ name = digit2name.get(digit, '') + '십'
128
+ name = name.replace('일십', '십')
129
+ else:
130
+ if i == 0:
131
+ name = digit2mod.get(digit, '')
132
+ elif i == 1:
133
+ name = digit2dec.get(digit, '')
134
+ if digit == '0':
135
+ if i % 4 == 0:
136
+ last_three = spelledout[-min(3, len(spelledout)):]
137
+ if ''.join(last_three) == '':
138
+ spelledout.append('')
139
+ continue
140
+ else:
141
+ spelledout.append('')
142
+ continue
143
+ if i == 2:
144
+ name = digit2name.get(digit, '') + '백'
145
+ name = name.replace('일백', '백')
146
+ elif i == 3:
147
+ name = digit2name.get(digit, '') + '천'
148
+ name = name.replace('일천', '천')
149
+ elif i == 4:
150
+ name = digit2name.get(digit, '') + '만'
151
+ name = name.replace('일만', '만')
152
+ elif i == 5:
153
+ name = digit2name.get(digit, '') + '십'
154
+ name = name.replace('일십', '십')
155
+ elif i == 6:
156
+ name = digit2name.get(digit, '') + '백'
157
+ name = name.replace('일백', '백')
158
+ elif i == 7:
159
+ name = digit2name.get(digit, '') + '천'
160
+ name = name.replace('일천', '천')
161
+ elif i == 8:
162
+ name = digit2name.get(digit, '') + '억'
163
+ elif i == 9:
164
+ name = digit2name.get(digit, '') + '십'
165
+ elif i == 10:
166
+ name = digit2name.get(digit, '') + '백'
167
+ elif i == 11:
168
+ name = digit2name.get(digit, '') + '천'
169
+ elif i == 12:
170
+ name = digit2name.get(digit, '') + '조'
171
+ elif i == 13:
172
+ name = digit2name.get(digit, '') + '십'
173
+ elif i == 14:
174
+ name = digit2name.get(digit, '') + '백'
175
+ elif i == 15:
176
+ name = digit2name.get(digit, '') + '천'
177
+ spelledout.append(name)
178
+ return ''.join(elem for elem in spelledout)
179
+
180
+
181
+ def number_to_hangul(text):
182
+ '''Reference https://github.com/Kyubyong/g2pK'''
183
+ tokens = set(re.findall(r'(\d[\d,]*)([\uac00-\ud71f]+)', text))
184
+ for token in tokens:
185
+ num, classifier = token
186
+ if classifier[:2] in _korean_classifiers or classifier[0] in _korean_classifiers:
187
+ spelledout = hangul_number(num, sino=False)
188
+ else:
189
+ spelledout = hangul_number(num, sino=True)
190
+ text = text.replace(f'{num}{classifier}', f'{spelledout}{classifier}')
191
+ # digit by digit for remaining digits
192
+ digits = '0123456789'
193
+ names = '영일이삼사오육칠팔구'
194
+ for d, n in zip(digits, names):
195
+ text = text.replace(d, n)
196
+ return text
197
+
198
+
199
+ def korean_to_lazy_ipa(text):
200
+ text = latin_to_hangul(text)
201
+ text = number_to_hangul(text)
202
+ text=re.sub('[\uac00-\ud7af]+',lambda x:ko_pron.romanise(x.group(0),'ipa').split('] ~ [')[0],text)
203
+ for regex, replacement in _ipa_to_lazy_ipa:
204
+ text = re.sub(regex, replacement, text)
205
+ return text
206
+
207
+
208
+ def korean_to_ipa(text):
209
+ text = korean_to_lazy_ipa(text)
210
+ return text.replace('ʧ','tʃ').replace('ʥ','dʑ')
text/symbols.py ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ '''
2
+ Defines the set of symbols used in text input to the model.
3
+ '''
4
+
5
+ '''# japanese_cleaners
6
+ _pad = '_'
7
+ _punctuation = ',.!?-'
8
+ _letters = 'AEINOQUabdefghijkmnoprstuvwyzʃʧ↓↑ '
9
+ '''
10
+
11
+ '''# japanese_cleaners2
12
+ _pad = '_'
13
+ _punctuation = ',.!?-~…'
14
+ _letters = 'AEINOQUabdefghijkmnoprstuvwyzʃʧʦ↓↑ '
15
+ '''
16
+
17
+ # korean_cleaners
18
+ _pad = '_'
19
+ _punctuation = ',.!?…~'
20
+ _letters = 'ㄱㄴㄷㄹㅁㅂㅅㅇㅈㅊㅋㅌㅍㅎㄲㄸㅃㅆㅉㅏㅓㅗㅜㅡㅣㅐㅔ '
21
+
22
+ '''# chinese_cleaners
23
+ _pad = '_'
24
+ _punctuation = ',。!?—…'
25
+ _letters = 'ㄅㄆㄇㄈㄉㄊㄋㄌㄍㄎㄏㄐㄑㄒㄓㄔㄕㄖㄗㄘㄙㄚㄛㄜㄝㄞㄟㄠㄡㄢㄣㄤㄥㄦㄧㄨㄩˉˊˇˋ˙ '
26
+ '''
27
+
28
+ '''# zh_ja_mixture_cleaners
29
+ _pad = '_'
30
+ _punctuation = ',.!?-~…'
31
+ _letters = 'AEINOQUabdefghijklmnoprstuvwyzʃʧʦɯɹəɥ⁼ʰ`→↓↑ '
32
+ '''
33
+
34
+ '''# sanskrit_cleaners
35
+ _pad = '_'
36
+ _punctuation = '।'
37
+ _letters = 'ँंःअआइईउऊऋएऐओऔकखगघङचछजझञटठडढणतथदधनपफबभमयरलळवशषसहऽािीुूृॄेैोौ्ॠॢ '
38
+ '''
39
+
40
+ '''# cjks_cleaners
41
+ _pad = '_'
42
+ _punctuation = ',.!?-~…'
43
+ _letters = 'NQabdefghijklmnopstuvwxyzʃʧʥʦɯɹəɥçɸɾβŋɦː⁼ʰ`^#*=→↓↑ '
44
+ '''
45
+
46
+ '''# thai_cleaners
47
+ _pad = '_'
48
+ _punctuation = '.!? '
49
+ _letters = 'กขฃคฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลวศษสหฬอฮฯะัาำิีึืุูเแโใไๅๆ็่้๊๋์'
50
+ '''
51
+
52
+ '''# cjke_cleaners2
53
+ _pad = '_'
54
+ _punctuation = ',.!?-~…'
55
+ _letters = 'NQabdefghijklmnopstuvwxyzɑæʃʑçɯɪɔɛɹðəɫɥɸʊɾʒθβŋɦ⁼ʰ`^#*=ˈˌ→↓↑ '
56
+ '''
57
+
58
+ '''# shanghainese_cleaners
59
+ _pad = '_'
60
+ _punctuation = ',.!?…'
61
+ _letters = 'abdfghiklmnopstuvyzøŋȵɑɔɕəɤɦɪɿʑʔʰ̩̃ᴀᴇ15678 '
62
+ '''
63
+
64
+ '''# chinese_dialect_cleaners
65
+ _pad = '_'
66
+ _punctuation = ',.!?~…─'
67
+ _letters = '#Nabdefghijklmnoprstuvwxyzæçøŋœȵɐɑɒɓɔɕɗɘəɚɛɜɣɤɦɪɭɯɵɷɸɻɾɿʂʅʊʋʌʏʑʔʦʮʰʷˀː˥˦˧˨˩̥̩̃̚αᴀᴇ↑↓∅ⱼ '
68
+ '''
69
+
70
+ # Export all symbols:
71
+ symbols = [_pad] + list(_punctuation) + list(_letters)
72
+
73
+ # Special symbol ids
74
+ SPACE_ID = symbols.index(" ")