skytnt commited on
Commit
e1ac136
1 Parent(s): 668c08a

update model

Browse files
README.md CHANGED
@@ -4,7 +4,7 @@ emoji: 😊🎙️
4
  colorFrom: red
5
  colorTo: pink
6
  sdk: gradio
7
- sdk_version: 3.3
8
  app_file: app.py
9
  pinned: false
10
  license: mit
 
4
  colorFrom: red
5
  colorTo: pink
6
  sdk: gradio
7
+ sdk_version: 3.3.1
8
  app_file: app.py
9
  pinned: false
10
  license: mit
requirements.txt CHANGED
@@ -12,6 +12,7 @@ Unidecode
12
  pyopenjtalk
13
  jamo
14
  pypinyin
 
15
  jieba
16
  cn2an
17
  gradio
 
12
  pyopenjtalk
13
  jamo
14
  pypinyin
15
+ ko_pron
16
  jieba
17
  cn2an
18
  gradio
saved_model/12/config.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d2d4fe332cfdbe95abc1152d977acdffebe88a28db6830f6c57e1cfb47a2799d
3
+ size 1397
saved_model/12/cover.jpg ADDED

Git LFS Details

  • SHA256: a4f93df7045805bcb028b92f464710e10961bae3ce43cddf2c289212673312e2
  • Pointer size: 130 Bytes
  • Size of remote file: 41 kB
saved_model/12/model.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cf8761f1f7818c961651d2c0d914821f742a9a1df8841aae376c888289ae5609
3
+ size 158888269
saved_model/6/model.pth CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b545a33fe870c214e3828da9ab8e756c6c75a30a6acee74670637fbbd3a58a0d
3
  size 158875981
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5461551d900d726e24fe5551c3773c0c27419c9237882fe7d400025344499f85
3
  size 158875981
saved_model/info.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:33b7a2042589990eb609c4e87044b7d5d6d80da206c88f54b70175ce0d2a535c
3
- size 1616
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:79bd8092d3351f000c280f51beee907fb0f090efc3bb4571e9ae10c6f7c3ec23
3
+ size 1733
text/__init__.py CHANGED
@@ -3,30 +3,30 @@ from text import cleaners
3
 
4
 
5
  def text_to_sequence(text, symbols, cleaner_names):
6
- '''Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
7
  Args:
8
  text: string to convert to a sequence
9
  cleaner_names: names of the cleaner functions to run the text through
10
  Returns:
11
  List of integers corresponding to the symbols in the text
12
  '''
13
- _symbol_to_id = {s: i for i, s in enumerate(symbols)}
14
 
15
- sequence = []
16
 
17
- clean_text = _clean_text(text, cleaner_names)
18
- for symbol in clean_text:
19
- if symbol not in _symbol_to_id.keys():
20
- continue
21
- symbol_id = _symbol_to_id[symbol]
22
- sequence += [symbol_id]
23
- return sequence
24
 
25
 
26
  def _clean_text(text, cleaner_names):
27
- for name in cleaner_names:
28
- cleaner = getattr(cleaners, name)
29
- if not cleaner:
30
- raise Exception('Unknown cleaner: %s' % name)
31
- text = cleaner(text)
32
- return text
 
3
 
4
 
5
  def text_to_sequence(text, symbols, cleaner_names):
6
+ '''Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
7
  Args:
8
  text: string to convert to a sequence
9
  cleaner_names: names of the cleaner functions to run the text through
10
  Returns:
11
  List of integers corresponding to the symbols in the text
12
  '''
13
+ _symbol_to_id = {s: i for i, s in enumerate(symbols)}
14
 
15
+ sequence = []
16
 
17
+ clean_text = _clean_text(text, cleaner_names)
18
+ for symbol in clean_text:
19
+ if symbol not in _symbol_to_id.keys():
20
+ continue
21
+ symbol_id = _symbol_to_id[symbol]
22
+ sequence += [symbol_id]
23
+ return sequence
24
 
25
 
26
  def _clean_text(text, cleaner_names):
27
+ for name in cleaner_names:
28
+ cleaner = getattr(cleaners, name)
29
+ if not cleaner:
30
+ raise Exception('Unknown cleaner: %s' % name)
31
+ text = cleaner(text)
32
+ return text
text/cleaners.py CHANGED
@@ -1,447 +1,10 @@
1
- """ from https://github.com/keithito/tacotron """
2
-
3
- '''
4
- Cleaners are transformations that run over the input text at both training and eval time.
5
- Cleaners can be selected by passing a comma-delimited list of cleaner names as the "cleaners"
6
- hyperparameter. Some cleaners are English-specific. You'll typically want to use:
7
- 1. "english_cleaners" for English text
8
- 2. "transliteration_cleaners" for non-English text that can be transliterated to ASCII using
9
- the Unidecode library (https://pypi.python.org/pypi/Unidecode)
10
- 3. "basic_cleaners" if you do not want to transliterate (in this case, you should also update
11
- the symbols in symbols.py to match your data).
12
- '''
13
-
14
- import os
15
  import re
16
- import sys
17
-
18
- import cn2an
19
- import jieba
20
- import pyopenjtalk
21
- from jamo import h2j, j2hcj
22
- from pypinyin import lazy_pinyin, BOPOMOFO
23
- from unidecode import unidecode
24
-
25
- jieba.initialize()
26
- pyopenjtalk._lazy_init()
27
-
28
- # This is a list of Korean classifiers preceded by pure Korean numerals.
29
- _korean_classifiers = '군데 권 개 그루 닢 대 두 마리 모 모금 뭇 발 발짝 방 번 벌 보루 살 수 술 시 쌈 움큼 정 짝 채 척 첩 축 켤레 톨 통'
30
-
31
- # Regular expression matching whitespace:
32
- _whitespace_re = re.compile(r'\s+')
33
-
34
- # Regular expression matching Japanese without punctuation marks:
35
- _japanese_characters = re.compile(
36
- r'[A-Za-z\d\u3005\u3040-\u30ff\u4e00-\u9fff\uff11-\uff19\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d]')
37
-
38
- # Regular expression matching non-Japanese characters or punctuation marks:
39
- _japanese_marks = re.compile(
40
- r'[^A-Za-z\d\u3005\u3040-\u30ff\u4e00-\u9fff\uff11-\uff19\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d]')
41
-
42
- # List of (regular expression, replacement) pairs for abbreviations:
43
- _abbreviations = [(re.compile('\\b%s\\.' % x[0], re.IGNORECASE), x[1]) for x in [
44
- ('mrs', 'misess'),
45
- ('mr', 'mister'),
46
- ('dr', 'doctor'),
47
- ('st', 'saint'),
48
- ('co', 'company'),
49
- ('jr', 'junior'),
50
- ('maj', 'major'),
51
- ('gen', 'general'),
52
- ('drs', 'doctors'),
53
- ('rev', 'reverend'),
54
- ('lt', 'lieutenant'),
55
- ('hon', 'honorable'),
56
- ('sgt', 'sergeant'),
57
- ('capt', 'captain'),
58
- ('esq', 'esquire'),
59
- ('ltd', 'limited'),
60
- ('col', 'colonel'),
61
- ('ft', 'fort'),
62
- ]]
63
-
64
- # List of (symbol, Japanese) pairs for marks:
65
- _symbols_to_japanese = [(re.compile('%s' % x[0], re.IGNORECASE), x[1]) for x in [
66
- ('%', 'パーセント')
67
- ]]
68
-
69
- # List of (hangul, hangul divided) pairs:
70
- _hangul_divided = [(re.compile('%s' % x[0]), x[1]) for x in [
71
- ('ㄳ', 'ㄱㅅ'),
72
- ('ㄵ', 'ㄴㅈ'),
73
- ('ㄶ', 'ㄴㅎ'),
74
- ('ㄺ', 'ㄹㄱ'),
75
- ('ㄻ', 'ㄹㅁ'),
76
- ('ㄼ', 'ㄹㅂ'),
77
- ('ㄽ', 'ㄹㅅ'),
78
- ('ㄾ', 'ㄹㅌ'),
79
- ('ㄿ', 'ㄹㅍ'),
80
- ('ㅀ', 'ㄹㅎ'),
81
- ('ㅄ', 'ㅂㅅ'),
82
- ('ㅘ', 'ㅗㅏ'),
83
- ('ㅙ', 'ㅗㅐ'),
84
- ('ㅚ', 'ㅗㅣ'),
85
- ('ㅝ', 'ㅜㅓ'),
86
- ('ㅞ', 'ㅜㅔ'),
87
- ('ㅟ', 'ㅜㅣ'),
88
- ('ㅢ', 'ㅡㅣ'),
89
- ('ㅑ', 'ㅣㅏ'),
90
- ('ㅒ', 'ㅣㅐ'),
91
- ('ㅕ', 'ㅣㅓ'),
92
- ('ㅖ', 'ㅣㅔ'),
93
- ('ㅛ', 'ㅣㅗ'),
94
- ('ㅠ', 'ㅣㅜ')
95
- ]]
96
-
97
- # List of (Latin alphabet, hangul) pairs:
98
- _latin_to_hangul = [(re.compile('%s' % x[0], re.IGNORECASE), x[1]) for x in [
99
- ('a', '에이'),
100
- ('b', '비'),
101
- ('c', '시'),
102
- ('d', '디'),
103
- ('e', '이'),
104
- ('f', '에프'),
105
- ('g', '지'),
106
- ('h', '에이치'),
107
- ('i', '아이'),
108
- ('j', '제이'),
109
- ('k', '케이'),
110
- ('l', '엘'),
111
- ('m', '엠'),
112
- ('n', '엔'),
113
- ('o', '오'),
114
- ('p', '피'),
115
- ('q', '큐'),
116
- ('r', '아르'),
117
- ('s', '에스'),
118
- ('t', '티'),
119
- ('u', '유'),
120
- ('v', '브이'),
121
- ('w', '더블유'),
122
- ('x', '엑스'),
123
- ('y', '와이'),
124
- ('z', '제트')
125
- ]]
126
-
127
- # List of (Latin alphabet, bopomofo) pairs:
128
- _latin_to_bopomofo = [(re.compile('%s' % x[0], re.IGNORECASE), x[1]) for x in [
129
- ('a', 'ㄟˉ'),
130
- ('b', 'ㄅㄧˋ'),
131
- ('c', 'ㄙㄧˉ'),
132
- ('d', 'ㄉㄧˋ'),
133
- ('e', 'ㄧˋ'),
134
- ('f', 'ㄝˊㄈㄨˋ'),
135
- ('g', 'ㄐㄧˋ'),
136
- ('h', 'ㄝˇㄑㄩˋ'),
137
- ('i', 'ㄞˋ'),
138
- ('j', 'ㄐㄟˋ'),
139
- ('k', 'ㄎㄟˋ'),
140
- ('l', 'ㄝˊㄛˋ'),
141
- ('m', 'ㄝˊㄇㄨˋ'),
142
- ('n', 'ㄣˉ'),
143
- ('o', 'ㄡˉ'),
144
- ('p', 'ㄆㄧˉ'),
145
- ('q', 'ㄎㄧㄡˉ'),
146
- ('r', 'ㄚˋ'),
147
- ('s', 'ㄝˊㄙˋ'),
148
- ('t', 'ㄊㄧˋ'),
149
- ('u', 'ㄧㄡˉ'),
150
- ('v', 'ㄨㄧˉ'),
151
- ('w', 'ㄉㄚˋㄅㄨˋㄌㄧㄡˋ'),
152
- ('x', 'ㄝˉㄎㄨˋㄙˋ'),
153
- ('y', 'ㄨㄞˋ'),
154
- ('z', 'ㄗㄟˋ')
155
- ]]
156
-
157
- # List of (bopomofo, romaji) pairs:
158
- _bopomofo_to_romaji = [(re.compile('%s' % x[0], re.IGNORECASE), x[1]) for x in [
159
- ('ㄅㄛ', 'p⁼wo'),
160
- ('ㄆㄛ', 'pʰwo'),
161
- ('ㄇㄛ', 'mwo'),
162
- ('ㄈㄛ', 'fwo'),
163
- ('ㄅ', 'p⁼'),
164
- ('ㄆ', 'pʰ'),
165
- ('ㄇ', 'm'),
166
- ('ㄈ', 'f'),
167
- ('ㄉ', 't⁼'),
168
- ('ㄊ', 'tʰ'),
169
- ('ㄋ', 'n'),
170
- ('ㄌ', 'l'),
171
- ('ㄍ', 'k⁼'),
172
- ('ㄎ', 'kʰ'),
173
- ('ㄏ', 'h'),
174
- ('ㄐ', 'ʧ⁼'),
175
- ('ㄑ', 'ʧʰ'),
176
- ('ㄒ', 'ʃ'),
177
- ('ㄓ', 'ʦ`⁼'),
178
- ('ㄔ', 'ʦ`ʰ'),
179
- ('ㄕ', 's`'),
180
- ('ㄖ', 'ɹ`'),
181
- ('ㄗ', 'ʦ⁼'),
182
- ('ㄘ', 'ʦʰ'),
183
- ('ㄙ', 's'),
184
- ('ㄚ', 'a'),
185
- ('ㄛ', 'o'),
186
- ('ㄜ', 'ə'),
187
- ('ㄝ', 'e'),
188
- ('ㄞ', 'ai'),
189
- ('ㄟ', 'ei'),
190
- ('ㄠ', 'au'),
191
- ('ㄡ', 'ou'),
192
- ('ㄧㄢ', 'yeNN'),
193
- ('ㄢ', 'aNN'),
194
- ('ㄧㄣ', 'iNN'),
195
- ('ㄣ', 'əNN'),
196
- ('ㄤ', 'aNg'),
197
- ('ㄧㄥ', 'iNg'),
198
- ('ㄨㄥ', 'uNg'),
199
- ('ㄩㄥ', 'yuNg'),
200
- ('ㄥ', 'əNg'),
201
- ('ㄦ', 'əɻ'),
202
- ('ㄧ', 'i'),
203
- ('ㄨ', 'u'),
204
- ('ㄩ', 'ɥ'),
205
- ('ˉ', '→'),
206
- ('ˊ', '↑'),
207
- ('ˇ', '↓↑'),
208
- ('ˋ', '↓'),
209
- ('˙', ''),
210
- (',', ','),
211
- ('。', '.'),
212
- ('!', '!'),
213
- ('?', '?'),
214
- ('—', '-')
215
- ]]
216
-
217
-
218
- def expand_abbreviations(text):
219
- for regex, replacement in _abbreviations:
220
- text = re.sub(regex, replacement, text)
221
- return text
222
-
223
-
224
- def lowercase(text):
225
- return text.lower()
226
-
227
-
228
- def collapse_whitespace(text):
229
- return re.sub(_whitespace_re, ' ', text)
230
-
231
-
232
- def convert_to_ascii(text):
233
- return unidecode(text)
234
-
235
-
236
- def symbols_to_japanese(text):
237
- for regex, replacement in _symbols_to_japanese:
238
- text = re.sub(regex, replacement, text)
239
- return text
240
-
241
-
242
- def japanese_to_romaji_with_accent(text):
243
- '''Reference https://r9y9.github.io/ttslearn/latest/notebooks/ch10_Recipe-Tacotron.html'''
244
- text = symbols_to_japanese(text)
245
- sentences = re.split(_japanese_marks, text)
246
- marks = re.findall(_japanese_marks, text)
247
- text = ''
248
- for i, sentence in enumerate(sentences):
249
- if re.match(_japanese_characters, sentence):
250
- if text != '':
251
- text += ' '
252
- labels = pyopenjtalk.extract_fullcontext(sentence)
253
- for n, label in enumerate(labels):
254
- phoneme = re.search(r'\-([^\+]*)\+', label).group(1)
255
- if phoneme not in ['sil', 'pau']:
256
- text += phoneme.replace('ch', 'ʧ').replace('sh', 'ʃ').replace('cl', 'Q')
257
- else:
258
- continue
259
- n_moras = int(re.search(r'/F:(\d+)_', label).group(1))
260
- a1 = int(re.search(r"/A:(\-?[0-9]+)\+", label).group(1))
261
- a2 = int(re.search(r"\+(\d+)\+", label).group(1))
262
- a3 = int(re.search(r"\+(\d+)/", label).group(1))
263
- if re.search(r'\-([^\+]*)\+', labels[n + 1]).group(1) in ['sil', 'pau']:
264
- a2_next = -1
265
- else:
266
- a2_next = int(re.search(r"\+(\d+)\+", labels[n + 1]).group(1))
267
- # Accent phrase boundary
268
- if a3 == 1 and a2_next == 1:
269
- text += ' '
270
- # Falling
271
- elif a1 == 0 and a2_next == a2 + 1 and a2 != n_moras:
272
- text += '↓'
273
- # Rising
274
- elif a2 == 1 and a2_next == 2:
275
- text += '↑'
276
- if i < len(marks):
277
- text += unidecode(marks[i]).replace(' ', '')
278
- return text
279
-
280
-
281
- def latin_to_hangul(text):
282
- for regex, replacement in _latin_to_hangul:
283
- text = re.sub(regex, replacement, text)
284
- return text
285
-
286
-
287
- def divide_hangul(text):
288
- for regex, replacement in _hangul_divided:
289
- text = re.sub(regex, replacement, text)
290
- return text
291
-
292
-
293
- def hangul_number(num, sino=True):
294
- '''Reference https://github.com/Kyubyong/g2pK'''
295
- num = re.sub(',', '', num)
296
-
297
- if num == '0':
298
- return '영'
299
- if not sino and num == '20':
300
- return '스무'
301
-
302
- digits = '123456789'
303
- names = '일이삼사오육칠팔구'
304
- digit2name = {d: n for d, n in zip(digits, names)}
305
-
306
- modifiers = '한 두 세 네 다섯 여섯 일곱 여덟 아홉'
307
- decimals = '열 스물 서른 마흔 쉰 예순 일흔 여든 아흔'
308
- digit2mod = {d: mod for d, mod in zip(digits, modifiers.split())}
309
- digit2dec = {d: dec for d, dec in zip(digits, decimals.split())}
310
-
311
- spelledout = []
312
- for i, digit in enumerate(num):
313
- i = len(num) - i - 1
314
- if sino:
315
- if i == 0:
316
- name = digit2name.get(digit, '')
317
- elif i == 1:
318
- name = digit2name.get(digit, '') + '십'
319
- name = name.replace('일십', '십')
320
- else:
321
- if i == 0:
322
- name = digit2mod.get(digit, '')
323
- elif i == 1:
324
- name = digit2dec.get(digit, '')
325
- if digit == '0':
326
- if i % 4 == 0:
327
- last_three = spelledout[-min(3, len(spelledout)):]
328
- if ''.join(last_three) == '':
329
- spelledout.append('')
330
- continue
331
- else:
332
- spelledout.append('')
333
- continue
334
- if i == 2:
335
- name = digit2name.get(digit, '') + '백'
336
- name = name.replace('일백', '백')
337
- elif i == 3:
338
- name = digit2name.get(digit, '') + '천'
339
- name = name.replace('일천', '천')
340
- elif i == 4:
341
- name = digit2name.get(digit, '') + '만'
342
- name = name.replace('일만', '만')
343
- elif i == 5:
344
- name = digit2name.get(digit, '') + '십'
345
- name = name.replace('일십', '십')
346
- elif i == 6:
347
- name = digit2name.get(digit, '') + '백'
348
- name = name.replace('일백', '백')
349
- elif i == 7:
350
- name = digit2name.get(digit, '') + '천'
351
- name = name.replace('일천', '천')
352
- elif i == 8:
353
- name = digit2name.get(digit, '') + '억'
354
- elif i == 9:
355
- name = digit2name.get(digit, '') + '십'
356
- elif i == 10:
357
- name = digit2name.get(digit, '') + '백'
358
- elif i == 11:
359
- name = digit2name.get(digit, '') + '천'
360
- elif i == 12:
361
- name = digit2name.get(digit, '') + '조'
362
- elif i == 13:
363
- name = digit2name.get(digit, '') + '십'
364
- elif i == 14:
365
- name = digit2name.get(digit, '') + '백'
366
- elif i == 15:
367
- name = digit2name.get(digit, '') + '천'
368
- spelledout.append(name)
369
- return ''.join(elem for elem in spelledout)
370
-
371
-
372
- def number_to_hangul(text):
373
- '''Reference https://github.com/Kyubyong/g2pK'''
374
- tokens = set(re.findall(r'(\d[\d,]*)([\uac00-\ud71f]+)', text))
375
- for token in tokens:
376
- num, classifier = token
377
- if classifier[:2] in _korean_classifiers or classifier[0] in _korean_classifiers:
378
- spelledout = hangul_number(num, sino=False)
379
- else:
380
- spelledout = hangul_number(num, sino=True)
381
- text = text.replace(f'{num}{classifier}', f'{spelledout}{classifier}')
382
- # digit by digit for remaining digits
383
- digits = '0123456789'
384
- names = '영일이삼사오육칠팔구'
385
- for d, n in zip(digits, names):
386
- text = text.replace(d, n)
387
- return text
388
-
389
-
390
- def number_to_chinese(text):
391
- numbers = re.findall(r'\d+(?:\.?\d+)?', text)
392
- for number in numbers:
393
- text = text.replace(number, cn2an.an2cn(number), 1)
394
- return text
395
-
396
-
397
- def chinese_to_bopomofo(text):
398
- text = text.replace('、', ',').replace(';', ',').replace(':', ',')
399
- words = jieba.lcut(text, cut_all=False)
400
- text = ''
401
- for word in words:
402
- bopomofos = lazy_pinyin(word, BOPOMOFO)
403
- if not re.search('[\u4e00-\u9fff]', word):
404
- text += word
405
- continue
406
- for i in range(len(bopomofos)):
407
- if re.match('[\u3105-\u3129]', bopomofos[i][-1]):
408
- bopomofos[i] += 'ˉ'
409
- if text != '':
410
- text += ' '
411
- text += ''.join(bopomofos)
412
- return text
413
-
414
-
415
- def latin_to_bopomofo(text):
416
- for regex, replacement in _latin_to_bopomofo:
417
- text = re.sub(regex, replacement, text)
418
- return text
419
-
420
-
421
- def bopomofo_to_romaji(text):
422
- for regex, replacement in _bopomofo_to_romaji:
423
- text = re.sub(regex, replacement, text)
424
- return text
425
-
426
-
427
- def basic_cleaners(text):
428
- '''Basic pipeline that lowercases and collapses whitespace without transliteration.'''
429
- text = lowercase(text)
430
- text = collapse_whitespace(text)
431
- return text
432
-
433
-
434
- def transliteration_cleaners(text):
435
- '''Pipeline for non-English text that transliterates to ASCII.'''
436
- text = convert_to_ascii(text)
437
- text = lowercase(text)
438
- text = collapse_whitespace(text)
439
- return text
440
 
441
 
442
  def japanese_cleaners(text):
 
443
  text = japanese_to_romaji_with_accent(text)
444
- if len(text) > 0 and re.match('[A-Za-z]', text[-1]):
445
  text += '.'
446
  return text
447
 
@@ -452,44 +15,73 @@ def japanese_cleaners2(text):
452
 
453
  def korean_cleaners(text):
454
  '''Pipeline for Korean text'''
 
455
  text = latin_to_hangul(text)
456
  text = number_to_hangul(text)
457
- text = j2hcj(h2j(text))
458
  text = divide_hangul(text)
459
- if len(text) > 0 and re.match('[\u3131-\u3163]', text[-1]):
460
  text += '.'
461
  return text
462
 
463
 
464
  def chinese_cleaners(text):
465
  '''Pipeline for Chinese text'''
 
466
  text = number_to_chinese(text)
467
  text = chinese_to_bopomofo(text)
468
  text = latin_to_bopomofo(text)
469
- if len(text) > 0 and re.match('[ˉˊˇˋ˙]', text[-1]):
470
  text += '。'
471
  return text
472
 
473
 
474
  def zh_ja_mixture_cleaners(text):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
475
  chinese_texts = re.findall(r'\[ZH\].*?\[ZH\]', text)
476
  japanese_texts = re.findall(r'\[JA\].*?\[JA\]', text)
 
 
477
  for chinese_text in chinese_texts:
478
- cleaned_text = number_to_chinese(chinese_text[4:-4])
479
- cleaned_text = chinese_to_bopomofo(cleaned_text)
480
- cleaned_text = latin_to_bopomofo(cleaned_text)
481
- cleaned_text = bopomofo_to_romaji(cleaned_text)
482
- cleaned_text = re.sub('i[aoe]', lambda x: 'y' + x.group(0)[1:], cleaned_text)
483
- cleaned_text = re.sub('u[aoəe]', lambda x: 'w' + x.group(0)[1:], cleaned_text)
484
- cleaned_text = re.sub('([ʦsɹ]`[⁼ʰ]?)([→↓↑]+)', lambda x: x.group(1) + 'ɹ`' + x.group(2), cleaned_text).replace(
485
- 'ɻ', 'ɹ`')
486
- cleaned_text = re.sub('([ʦs][⁼ʰ]?)([→↓↑]+)', lambda x: x.group(1) + 'ɹ' + x.group(2), cleaned_text)
487
  text = text.replace(chinese_text, cleaned_text + ' ', 1)
488
  for japanese_text in japanese_texts:
489
- cleaned_text = japanese_to_romaji_with_accent(japanese_text[4:-4]).replace('ts', 'ʦ').replace('u', 'ɯ').replace(
490
- '...', '…')
491
  text = text.replace(japanese_text, cleaned_text + ' ', 1)
 
 
 
 
 
 
492
  text = text[:-1]
493
- if len(text) > 0 and re.match('[A-Za-zɯɹəɥ→↓↑]', text[-1]):
494
  text += '.'
495
  return text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import re
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
 
3
 
4
  def japanese_cleaners(text):
5
+ from text.japanese import japanese_to_romaji_with_accent
6
  text = japanese_to_romaji_with_accent(text)
7
+ if len(text) == 0 or re.match('[A-Za-z]', text[-1]):
8
  text += '.'
9
  return text
10
 
 
15
 
16
  def korean_cleaners(text):
17
  '''Pipeline for Korean text'''
18
+ from text.korean import latin_to_hangul, number_to_hangul, divide_hangul
19
  text = latin_to_hangul(text)
20
  text = number_to_hangul(text)
 
21
  text = divide_hangul(text)
22
+ if len(text) == 0 or re.match('[\u3131-\u3163]', text[-1]):
23
  text += '.'
24
  return text
25
 
26
 
27
  def chinese_cleaners(text):
28
  '''Pipeline for Chinese text'''
29
+ from text.mandarin import number_to_chinese, chinese_to_bopomofo, latin_to_bopomofo
30
  text = number_to_chinese(text)
31
  text = chinese_to_bopomofo(text)
32
  text = latin_to_bopomofo(text)
33
+ if len(text) == 0 or re.match('[ˉˊˇˋ˙]', text[-1]):
34
  text += '。'
35
  return text
36
 
37
 
38
  def zh_ja_mixture_cleaners(text):
39
+ from text.mandarin import chinese_to_romaji
40
+ from text.japanese import japanese_to_romaji_with_accent
41
+ chinese_texts = re.findall(r'\[ZH\].*?\[ZH\]', text)
42
+ japanese_texts = re.findall(r'\[JA\].*?\[JA\]', text)
43
+ for chinese_text in chinese_texts:
44
+ cleaned_text = chinese_to_romaji(chinese_text[4:-4])
45
+ text = text.replace(chinese_text, cleaned_text + ' ', 1)
46
+ for japanese_text in japanese_texts:
47
+ cleaned_text = japanese_to_romaji_with_accent(
48
+ japanese_text[4:-4]).replace('ts', 'ʦ').replace('u', 'ɯ').replace('...', '…')
49
+ text = text.replace(japanese_text, cleaned_text + ' ', 1)
50
+ text = text[:-1]
51
+ if len(text) == 0 or re.match('[A-Za-zɯɹəɥ→↓↑]', text[-1]):
52
+ text += '.'
53
+ return text
54
+
55
+
56
+ def sanskrit_cleaners(text):
57
+ text = text.replace('॥', '।').replace('ॐ', 'ओम्')
58
+ if len(text) == 0 or text[-1] != '।':
59
+ text += ' ।'
60
+ return text
61
+
62
+
63
+ def cjks_cleaners(text):
64
+ from text.mandarin import chinese_to_lazy_ipa
65
+ from text.japanese import japanese_to_ipa
66
+ from text.korean import korean_to_lazy_ipa
67
+ from text.sanskrit import devanagari_to_ipa
68
  chinese_texts = re.findall(r'\[ZH\].*?\[ZH\]', text)
69
  japanese_texts = re.findall(r'\[JA\].*?\[JA\]', text)
70
+ korean_texts = re.findall(r'\[KO\].*?\[KO\]', text)
71
+ sanskrit_texts = re.findall(r'\[SA\].*?\[SA\]', text)
72
  for chinese_text in chinese_texts:
73
+ cleaned_text = chinese_to_lazy_ipa(chinese_text[4:-4])
 
 
 
 
 
 
 
 
74
  text = text.replace(chinese_text, cleaned_text + ' ', 1)
75
  for japanese_text in japanese_texts:
76
+ cleaned_text = japanese_to_ipa(japanese_text[4:-4])
 
77
  text = text.replace(japanese_text, cleaned_text + ' ', 1)
78
+ for korean_text in korean_texts:
79
+ cleaned_text = korean_to_lazy_ipa(korean_text[4:-4])
80
+ text = text.replace(korean_text, cleaned_text + ' ', 1)
81
+ for sanskrit_text in sanskrit_texts:
82
+ cleaned_text = devanagari_to_ipa(sanskrit_text[4:-4])
83
+ text = text.replace(sanskrit_text, cleaned_text + ' ', 1)
84
  text = text[:-1]
85
+ if len(text) == 0 or re.match(r'[^\.,!\?\-…~]', text[-1]):
86
  text += '.'
87
  return text
text/japanese.py ADDED
@@ -0,0 +1,132 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+ from unidecode import unidecode
3
+ import pyopenjtalk
4
+
5
+
6
+ # Regular expression matching Japanese without punctuation marks:
7
+ _japanese_characters = re.compile(
8
+ r'[A-Za-z\d\u3005\u3040-\u30ff\u4e00-\u9fff\uff11-\uff19\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d]')
9
+
10
+ # Regular expression matching non-Japanese characters or punctuation marks:
11
+ _japanese_marks = re.compile(
12
+ r'[^A-Za-z\d\u3005\u3040-\u30ff\u4e00-\u9fff\uff11-\uff19\uff21-\uff3a\uff41-\uff5a\uff66-\uff9d]')
13
+
14
+ # List of (symbol, Japanese) pairs for marks:
15
+ _symbols_to_japanese = [(re.compile('%s' % x[0]), x[1]) for x in [
16
+ ('%', 'パーセント')
17
+ ]]
18
+
19
+ # List of (romaji, ipa) pairs for marks:
20
+ _romaji_to_ipa = [(re.compile('%s' % x[0], re.IGNORECASE), x[1]) for x in [
21
+ ('ts', 'ʦ'),
22
+ ('u', 'ɯ'),
23
+ ('...', '…'),
24
+ ('j', 'ʥ'),
25
+ ('y', 'j'),
26
+ ('ni', 'n^i'),
27
+ ('nj', 'n^'),
28
+ ('hi', 'çi'),
29
+ ('hj', 'ç'),
30
+ ('f', 'ɸ'),
31
+ ('I', 'i*'),
32
+ ('U', 'ɯ*'),
33
+ ('r', 'ɾ')
34
+ ]]
35
+
36
+ # Dictinary of (consonant, sokuon) pairs:
37
+ _real_sokuon = {
38
+ 'k': 'k#',
39
+ 'g': 'k#',
40
+ 't': 't#',
41
+ 'd': 't#',
42
+ 'ʦ': 't#',
43
+ 'ʧ': 't#',
44
+ 'ʥ': 't#',
45
+ 'j': 't#',
46
+ 's': 's',
47
+ 'ʃ': 's',
48
+ 'p': 'p#',
49
+ 'b': 'p#'
50
+ }
51
+
52
+ # Dictinary of (consonant, hatsuon) pairs:
53
+ _real_hatsuon = {
54
+ 'p': 'm',
55
+ 'b': 'm',
56
+ 'm': 'm',
57
+ 't': 'n',
58
+ 'd': 'n',
59
+ 'n': 'n',
60
+ 'ʧ': 'n^',
61
+ 'ʥ': 'n^',
62
+ 'k': 'ŋ',
63
+ 'g': 'ŋ'
64
+ }
65
+
66
+
67
+ def symbols_to_japanese(text):
68
+ for regex, replacement in _symbols_to_japanese:
69
+ text = re.sub(regex, replacement, text)
70
+ return text
71
+
72
+
73
+ def japanese_to_romaji_with_accent(text):
74
+ '''Reference https://r9y9.github.io/ttslearn/latest/notebooks/ch10_Recipe-Tacotron.html'''
75
+ text = symbols_to_japanese(text)
76
+ sentences = re.split(_japanese_marks, text)
77
+ marks = re.findall(_japanese_marks, text)
78
+ text = ''
79
+ for i, sentence in enumerate(sentences):
80
+ if re.match(_japanese_characters, sentence):
81
+ if text != '':
82
+ text += ' '
83
+ labels = pyopenjtalk.extract_fullcontext(sentence)
84
+ for n, label in enumerate(labels):
85
+ phoneme = re.search(r'\-([^\+]*)\+', label).group(1)
86
+ if phoneme not in ['sil', 'pau']:
87
+ text += phoneme.replace('ch', 'ʧ').replace('sh',
88
+ 'ʃ').replace('cl', 'Q')
89
+ else:
90
+ continue
91
+ # n_moras = int(re.search(r'/F:(\d+)_', label).group(1))
92
+ a1 = int(re.search(r"/A:(\-?[0-9]+)\+", label).group(1))
93
+ a2 = int(re.search(r"\+(\d+)\+", label).group(1))
94
+ a3 = int(re.search(r"\+(\d+)/", label).group(1))
95
+ if re.search(r'\-([^\+]*)\+', labels[n + 1]).group(1) in ['sil', 'pau']:
96
+ a2_next = -1
97
+ else:
98
+ a2_next = int(
99
+ re.search(r"\+(\d+)\+", labels[n + 1]).group(1))
100
+ # Accent phrase boundary
101
+ if a3 == 1 and a2_next == 1:
102
+ text += ' '
103
+ # Falling
104
+ elif a1 == 0 and a2_next == a2 + 1:
105
+ text += '↓'
106
+ # Rising
107
+ elif a2 == 1 and a2_next == 2:
108
+ text += '↑'
109
+ if i < len(marks):
110
+ text += unidecode(marks[i]).replace(' ', '')
111
+ return text
112
+
113
+
114
+ def get_real_sokuon(text):
115
+ text=re.sub('Q[↑↓]*(.)',lambda x:_real_sokuon[x.group(1)]+x.group(0)[1:] if x.group(1) in _real_sokuon.keys() else x.group(0),text)
116
+ return text
117
+
118
+
119
+ def get_real_hatsuon(text):
120
+ text=re.sub('N[↑↓]*(.)',lambda x:_real_hatsuon[x.group(1)]+x.group(0)[1:] if x.group(1) in _real_hatsuon.keys() else x.group(0),text)
121
+ return text
122
+
123
+
124
+ def japanese_to_ipa(text):
125
+ text=japanese_to_romaji_with_accent(text)
126
+ for regex, replacement in _romaji_to_ipa:
127
+ text = re.sub(regex, replacement, text)
128
+ text = re.sub(
129
+ r'([A-Za-zɯ])\1+', lambda x: x.group(0)[0]+'ː'*(len(x.group(0))-1), text)
130
+ text = get_real_sokuon(text)
131
+ text = get_real_hatsuon(text)
132
+ return text
text/korean.py ADDED
@@ -0,0 +1,205 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+ from jamo import h2j, j2hcj
3
+ import ko_pron
4
+
5
+
6
+ # This is a list of Korean classifiers preceded by pure Korean numerals.
7
+ _korean_classifiers = '군데 권 개 그루 닢 대 두 마리 모 모금 뭇 발 발짝 방 번 벌 보루 살 수 술 시 쌈 움큼 정 짝 채 척 첩 축 켤레 톨 통'
8
+
9
+ # List of (hangul, hangul divided) pairs:
10
+ _hangul_divided = [(re.compile('%s' % x[0]), x[1]) for x in [
11
+ ('ㄳ', 'ㄱㅅ'),
12
+ ('ㄵ', 'ㄴㅈ'),
13
+ ('ㄶ', 'ㄴㅎ'),
14
+ ('ㄺ', 'ㄹㄱ'),
15
+ ('ㄻ', 'ㄹㅁ'),
16
+ ('ㄼ', 'ㄹㅂ'),
17
+ ('ㄽ', 'ㄹㅅ'),
18
+ ('ㄾ', 'ㄹㅌ'),
19
+ ('ㄿ', 'ㄹㅍ'),
20
+ ('ㅀ', 'ㄹㅎ'),
21
+ ('ㅄ', 'ㅂㅅ'),
22
+ ('ㅘ', 'ㅗㅏ'),
23
+ ('ㅙ', 'ㅗㅐ'),
24
+ ('ㅚ', 'ㅗㅣ'),
25
+ ('ㅝ', 'ㅜㅓ'),
26
+ ('ㅞ', 'ㅜㅔ'),
27
+ ('ㅟ', 'ㅜㅣ'),
28
+ ('ㅢ', 'ㅡㅣ'),
29
+ ('ㅑ', 'ㅣㅏ'),
30
+ ('ㅒ', 'ㅣㅐ'),
31
+ ('ㅕ', 'ㅣㅓ'),
32
+ ('ㅖ', 'ㅣㅔ'),
33
+ ('ㅛ', 'ㅣㅗ'),
34
+ ('ㅠ', 'ㅣㅜ')
35
+ ]]
36
+
37
+ # List of (Latin alphabet, hangul) pairs:
38
+ _latin_to_hangul = [(re.compile('%s' % x[0], re.IGNORECASE), x[1]) for x in [
39
+ ('a', '에이'),
40
+ ('b', '비'),
41
+ ('c', '시'),
42
+ ('d', '디'),
43
+ ('e', '이'),
44
+ ('f', '에프'),
45
+ ('g', '지'),
46
+ ('h', '에이치'),
47
+ ('i', '아이'),
48
+ ('j', '제이'),
49
+ ('k', '케이'),
50
+ ('l', '엘'),
51
+ ('m', '엠'),
52
+ ('n', '엔'),
53
+ ('o', '오'),
54
+ ('p', '피'),
55
+ ('q', '큐'),
56
+ ('r', '아르'),
57
+ ('s', '에스'),
58
+ ('t', '티'),
59
+ ('u', '유'),
60
+ ('v', '브이'),
61
+ ('w', '더블유'),
62
+ ('x', '엑스'),
63
+ ('y', '와이'),
64
+ ('z', '제트')
65
+ ]]
66
+
67
+ # List of (ipa, lazy ipa) pairs:
68
+ _ipa_to_lazy_ipa = [(re.compile('%s' % x[0], re.IGNORECASE), x[1]) for x in [
69
+ ('t͡ɕ','ʧ'),
70
+ ('d͡ʑ','ʥ'),
71
+ ('ɲ','n^'),
72
+ ('ɕ','ʃ'),
73
+ ('ʷ','w'),
74
+ ('ɭ','l`'),
75
+ ('ʎ','ɾ'),
76
+ ('ɣ','ŋ'),
77
+ ('ɰ','ɯ'),
78
+ ('ʝ','j'),
79
+ ('ʌ','ə'),
80
+ ('ɡ','g'),
81
+ ('\u031a','#'),
82
+ ('\u0348','='),
83
+ ('\u031e',''),
84
+ ('\u0320',''),
85
+ ('\u0339','')
86
+ ]]
87
+
88
+
89
+ def latin_to_hangul(text):
90
+ for regex, replacement in _latin_to_hangul:
91
+ text = re.sub(regex, replacement, text)
92
+ return text
93
+
94
+
95
+ def divide_hangul(text):
96
+ text = j2hcj(h2j(text))
97
+ for regex, replacement in _hangul_divided:
98
+ text = re.sub(regex, replacement, text)
99
+ return text
100
+
101
+
102
+ def hangul_number(num, sino=True):
103
+ '''Reference https://github.com/Kyubyong/g2pK'''
104
+ num = re.sub(',', '', num)
105
+
106
+ if num == '0':
107
+ return '영'
108
+ if not sino and num == '20':
109
+ return '스무'
110
+
111
+ digits = '123456789'
112
+ names = '일이삼사오육칠팔구'
113
+ digit2name = {d: n for d, n in zip(digits, names)}
114
+
115
+ modifiers = '한 두 세 네 다섯 여섯 일곱 여덟 아홉'
116
+ decimals = '열 스물 서른 마흔 쉰 예순 일흔 여든 아흔'
117
+ digit2mod = {d: mod for d, mod in zip(digits, modifiers.split())}
118
+ digit2dec = {d: dec for d, dec in zip(digits, decimals.split())}
119
+
120
+ spelledout = []
121
+ for i, digit in enumerate(num):
122
+ i = len(num) - i - 1
123
+ if sino:
124
+ if i == 0:
125
+ name = digit2name.get(digit, '')
126
+ elif i == 1:
127
+ name = digit2name.get(digit, '') + '십'
128
+ name = name.replace('일십', '십')
129
+ else:
130
+ if i == 0:
131
+ name = digit2mod.get(digit, '')
132
+ elif i == 1:
133
+ name = digit2dec.get(digit, '')
134
+ if digit == '0':
135
+ if i % 4 == 0:
136
+ last_three = spelledout[-min(3, len(spelledout)):]
137
+ if ''.join(last_three) == '':
138
+ spelledout.append('')
139
+ continue
140
+ else:
141
+ spelledout.append('')
142
+ continue
143
+ if i == 2:
144
+ name = digit2name.get(digit, '') + '백'
145
+ name = name.replace('일백', '백')
146
+ elif i == 3:
147
+ name = digit2name.get(digit, '') + '천'
148
+ name = name.replace('일천', '천')
149
+ elif i == 4:
150
+ name = digit2name.get(digit, '') + '만'
151
+ name = name.replace('일만', '만')
152
+ elif i == 5:
153
+ name = digit2name.get(digit, '') + '십'
154
+ name = name.replace('일십', '십')
155
+ elif i == 6:
156
+ name = digit2name.get(digit, '') + '백'
157
+ name = name.replace('일백', '백')
158
+ elif i == 7:
159
+ name = digit2name.get(digit, '') + '천'
160
+ name = name.replace('일천', '천')
161
+ elif i == 8:
162
+ name = digit2name.get(digit, '') + '억'
163
+ elif i == 9:
164
+ name = digit2name.get(digit, '') + '십'
165
+ elif i == 10:
166
+ name = digit2name.get(digit, '') + '백'
167
+ elif i == 11:
168
+ name = digit2name.get(digit, '') + '천'
169
+ elif i == 12:
170
+ name = digit2name.get(digit, '') + '조'
171
+ elif i == 13:
172
+ name = digit2name.get(digit, '') + '십'
173
+ elif i == 14:
174
+ name = digit2name.get(digit, '') + '백'
175
+ elif i == 15:
176
+ name = digit2name.get(digit, '') + '천'
177
+ spelledout.append(name)
178
+ return ''.join(elem for elem in spelledout)
179
+
180
+
181
+ def number_to_hangul(text):
182
+ '''Reference https://github.com/Kyubyong/g2pK'''
183
+ tokens = set(re.findall(r'(\d[\d,]*)([\uac00-\ud71f]+)', text))
184
+ for token in tokens:
185
+ num, classifier = token
186
+ if classifier[:2] in _korean_classifiers or classifier[0] in _korean_classifiers:
187
+ spelledout = hangul_number(num, sino=False)
188
+ else:
189
+ spelledout = hangul_number(num, sino=True)
190
+ text = text.replace(f'{num}{classifier}', f'{spelledout}{classifier}')
191
+ # digit by digit for remaining digits
192
+ digits = '0123456789'
193
+ names = '영일이삼사오육칠팔구'
194
+ for d, n in zip(digits, names):
195
+ text = text.replace(d, n)
196
+ return text
197
+
198
+
199
+ def korean_to_lazy_ipa(text):
200
+ text = latin_to_hangul(text)
201
+ text = number_to_hangul(text)
202
+ text=re.sub('[\uac00-\ud7af]+',lambda x:ko_pron.romanise(x.group(0),'ipa'),text).split('] ~ [')[0]
203
+ for regex, replacement in _ipa_to_lazy_ipa:
204
+ text = re.sub(regex, replacement, text)
205
+ return text
text/mandarin.py ADDED
@@ -0,0 +1,171 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import sys
3
+ import re
4
+ from pypinyin import lazy_pinyin, BOPOMOFO
5
+ import jieba
6
+ import cn2an
7
+ import logging
8
+
9
+ logging.getLogger('jieba').setLevel(logging.WARNING)
10
+ jieba.initialize()
11
+
12
+
13
+ # List of (Latin alphabet, bopomofo) pairs:
14
+ _latin_to_bopomofo = [(re.compile('%s' % x[0], re.IGNORECASE), x[1]) for x in [
15
+ ('a', 'ㄟˉ'),
16
+ ('b', 'ㄅㄧˋ'),
17
+ ('c', 'ㄙㄧˉ'),
18
+ ('d', 'ㄉㄧˋ'),
19
+ ('e', 'ㄧˋ'),
20
+ ('f', 'ㄝˊㄈㄨˋ'),
21
+ ('g', 'ㄐㄧˋ'),
22
+ ('h', 'ㄝˇㄑㄩˋ'),
23
+ ('i', 'ㄞˋ'),
24
+ ('j', 'ㄐㄟˋ'),
25
+ ('k', 'ㄎㄟˋ'),
26
+ ('l', 'ㄝˊㄛˋ'),
27
+ ('m', 'ㄝˊㄇㄨˋ'),
28
+ ('n', 'ㄣˉ'),
29
+ ('o', 'ㄡˉ'),
30
+ ('p', 'ㄆㄧˉ'),
31
+ ('q', 'ㄎㄧㄡˉ'),
32
+ ('r', 'ㄚˋ'),
33
+ ('s', 'ㄝˊㄙˋ'),
34
+ ('t', 'ㄊㄧˋ'),
35
+ ('u', 'ㄧㄡˉ'),
36
+ ('v', 'ㄨㄧˉ'),
37
+ ('w', 'ㄉㄚˋㄅㄨˋㄌㄧㄡˋ'),
38
+ ('x', 'ㄝˉㄎㄨˋㄙˋ'),
39
+ ('y', 'ㄨㄞˋ'),
40
+ ('z', 'ㄗㄟˋ')
41
+ ]]
42
+
43
+ # List of (bopomofo, romaji) pairs:
44
+ _bopomofo_to_romaji = [(re.compile('%s' % x[0]), x[1]) for x in [
45
+ ('ㄅㄛ', 'p⁼wo'),
46
+ ('ㄆㄛ', 'pʰwo'),
47
+ ('ㄇㄛ', 'mwo'),
48
+ ('ㄈㄛ', 'fwo'),
49
+ ('ㄅ', 'p⁼'),
50
+ ('ㄆ', 'pʰ'),
51
+ ('ㄇ', 'm'),
52
+ ('ㄈ', 'f'),
53
+ ('ㄉ', 't⁼'),
54
+ ('ㄊ', 'tʰ'),
55
+ ('ㄋ', 'n'),
56
+ ('ㄌ', 'l'),
57
+ ('ㄍ', 'k⁼'),
58
+ ('ㄎ', 'kʰ'),
59
+ ('ㄏ', 'h'),
60
+ ('ㄐ', 'ʧ⁼'),
61
+ ('ㄑ', 'ʧʰ'),
62
+ ('ㄒ', 'ʃ'),
63
+ ('ㄓ', 'ʦ`⁼'),
64
+ ('ㄔ', 'ʦ`ʰ'),
65
+ ('ㄕ', 's`'),
66
+ ('ㄖ', 'ɹ`'),
67
+ ('ㄗ', 'ʦ⁼'),
68
+ ('ㄘ', 'ʦʰ'),
69
+ ('ㄙ', 's'),
70
+ ('ㄚ', 'a'),
71
+ ('ㄛ', 'o'),
72
+ ('ㄜ', 'ə'),
73
+ ('ㄝ', 'e'),
74
+ ('ㄞ', 'ai'),
75
+ ('ㄟ', 'ei'),
76
+ ('ㄠ', 'au'),
77
+ ('ㄡ', 'ou'),
78
+ ('ㄧㄢ', 'yeNN'),
79
+ ('ㄢ', 'aNN'),
80
+ ('ㄧㄣ', 'iNN'),
81
+ ('ㄣ', 'əNN'),
82
+ ('ㄤ', 'aNg'),
83
+ ('ㄧㄥ', 'iNg'),
84
+ ('ㄨㄥ', 'uNg'),
85
+ ('ㄩㄥ', 'yuNg'),
86
+ ('ㄥ', 'əNg'),
87
+ ('ㄦ', 'əɻ'),
88
+ ('ㄧ', 'i'),
89
+ ('ㄨ', 'u'),
90
+ ('ㄩ', 'ɥ'),
91
+ ('ˉ', '→'),
92
+ ('ˊ', '↑'),
93
+ ('ˇ', '↓↑'),
94
+ ('ˋ', '↓'),
95
+ ('˙', ''),
96
+ (',', ','),
97
+ ('。', '.'),
98
+ ('!', '!'),
99
+ ('?', '?'),
100
+ ('—', '-')
101
+ ]]
102
+
103
+
104
+ # List of (romaji, ipa) pairs:
105
+ _romaji_to_ipa = [(re.compile('%s' % x[0], re.IGNORECASE), x[1]) for x in [
106
+ ('ʃy', 'ʃ'),
107
+ ('ʧʰy', 'ʧʰ'),
108
+ ('ʧ⁼y', 'ʧ⁼'),
109
+ ('NN', 'n'),
110
+ ('Ng', 'ŋ'),
111
+ ('y', 'j'),
112
+ ('h', 'x')
113
+ ]]
114
+
115
+
116
+ def number_to_chinese(text):
117
+ numbers = re.findall(r'\d+(?:\.?\d+)?', text)
118
+ for number in numbers:
119
+ text = text.replace(number, cn2an.an2cn(number), 1)
120
+ return text
121
+
122
+
123
+ def chinese_to_bopomofo(text):
124
+ text = text.replace('、', ',').replace(';', ',').replace(':', ',')
125
+ words = jieba.lcut(text, cut_all=False)
126
+ text = ''
127
+ for word in words:
128
+ bopomofos = lazy_pinyin(word, BOPOMOFO)
129
+ if not re.search('[\u4e00-\u9fff]', word):
130
+ text += word
131
+ continue
132
+ for i in range(len(bopomofos)):
133
+ if re.match('[\u3105-\u3129]', bopomofos[i][-1]):
134
+ bopomofos[i] += 'ˉ'
135
+ if text != '':
136
+ text += ' '
137
+ text += ''.join(bopomofos)
138
+ return text
139
+
140
+
141
+ def latin_to_bopomofo(text):
142
+ for regex, replacement in _latin_to_bopomofo:
143
+ text = re.sub(regex, replacement, text)
144
+ return text
145
+
146
+
147
+ def bopomofo_to_romaji(text):
148
+ for regex, replacement in _bopomofo_to_romaji:
149
+ text = re.sub(regex, replacement, text)
150
+ return text
151
+
152
+
153
+ def chinese_to_romaji(text):
154
+ text = number_to_chinese(text)
155
+ text = chinese_to_bopomofo(text)
156
+ text = latin_to_bopomofo(text)
157
+ text = bopomofo_to_romaji(text)
158
+ text = re.sub('i[aoe]', lambda x: 'y'+x.group(0)[1:], text)
159
+ text = re.sub('u[aoəe]', lambda x: 'w'+x.group(0)[1:], text)
160
+ text = re.sub('([ʦsɹ]`[⁼ʰ]?)([→↓↑ ]+|$)', lambda x: x.group(1) +
161
+ 'ɹ`'+x.group(2), text).replace('ɻ', 'ɹ`')
162
+ text = re.sub('([ʦs][⁼ʰ]?)([→↓↑ ]+|$)',
163
+ lambda x: x.group(1)+'ɹ'+x.group(2), text)
164
+ return text
165
+
166
+
167
+ def chinese_to_lazy_ipa(text):
168
+ text = chinese_to_romaji(text)
169
+ for regex, replacement in _romaji_to_ipa:
170
+ text = re.sub(regex, replacement, text)
171
+ return text
text/sanskrit.py ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+ from indic_transliteration import sanscript
3
+
4
+
5
+ # List of (iast, ipa) pairs:
6
+ _iast_to_ipa = [(re.compile('%s' % x[0]), x[1]) for x in [
7
+ ('a', 'ə'),
8
+ ('ā', 'aː'),
9
+ ('ī', 'iː'),
10
+ ('ū', 'uː'),
11
+ ('ṛ', 'ɹ`'),
12
+ ('ṝ', 'ɹ`ː'),
13
+ ('ḷ', 'l`'),
14
+ ('ḹ', 'l`ː'),
15
+ ('e', 'eː'),
16
+ ('o', 'oː'),
17
+ ('k', 'k⁼'),
18
+ ('k⁼h', 'kʰ'),
19
+ ('g', 'g⁼'),
20
+ ('g⁼h', 'gʰ'),
21
+ ('ṅ', 'ŋ'),
22
+ ('c', 'ʧ⁼'),
23
+ ('ʧ⁼h', 'ʧʰ'),
24
+ ('j', 'ʥ⁼'),
25
+ ('ʥ⁼h', 'ʥʰ'),
26
+ ('ñ', 'n^'),
27
+ ('ṭ', 't`⁼'),
28
+ ('t`⁼h', 't`ʰ'),
29
+ ('ḍ', 'd`⁼'),
30
+ ('d`⁼h', 'd`ʰ'),
31
+ ('ṇ', 'n`'),
32
+ ('t', 't⁼'),
33
+ ('t⁼h', 'tʰ'),
34
+ ('d', 'd⁼'),
35
+ ('d⁼h', 'dʰ'),
36
+ ('p', 'p⁼'),
37
+ ('p⁼h', 'pʰ'),
38
+ ('b', 'b⁼'),
39
+ ('b⁼h', 'bʰ'),
40
+ ('y', 'j'),
41
+ ('ś', 'ʃ'),
42
+ ('ṣ', 's`'),
43
+ ('r', 'ɾ'),
44
+ ('l̤', 'l`'),
45
+ ('h', 'ɦ'),
46
+ ("'", ''),
47
+ ('~', '^'),
48
+ ('ṃ', '^')
49
+ ]]
50
+
51
+
52
+ def devanagari_to_ipa(text):
53
+ text = text.replace('ॐ', 'ओम्')
54
+ text = re.sub(r'\s*।\s*$', '.', text)
55
+ text = re.sub(r'\s*।\s*', ', ', text)
56
+ text = re.sub(r'\s*॥', '.', text)
57
+ text = sanscript.transliterate(text, sanscript.DEVANAGARI, sanscript.IAST)
58
+ for regex, replacement in _iast_to_ipa:
59
+ text = re.sub(regex, replacement, text)
60
+ text = re.sub('(.)[`ː]*ḥ', lambda x: x.group(0)
61
+ [:-1]+'h'+x.group(1)+'*', text)
62
+ return text