# https://github.com/polm/cutlet/blob/master/cutlet/cutlet.py from dataclasses import dataclass from fugashi import Tagger from num2kana import Convert import mojimoji import re import unicodedata HEPBURN = { chr(12449):'a', #ァ chr(12450):'a', #ア chr(12451):'i', #ィ chr(12452):'i', #イ chr(12453):'ɯ', #ゥ chr(12454):'ɯ', #ウ chr(12455):'e', #ェ chr(12456):'e', #エ chr(12457):'o', #ォ chr(12458):'o', #オ chr(12459):'ka', #カ chr(12460):'ɡa', #ガ chr(12461):'ki', #キ chr(12462):'ɡi', #ギ chr(12463):'kɯ', #ク chr(12464):'ɡɯ', #グ chr(12465):'ke', #ケ chr(12466):'ɡe', #ゲ chr(12467):'ko', #コ chr(12468):'ɡo', #ゴ chr(12469):'sa', #サ chr(12470):'za', #ザ chr(12471):'ɕi', #シ chr(12472):'dʑi', #ジ chr(12473):'sɨ', #ス chr(12474):'zɨ', #ズ chr(12475):'se', #セ chr(12476):'ze', #ゼ chr(12477):'so', #ソ chr(12478):'zo', #ゾ chr(12479):'ta', #タ chr(12480):'da', #ダ chr(12481):'tɕi', #チ chr(12482):'dʑi', #ヂ # chr(12483) #ッ chr(12484):'tsɨ', #ツ chr(12485):'zɨ', #ヅ chr(12486):'te', #テ chr(12487):'de', #デ chr(12488):'to', #ト chr(12489):'do', #ド chr(12490):'na', #ナ chr(12491):'ɲi', #ニ chr(12492):'nɯ', #ヌ chr(12493):'ne', #ネ chr(12494):'no', #ノ chr(12495):'ha', #ハ chr(12496):'ba', #バ chr(12497):'pa', #パ chr(12498):'çi', #ヒ chr(12499):'bi', #ビ chr(12500):'pi', #ピ chr(12501):'ɸɯ', #フ chr(12502):'bɯ', #ブ chr(12503):'pɯ', #プ chr(12504):'he', #ヘ chr(12505):'be', #ベ chr(12506):'pe', #ペ chr(12507):'ho', #ホ chr(12508):'bo', #ボ chr(12509):'po', #ポ chr(12510):'ma', #マ chr(12511):'mi', #ミ chr(12512):'mɯ', #ム chr(12513):'me', #メ chr(12514):'mo', #モ chr(12515):'ja', #ャ chr(12516):'ja', #ヤ chr(12517):'jɯ', #ュ chr(12518):'jɯ', #ユ chr(12519):'jo', #ョ chr(12520):'jo', #ヨ chr(12521):'ra', #ラ chr(12522):'ri', #リ chr(12523):'rɯ', #ル chr(12524):'re', #レ chr(12525):'ro', #ロ chr(12526):'wa', #ヮ chr(12527):'wa', #ワ chr(12528):'i', #ヰ chr(12529):'e', #ヱ chr(12530):'o', #ヲ # chr(12531) #ン chr(12532):'vɯ', #ヴ chr(12533):'ka', #ヵ chr(12534):'ke', #ヶ } assert len(HEPBURN) == 84 and all(i in {12483, 12531} or chr(i) in HEPBURN for i in range(12449, 12535)) for k, v in list(HEPBURN.items()): HEPBURN[chr(ord(k)-96)] = v assert len(HEPBURN) == 84*2 HEPBURN.update({ chr(12535):'va', #ヷ chr(12536):'vi', #ヸ chr(12537):'ve', #ヹ chr(12538):'vo', #ヺ }) assert len(HEPBURN) == 84*2+4 and all(chr(i) in HEPBURN for i in range(12535, 12539)) HEPBURN.update({ chr(12784):'kɯ', #ㇰ chr(12785):'ɕi', #ㇱ chr(12786):'sɨ', #ㇲ chr(12787):'to', #ㇳ chr(12788):'nɯ', #ㇴ chr(12789):'ha', #ㇵ chr(12790):'çi', #ㇶ chr(12791):'ɸɯ', #ㇷ chr(12792):'he', #ㇸ chr(12793):'ho', #ㇹ chr(12794):'mɯ', #ㇺ chr(12795):'ra', #ㇻ chr(12796):'ri', #ㇼ chr(12797):'rɯ', #ㇽ chr(12798):'re', #ㇾ chr(12799):'ro', #ㇿ }) assert len(HEPBURN) == 84*2+4+16 and all(chr(i) in HEPBURN for i in range(12784, 12800)) HEPBURN.update({ chr(12452)+chr(12455):'je', #イェ chr(12454)+chr(12451):'wi', #ウィ chr(12454)+chr(12455):'we', #ウェ chr(12454)+chr(12457):'wo', #ウォ chr(12461)+chr(12455):'kʲe', #キェ chr(12461)+chr(12515):'kʲa', #キャ chr(12461)+chr(12517):'kʲɨ', #キュ chr(12461)+chr(12519):'kʲo', #キョ chr(12462)+chr(12515):'ɡʲa', #ギャ chr(12462)+chr(12517):'ɡʲɨ', #ギュ chr(12462)+chr(12519):'ɡʲo', #ギョ chr(12463)+chr(12449):'kʷa', #クァ chr(12463)+chr(12451):'kʷi', #クィ chr(12463)+chr(12455):'kʷe', #クェ chr(12463)+chr(12457):'kʷo', #クォ chr(12464)+chr(12449):'ɡʷa', #グァ chr(12464)+chr(12451):'ɡʷi', #グィ chr(12464)+chr(12455):'ɡʷe', #グェ chr(12464)+chr(12457):'ɡʷo', #グォ chr(12471)+chr(12455):'ɕe', #シェ chr(12471)+chr(12515):'ɕa', #シャ chr(12471)+chr(12517):'ɕɨ', #シュ chr(12471)+chr(12519):'ɕo', #ショ chr(12472)+chr(12455):'dʑe', #ジェ chr(12472)+chr(12515):'dʑa', #ジャ chr(12472)+chr(12517):'dʑɨ', #ジュ chr(12472)+chr(12519):'dʑo', #ジョ chr(12481)+chr(12455):'tɕe', #チェ chr(12481)+chr(12515):'tɕa', #チャ chr(12481)+chr(12517):'tɕɨ', #チュ chr(12481)+chr(12519):'tɕo', #チョ chr(12482)+chr(12515):'dʑa', #ヂャ chr(12482)+chr(12517):'dʑɨ', #ヂュ chr(12482)+chr(12519):'dʑo', #ヂョ chr(12484)+chr(12449):'tsa', #ツァ chr(12484)+chr(12451):'tsi', #ツィ chr(12484)+chr(12455):'tse', #ツェ chr(12484)+chr(12457):'tso', #ツォ chr(12486)+chr(12451):'ti', #ティ chr(12486)+chr(12517):'tʲɨ', #テュ chr(12487)+chr(12451):'di', #ディ chr(12487)+chr(12517):'dʲɨ', #デュ chr(12488)+chr(12453):'tɯ', #トゥ chr(12489)+chr(12453):'dɯ', #ドゥ chr(12491)+chr(12455):'ɲe', #ニェ chr(12491)+chr(12515):'ɲa', #ニャ chr(12491)+chr(12517):'ɲɨ', #ニュ chr(12491)+chr(12519):'ɲo', #ニョ chr(12498)+chr(12455):'çe', #ヒェ chr(12498)+chr(12515):'ça', #ヒャ chr(12498)+chr(12517):'çɨ', #ヒュ chr(12498)+chr(12519):'ço', #ヒョ chr(12499)+chr(12515):'bʲa', #ビャ chr(12499)+chr(12517):'bʲɨ', #ビュ chr(12499)+chr(12519):'bʲo', #ビョ chr(12500)+chr(12515):'pʲa', #ピャ chr(12500)+chr(12517):'pʲɨ', #ピュ chr(12500)+chr(12519):'pʲo', #ピョ chr(12501)+chr(12449):'ɸa', #ファ chr(12501)+chr(12451):'ɸi', #フィ chr(12501)+chr(12455):'ɸe', #フェ chr(12501)+chr(12457):'ɸo', #フォ chr(12501)+chr(12517):'ɸʲɨ', #フュ chr(12501)+chr(12519):'ɸʲo', #フョ chr(12511)+chr(12515):'mʲa', #ミャ chr(12511)+chr(12517):'mʲɨ', #ミュ chr(12511)+chr(12519):'mʲo', #ミョ chr(12522)+chr(12515):'rʲa', #リャ chr(12522)+chr(12517):'rʲɨ', #リュ chr(12522)+chr(12519):'rʲo', #リョ chr(12532)+chr(12449):'va', #ヴァ chr(12532)+chr(12451):'vi', #ヴィ chr(12532)+chr(12455):'ve', #ヴェ chr(12532)+chr(12457):'vo', #ヴォ chr(12532)+chr(12517):'vʲɨ', #ヴュ chr(12532)+chr(12519):'vʲo', #ヴョ }) assert len(HEPBURN) == 84*2+4+16+76 for k, v in list(HEPBURN.items()): if len(k) != 2: continue a, b = k assert a in HEPBURN and b in HEPBURN, (a, b) a = chr(ord(a)-96) b = chr(ord(b)-96) assert a in HEPBURN and b in HEPBURN, (a, b) HEPBURN[a+b] = v assert len(HEPBURN) == 84*2+4+16+76*2 HEPBURN.update({ # symbols # 'ー': '-', # 長音符, only used when repeated '。': '.', '、': ',', '？': '?', '！': '!', '「': '"', '」': '"', '『': '"', '』': '"', '：': ':', '（': '(', '）': ')', '《': '(', '》': ')', '【': '[', '】': ']', '・': ' ',#'/', '，': ',', '～': '—', '〜': '—', '—': '—', '«': '«', '»': '»', # other '゚': '', # combining handakuten by itself, just discard '゙': '', # combining dakuten by itself }) def add_dakuten(kk): """Given a kana (single-character string), add a dakuten.""" try: # ii = 'かきくけこさしすせそたちつてとはひふへほ'.index(kk) ii = 'カキクケコサシスセソタチツテトハヒフヘホ'.index(kk) return 'ガギグゲゴザジズゼゾダヂヅデドバビブベボ'[ii] # return 'がぎぐげござじずぜぞだぢづでどばびぶべぼ'[ii] except ValueError: # this is normal if the input is nonsense return None SUTEGANA = 'ャュョァィゥェォ' #'ゃゅょぁぃぅぇぉ' PUNCT = '\'".!?(),;:-' ODORI = '々〃ゝゞヽゞ' @dataclass class Token: surface: str space: bool # if a space should follow def __str__(self): sp = " " if self.space else "" return f"{self.surface}{sp}" class Katsu: def __init__(self): """Create a Katsu object, which holds configuration as well as tokenizer state. Typical usage: ```python katsu = Katsu() roma = katsu.romaji("カツカレーを食べた") # "Cutlet curry wo tabeta" ``` """ self.tagger = Tagger() self.table = dict(HEPBURN) # make a copy so we can modify it self.exceptions = {} def romaji(self, text): """Build a complete string from input text.""" if not text: return '' text = self._normalize_text(text) words = self.tagger(text) tokens = self._romaji_tokens(words) out = ''.join([str(tok) for tok in tokens]) return re.sub(r'\s+', ' ', out.strip()) def phonemize(self, texts): # espeak-ng API return [self.romaji(text) for text in texts] def _normalize_text(self, text): """Given text, normalize variations in Japanese. This specifically removes variations that are meaningless for romaji conversion using the following steps: - Unicode NFKC normalization - Full-width Latin to half-width - Half-width katakana to full-width """ # perform unicode normalization text = re.sub(r'[〜～](?=\d)', 'から', text) # wave dash range text = unicodedata.normalize('NFKC', text) # convert all full-width alphanum to half-width, since it can go out as-is text = mojimoji.zen_to_han(text, kana=False) # replace half-width katakana with full-width text = mojimoji.han_to_zen(text, digit=False, ascii=False) return ''.join([(' '+Convert(t)) if t.isdigit() else t for t in re.findall(r'\d+|\D+', text)]) def _romaji_tokens(self, words): """Build a list of tokens from input nodes.""" out = [] for wi, word in enumerate(words): po = out[-1] if out else None pw = words[wi - 1] if wi > 0 else None nw = words[wi + 1] if wi < len(words) - 1 else None roma = self._romaji_word(word) tok = Token(roma, False) # handle punctuation with atypical spacing surface = word.surface#['orig'] if surface in '「『' or roma in '([': if po: po.space = True elif surface in '」』' or roma in ']).,?!:': if po: po.space = False tok.space = True elif roma == ' ': tok.space = False else: tok.space = True out.append(tok) # remove any leftover sokuon for tok in out: tok.surface = tok.surface.replace(chr(12483), '') return out def _romaji_word(self, word): """Return the romaji for a single word (node).""" surface = word.surface#['orig'] if surface in self.exceptions: return self.exceptions[surface] assert not surface.isdigit(), surface if surface.isascii(): return surface kana = word.feature.pron or word.feature.kana or surface if word.is_unk: if word.char_type == 7: # katakana pass elif word.char_type == 3: # symbol return ''.join(map(lambda c: self.table.get(c, c), surface)) else: return '' # TODO: silently fail out = '' for ki, char in enumerate(kana): nk = kana[ki + 1] if ki < len(kana) - 1 else None pk = kana[ki - 1] if ki > 0 else None out += self._get_single_mapping(pk, char, nk) return out def _get_single_mapping(self, pk, kk, nk): """Given a single kana and its neighbors, return the mapped romaji.""" # handle odoriji # NOTE: This is very rarely useful at present because odoriji are not # left in readings for dictionary words, and we can't follow kana # across word boundaries. if kk in ODORI: if kk in 'ゝヽ': if pk: return pk else: return '' # invalid but be nice if kk in 'ゞヾ': # repeat with voicing if not pk: return '' vv = add_dakuten(pk) if vv: return self.table[vv] else: return '' # remaining are 々 for kanji and 〃 for symbols, but we can't # infer their span reliably (or handle rendaku) return '' # handle digraphs if pk and (pk + kk) in self.table: return self.table[pk + kk] if nk and (kk + nk) in self.table: return '' if nk and nk in SUTEGANA: if kk == 'ッ': return '' # never valid, just ignore return self.table[kk][:-1] + self.table[nk] if kk in SUTEGANA: return '' if kk == 'ー': # 長音符 return 'ː' if ord(kk) in {12387, 12483}: # っ or ッ tnk = self.table.get(nk) if tnk and tnk[0] in 'bdɸɡhçijkmnɲoprstɯvwz': return tnk[0] return kk if ord(kk) in {12435, 12531}: # ん or ン # https://en.wikipedia.org/wiki/N_(kana) # m before m,p,b # ŋ before k,g # ɲ before ɲ,tɕ,dʑ # n before n,t,d,r,z # ɴ otherwise tnk = self.table.get(nk) if tnk: if tnk[0] in 'mpb': return 'm' elif tnk[0] in 'kɡ': return 'ŋ' elif any(tnk.startswith(p) for p in ('ɲ','tɕ','dʑ')): return 'ɲ' elif tnk[0] in 'ntdrz': return 'n' return 'ɴ' return self.table.get(kk, '')