add tokenizer

158a838 over 2 years ago

No virus

1.07 kB

{"a": 1, "c": 2, "d": 3, "f": 4, "g": 5, "m": 6, "n": 7, "o": 8, "p": 9, "s": 10, "u": 11, "x": 12, "z": 13, "々": 14, "ぁ": 15, "あ": 16, "ぃ": 17, "い": 18, "ぅ": 19, "う": 20, "ぇ": 21, "え": 22, "ぉ": 23, "お": 24, "か": 25, "が": 26, "き": 27, "ぎ": 28, "く": 29, "ぐ": 30, "け": 31, "げ": 32, "こ": 33, "ご": 34, "さ": 35, "ざ": 36, "し": 37, "じ": 38, "す": 39, "ず": 40, "せ": 41, "ぜ": 42, "そ": 43, "ぞ": 44, "た": 45, "だ": 46, "ち": 47, "ぢ": 48, "っ": 49, "つ": 50, "づ": 51, "て": 52, "で": 53, "と": 54, "ど": 55, "な": 56, "に": 57, "ぬ": 58, "ね": 59, "の": 60, "は": 61, "ば": 62, "ぱ": 63, "ひ": 64, "び": 65, "ぴ": 66, "ふ": 67, "ぶ": 68, "ぷ": 69, "へ": 70, "べ": 71, "ぺ": 72, "ほ": 73, "ぼ": 74, "ぽ": 75, "ま": 76, "み": 77, "む": 78, "め": 79, "も": 80, "ゃ": 81, "や": 82, "ゅ": 83, "ゆ": 84, "ょ": 85, "よ": 86, "ら": 87, "り": 88, "る": 89, "れ": 90, "ろ": 91, "わ": 92, "を": 93, "ん": 94, "ゔ": 95, "ゖ": 96, "ー": 97, "|": 0, "[UNK]": 98, "[PAD]": 99}