hyyoka's picture
add tokenizer
937dcd1
{"ㅞ": 0, "ㅆ": 1, "ㅐ": 2, "ㅋ": 3, "ㅢ": 4, "ㄹ": 5, "ㅓ": 6, "ㄻ": 7, "ㅣ": 8, "ㄼ": 9, "ㅄ": 10, "ㅝ": 11, "ㅇ": 12, "ㅕ": 13, "ㅟ": 14, "ㄱ": 15, "ㅒ": 16, "ㅘ": 17, "ㅛ": 18, "ㄵ": 19, "ㅠ": 20, "ㄶ": 21, "ㅈ": 22, "ㅜ": 23, "ㅀ": 24, "ㅏ": 25, "ㄲ": 26, "ㄷ": 27, "ㅍ": 28, "ㅗ": 29, "ㅎ": 30, "ㅑ": 31, "ㅡ": 32, "ㅃ": 33, "ㅌ": 34, "ㅅ": 35, "ㅙ": 36, "ㅚ": 37, "ㅁ": 38, "ㅖ": 40, "ㅂ": 41, "ㄴ": 42, "ㅉ": 43, "ㄺ": 44, "ㅔ": 45, "ㄸ": 46, "ㅊ": 47, "|": 39, "[UNK]": 48, "[PAD]": 49}