NX2411's picture
add tokenizer
0fe3920
{
"[PAD]": 50,
"[UNK]": 49,
"|": 38,
"γ„±": 35,
"γ„²": 26,
"γ„³": 27,
"γ„΄": 12,
"γ„΅": 31,
"γ„Ά": 6,
"γ„·": 14,
"γ„Έ": 45,
"γ„Ή": 23,
"γ„Ί": 28,
"γ„»": 41,
"γ„Ό": 43,
"γ…€": 9,
"ㅁ": 29,
"γ…‚": 3,
"γ…ƒ": 17,
"γ…„": 2,
"γ……": 11,
"γ…†": 36,
"γ…‡": 19,
"γ…ˆ": 16,
"γ…‰": 7,
"γ…Š": 8,
"γ…‹": 25,
"γ…Œ": 42,
"ㅍ": 46,
"γ…Ž": 24,
"ㅏ": 10,
"ㅐ": 32,
"γ…‘": 39,
"γ…’": 5,
"γ…“": 22,
"γ…”": 20,
"γ…•": 40,
"γ…–": 44,
"γ…—": 47,
"γ…˜": 15,
"γ…™": 0,
"γ…š": 48,
"γ…›": 13,
"γ…œ": 34,
"ㅝ": 1,
"γ…ž": 33,
"γ…Ÿ": 4,
"γ… ": 30,
"γ…‘": 18,
"γ…’": 37,
"γ…£": 21
}