juierror's picture
add tokenizer
114eb5b
{"ก": 1, "ข": 2, "ค": 3, "ฆ": 4, "ง": 5, "จ": 6, "ฉ": 7, "ช": 8, "ซ": 9, "ญ": 10, "ฎ": 11, "ฏ": 12, "ฐ": 13, "ฑ": 14, "ฒ": 15, "ณ": 16, "ด": 17, "ต": 18, "ถ": 19, "ท": 20, "ธ": 21, "น": 22, "บ": 23, "ป": 24, "ผ": 25, "ฝ": 26, "พ": 27, "ฟ": 28, "ภ": 29, "ม": 30, "ย": 31, "ร": 32, "ฤ": 33, "ล": 34, "ว": 35, "ศ": 36, "ษ": 37, "ส": 38, "ห": 39, "ฬ": 40, "อ": 41, "ฮ": 42, "ะ": 43, "ั": 44, "า": 45, "ำ": 46, "ิ": 47, "ี": 48, "ึ": 49, "ื": 50, "ุ": 51, "ู": 52, "เ": 53, "แ": 54, "โ": 55, "ใ": 56, "ไ": 57, "ๅ": 58, "ๆ": 59, "็": 60, "่": 61, "้": 62, "๊": 63, "๋": 64, "์": 65, "ํ": 66, "|": 0, "[UNK]": 67, "[PAD]": 68}