cstorm125's picture
add tokenizer
cb44b86
raw
history blame
762 Bytes
{"ง": 0, "ึ": 1, "ห": 2, "ซ": 3, "ศ": 4, "ฎ": 5, "ษ": 6, "ฌ": 7, "อ": 8, "ำ": 9, "ๅ": 10, "ฉ": 11, "์": 12, "ผ": 13, "ื": 14, "ณ": 15, "ล": 16, "ไ": 17, "ข": 18, "ด": 19, "ส": 20, "ฯ": 21, "น": 22, "ะ": 23, "ี": 24, "โ": 25, "ธ": 26, "ว": 27, "ฐ": 28, "ฬ": 29, "บ": 30, "ป": 31, "ภ": 32, "๋": 33, "ช": 34, "ิ": 35, "ย": 36, "ม": 37, "เ": 38, "้": 39, "ั": 40, "ฝ": 41, "ต": 42, "่": 43, "ู": 44, "ฆ": 45, "ใ": 46, "ุ": 47, "ค": 48, "ฮ": 49, "ฟ": 50, "๊": 51, "ญ": 52, "ฤ": 53, "ฒ": 54, "ถ": 55, "ฏ": 56, "็": 57, "ฑ": 58, "ฃ": 59, "จ": 60, "า": 61, "ก": 63, "ร": 64, "ท": 65, "พ": 66, "แ": 67, "|": 62, "[UNK]": 68, "[PAD]": 69}