ndeclarke's picture
Upload tokenizer
7cc1c71 verified
raw
history blame
1.65 kB
{
"tam-128": {
"&": 1,
"[PAD]": 53,
"[UNK]": 52,
"_": 2,
"|": 0,
"ஃ": 3,
"அ": 4,
"ஆ": 5,
"இ": 6,
"ஈ": 7,
"உ": 8,
"ஊ": 9,
"எ": 10,
"ஏ": 11,
"ஐ": 12,
"ஒ": 13,
"ஓ": 14,
"ஔ": 15,
"க": 16,
"ங": 17,
"ச": 18,
"ஜ": 19,
"ஞ": 20,
"ட": 21,
"ண": 22,
"த": 23,
"ந": 24,
"ன": 25,
"ப": 26,
"ம": 27,
"ய": 28,
"ர": 29,
"ற": 30,
"ல": 31,
"ள": 32,
"ழ": 33,
"வ": 34,
"ஷ": 35,
"ஸ": 36,
"ஹ": 37,
"ா": 38,
"ி": 39,
"ீ": 40,
"ு": 41,
"ூ": 42,
"ெ": 43,
"ே": 44,
"ை": 45,
"ொ": 46,
"ோ": 47,
"ௌ": 48,
"்": 49,
"ௗ": 50,
"ഥ": 51
},
"tam-32": {
"&": 1,
"[PAD]": 54,
"[UNK]": 53,
"_": 2,
"|": 0,
"¾": 3,
"ஃ": 4,
"அ": 5,
"ஆ": 6,
"இ": 7,
"ஈ": 8,
"உ": 9,
"ஊ": 10,
"எ": 11,
"ஏ": 12,
"ஐ": 13,
"ஒ": 14,
"ஓ": 15,
"ஔ": 16,
"க": 17,
"ங": 18,
"ச": 19,
"ஜ": 20,
"ஞ": 21,
"ட": 22,
"ண": 23,
"த": 24,
"ந": 25,
"ன": 26,
"ப": 27,
"ம": 28,
"ய": 29,
"ர": 30,
"ற": 31,
"ல": 32,
"ள": 33,
"ழ": 34,
"வ": 35,
"ஷ": 36,
"ஸ": 37,
"ஹ": 38,
"ா": 39,
"ி": 40,
"ீ": 41,
"ு": 42,
"ூ": 43,
"ெ": 44,
"ே": 45,
"ை": 46,
"ொ": 47,
"ோ": 48,
"ௌ": 49,
"்": 50,
"ௗ": 51,
"ഥ": 52
}
}