Sajjo's picture
Upload tokenizer
5e540b3 verified
raw
history blame contribute delete
942 Bytes
{
"/": 1,
"[PAD]": 72,
"[UNK]": 71,
"|": 0,
"।": 2,
"॥": 3,
"ঁ": 4,
"ং": 5,
"ঃ": 6,
"অ": 7,
"আ": 8,
"ই": 9,
"ঈ": 10,
"উ": 11,
"ঊ": 12,
"ঋ": 13,
"এ": 14,
"ঐ": 15,
"ও": 16,
"ঔ": 17,
"ক": 18,
"খ": 19,
"গ": 20,
"ঘ": 21,
"ঙ": 22,
"চ": 23,
"ছ": 24,
"জ": 25,
"ঝ": 26,
"ঞ": 27,
"ট": 28,
"ঠ": 29,
"ড": 30,
"ঢ": 31,
"ণ": 32,
"ত": 33,
"থ": 34,
"দ": 35,
"ধ": 36,
"ন": 37,
"প": 38,
"ফ": 39,
"ব": 40,
"ভ": 41,
"ম": 42,
"য": 43,
"র": 44,
"ল": 45,
"শ": 46,
"ষ": 47,
"স": 48,
"হ": 49,
"়": 50,
"া": 51,
"ি": 52,
"ী": 53,
"ু": 54,
"ূ": 55,
"ৃ": 56,
"ে": 57,
"ৈ": 58,
"ো": 59,
"ৌ": 60,
"্": 61,
"ৎ": 62,
"ড়": 63,
"ঢ়": 64,
"য়": 65,
"ৰ": 66,
"–": 67,
"—": 68,
"’": 69,
"‚": 70
}