Hemantrao's picture
Upload tokenizer
94eaa67 verified
raw
history blame contribute delete
No virus
1.33 kB
{
" ": 0,
"!": 1,
"\"": 2,
"&": 3,
"'": 4,
",": 5,
"-": 6,
".": 7,
":": 8,
";": 9,
"?": 10,
"[PAD]": 103,
"[UNK]": 102,
"|": 11,
"ँ": 12,
"ं": 13,
"ः": 14,
"अ": 15,
"आ": 16,
"इ": 17,
"ई": 18,
"उ": 19,
"ऊ": 20,
"ऋ": 21,
"ऌ": 22,
"ऍ": 23,
"ऎ": 24,
"ए": 25,
"ऐ": 26,
"ऑ": 27,
"ओ": 28,
"औ": 29,
"क": 30,
"ख": 31,
"ग": 32,
"घ": 33,
"ङ": 34,
"च": 35,
"छ": 36,
"ज": 37,
"झ": 38,
"ञ": 39,
"ट": 40,
"ठ": 41,
"ड": 42,
"ढ": 43,
"ण": 44,
"त": 45,
"थ": 46,
"द": 47,
"ध": 48,
"न": 49,
"प": 50,
"फ": 51,
"ब": 52,
"भ": 53,
"म": 54,
"य": 55,
"र": 56,
"ऱ": 57,
"ल": 58,
"ळ": 59,
"व": 60,
"श": 61,
"ष": 62,
"स": 63,
"ह": 64,
"़": 65,
"ऽ": 66,
"ा": 67,
"ि": 68,
"ी": 69,
"ु": 70,
"ू": 71,
"ृ": 72,
"ॄ": 73,
"ॅ": 74,
"ॆ": 75,
"े": 76,
"ै": 77,
"ॉ": 78,
"ॊ": 79,
"ो": 80,
"ौ": 81,
"्": 82,
"॓": 83,
"॔": 84,
"क़": 85,
"ख़": 86,
"ग़": 87,
"ज़": 88,
"ड़": 89,
"ढ़": 90,
"फ़": 91,
"य़": 92,
"ॢ": 93,
"।": 94,
"॥": 95,
"ॲ": 96,
"–": 97,
"‘": 98,
"’": 99,
"“": 100,
"”": 101
}