Bharathdamu's picture
add tokenizer
1c631c4
{"उ": 0, "स": 1, "ी": 3, "ु": 4, "न": 5, "र": 6, "घ": 7, "ड़": 8, "ि": 9, "य": 10, "क": 11, "त": 12, "अ": 13, "ज": 14, "छ": 15, "भ": 16, "ै": 17, "ग": 18, "ह": 19, "ट": 20, "्": 21, "प": 22, "ब": 23, "द": 24, "ँ": 25, "ा": 26, "ई": 27, "ो": 28, "म": 29, "े": 30, "ं": 31, "ू": 32, "ल": 33, "व": 34, "औ": 35, "ष": 36, "|": 2, "[UNK]": 37, "[PAD]": 38}