reichenbach's picture
add tokenizer
83120c6
raw
history blame
1.01 kB
{"a": 1, "b": 2, "c": 3, "d": 4, "e": 5, "f": 6, "g": 7, "h": 8, "i": 9, "j": 10, "k": 11, "l": 12, "m": 13, "n": 14, "o": 15, "p": 16, "r": 17, "s": 18, "t": 19, "u": 20, "v": 21, "w": 22, "x": 23, "y": 24, "z": 25, "|": 26, "ँ": 27, "ं": 28, "ः": 29, "अ": 30, "आ": 31, "इ": 32, "ई": 33, "उ": 34, "ऊ": 35, "ऋ": 36, "ए": 37, "ऐ": 38, "ऑ": 39, "ओ": 40, "औ": 41, "क": 42, "ख": 43, "ग": 44, "घ": 45, "च": 46, "छ": 47, "ज": 48, "झ": 49, "ञ": 50, "ट": 51, "ठ": 52, "ड": 53, "ढ": 54, "ण": 55, "त": 56, "थ": 57, "द": 58, "ध": 59, "न": 60, "प": 61, "फ": 62, "ब": 63, "भ": 64, "म": 65, "य": 66, "र": 67, "ल": 68, "व": 69, "श": 70, "ष": 71, "स": 72, "ह": 73, "़": 74, "ा": 75, "ि": 76, "ी": 77, "ु": 78, "ू": 79, "ृ": 80, "ॅ": 81, "े": 82, "ै": 83, "ॉ": 84, "ो": 85, "ौ": 86, "्": 87, "क़": 88, "ग़": 89, "ज़": 90, "ड़": 91, "ढ़": 92, "।": 93, "’": 94, "^": 0, "[UNK]": 95, "[PAD]": 96}