add tokenizer

runs/Nov05_12-57-59_368dc99567e0/1636117097.595878/events.out.tfevents.1636117097.368dc99567e0.785.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:874fea0bf0e0adc7304def2b3380e304c1cd82ba479af72f67172df4af454f05
+size 4573

runs/Nov05_12-57-59_368dc99567e0/events.out.tfevents.1636117097.368dc99567e0.785.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e18e09dd904368775a50f073a77ba5ae25a04a1d71b8e296e1abab3db9d462de
+size 4604

vocab.json

@@ -1 +1 @@
-{"व": 0, "ण": 1, "e": 2, "ए": 3, "ग": 4, "ि": 5, "य": 7, "ड़": 8, "उ": 9, "े": 10, "f": 11, "ल": 12, "i": 13, "ौ": 14, "र": 15, "ै": 16, "द": 17, "इ": 18, "क": 19, "ृ": 20, "अ": 21, "p": 22, "a": 23, "।": 24, "r": 25, "ज": 26, "ा": 27, "u": 28, "ई": 29, "w": 30, "ऐ": 31, "ठ": 32, "ख": 33, "'": 34, "भ": 35, "छ": 36, "म": 37, "आ": 38, "ः": 39, "ं": 40, "्": 41, "ह": 42, "ष": 43, "ओ": 44, "ँ": 45, "ू": 46, "च": 47, "स": 48, "ब": 49, "श": 50, "झ": 51, "ो": 52, "m": 53, "ट": 54, "प": 55, "ऑ": 56, "ी": 57, "फ": 58, "घ": 59, "ु": 60, "औ": 61, "ॉ": 62, "l": 63, "ऊ": 64, "त": 65, "थ": 66, "ड": 67, "़": 68, "न": 69, "ध": 70, "ढ": 71, "|": 6, "[UNK]": 72, "[PAD]": 73}
+{"ऑ": 0, "स": 1, "क": 2, "p": 3, "श": 4, "ई": 5, "ब": 6, "f": 7, "च": 8, "a": 9, "छ": 10, "ध": 11, "फ": 12, "ि": 13, "भ": 14, "थ": 15, "m": 16, "ः": 17, "घ": 18, "ह": 19, "प": 20, "य": 21, "ड": 22, "ल": 23, "l": 24, "त": 25, "ऊ": 26, "ा": 27, "द": 28, "ृ": 29, "'": 30, "e": 31, "ु": 32, "़": 33, "ओ": 34, "ौ": 35, "ऐ": 36, "े": 37, "w": 38, "i": 39, "ॉ": 40, "झ": 41, "ू": 42, "र": 43, "ए": 44, "म": 45, "r": 46, "इ": 47, "ं": 48, "ख": 49, "्": 50, "ग": 51, "औ": 52, "ड़": 53, "।": 54, "ज": 55, "ी": 56, "व": 57, "u": 58, "ठ": 59, "ष": 60, "ो": 61, "न": 62, "ण": 63, "ँ": 64, "ै": 65, "अ": 67, "ढ": 68, "उ": 69, "ट": 70, "आ": 71, "|": 66, "[UNK]": 72, "[PAD]": 73}