add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

runs/Feb06_16-38-46_1b71e7ec4aa7/1644165609.2940743/events.out.tfevents.1644165609.1b71e7ec4aa7.94.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c228f74f0cd36152270da0c2d47ef9e6a440e1cee4f286efa46b6b3e313d20a8
+size 4792

runs/Feb06_16-38-46_1b71e7ec4aa7/1644165721.2387004/events.out.tfevents.1644165721.1b71e7ec4aa7.94.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b317821b81d60058bae311cf35333af954a6ac0a7962352168fe43cc72025deb
+size 4792

runs/Feb06_16-38-46_1b71e7ec4aa7/events.out.tfevents.1644165609.1b71e7ec4aa7.94.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6523e98dbe0856f53ca052980434c5ab382445dddbc89b54f63e0dc0a70038e1
+size 9460

vocab.json

@@ -1 +1 @@
-{"ॅ": 0, "र": 1, "अ": 2, "z": 3, "ौ": 4, "आ": 5, "य": 6, "m": 7, "क": 8, "ण": 9, "्": 10, "c": 11, "ञ": 12, "ज": 13, "द": 14, "व": 15, "ै": 16, "ँ": 17, "ं": 18, "v": 19, "श": 20, "ऐ": 21, "प": 22, "ई": 23, "ओ": 24, "त": 25, "w": 26, "स": 27, "y": 28, "d": 29, "ल": 30, "k": 31, "े": 32, "s": 33, "थ": 34, "ड": 35, "फ": 36, "ु": 37, "i": 38, "ा": 40, "ः": 41, "ऊ": 42, "u": 43, "ए": 44, "औ": 45, "e": 46, "j": 47, "म": 48, "t": 49, "ढ": 50, "ड़": 51, "ी": 52, "ट": 53, "न": 54, "उ": 55, "ध": 56, "ह": 57, "़": 58, "ढ़": 59, "इ": 60, "ू": 61, "भ": 62, "झ": 63, "ृ": 64, "f": 65, "x": 66, "ऑ": 67, "ब": 68, "ष": 69, "ॉ": 70, "l": 71, "छ": 72, "a": 73, "च": 74, "h": 75, "r": 76, "ऋ": 77, "ज़": 78, "ग़": 79, "ग": 80, "ि": 81, "ख": 82, "क़": 83, "'": 84, "n": 85, "o": 86, "b": 87, "g": 88, "p": 89, "ठ": 90, "घ": 91, "ो": 92, "|": 39, "[UNK]": 93, "[PAD]": 94}
+{"ः": 0, "ल": 1, "उ": 2, "z": 3, "च": 4, "b": 5, "आ": 6, "ज़": 7, "ऋ": 8, "a": 9, "ग": 10, "ू": 11, "'": 12, "ऐ": 13, "ग़": 14, "े": 15, "इ": 16, "ै": 17, "ढ़": 18, "ौ": 19, "j": 20, "ब": 21, "घ": 22, "s": 23, "x": 24, "ि": 25, "ा": 26, "क": 27, "v": 28, "ऑ": 29, "r": 30, "ण": 31, "ञ": 32, "ड": 33, "t": 34, "c": 35, "त": 36, "श": 37, "थ": 38, "ॉ": 39, "ई": 40, "ॅ": 41, "र": 42, "y": 43, "व": 44, "्": 45, "p": 46, "m": 47, "ँ": 48, "य": 49, "ध": 50, "ं": 51, "ृ": 52, "ओ": 53, "l": 54, "भ": 55, "ो": 56, "ठ": 57, "i": 58, "ष": 59, "औ": 60, "ज": 61, "छ": 62, "स": 63, "ढ": 64, "म": 65, "अ": 66, "h": 67, "क़": 68, "ी": 69, "द": 70, "प": 71, "d": 72, "g": 73, "e": 74, "ड़": 75, "f": 76, "़": 77, "फ": 78, "ए": 79, "न": 80, "u": 81, "o": 83, "ह": 84, "n": 85, "ख": 86, "w": 87, "ु": 88, "ऊ": 89, "k": 90, "झ": 91, "ट": 92, "|": 82, "[UNK]": 93, "[PAD]": 94}