add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

runs/Nov05_12-47-57_368dc99567e0/1636116515.785063/events.out.tfevents.1636116515.368dc99567e0.77.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e4d0e6b6cecffe7a0f9ce78aea9aa714c94ca5457e30f34b65bd60f6d6bff5a
+size 4576

runs/Nov05_12-47-57_368dc99567e0/1636116544.3928304/events.out.tfevents.1636116544.368dc99567e0.77.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a7b1bd31ba4e0d14774831bd45caccc736256896c729e74186273587d8dc2f3
+size 4576

runs/Nov05_12-47-57_368dc99567e0/events.out.tfevents.1636116515.368dc99567e0.77.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b56b254e23f1df56dbe83ccdf2e5bbcdd76d0b5302ec666e7a8378bb3bf7a138
+size 8860

vocab.json

@@ -1 +1 @@
-{"प": 0, "न": 2, "ी": 3, "ह": 4, "ख": 5, "य": 6, "ड़": 7, "च": 8, "l": 9, "घ": 10, "़": 11, "स": 12, "e": 13, "म": 14, "ध": 15, "ज": 16, "ओ": 17, "ा": 18, "झ": 19, "w": 20, "्": 21, "m": 22, "'": 23, "f": 24, "ऑ": 25, "थ": 26, "उ": 27, "ऊ": 28, "ौ": 29, "ृ": 30, "इ": 31, "p": 32, "ँ": 33, "श": 34, "ढ": 35, "ऐ": 36, "ई": 37, "ि": 38, "त": 39, "ष": 40, "छ": 41, "द": 42, "ू": 43, "।": 44, "क": 45, "ड": 46, "ल": 47, "आ": 48, "ण": 49, "औ": 50, "ु": 51, "अ": 52, "व": 53, "ः": 54, "र": 55, "फ": 56, "ै": 57, "r": 58, "ॉ": 59, "ब": 60, "ो": 61, "i": 62, "ट": 63, "ठ": 64, "ग": 65, "े": 66, "भ": 67, "ं": 68, "ए": 69, "a": 70, "u": 71, "|": 1, "[UNK]": 72, "[PAD]": 73}
+{"व": 0, "ण": 1, "e": 2, "ए": 3, "ग": 4, "ि": 5, "य": 7, "ड़": 8, "उ": 9, "े": 10, "f": 11, "ल": 12, "i": 13, "ौ": 14, "र": 15, "ै": 16, "द": 17, "इ": 18, "क": 19, "ृ": 20, "अ": 21, "p": 22, "a": 23, "।": 24, "r": 25, "ज": 26, "ा": 27, "u": 28, "ई": 29, "w": 30, "ऐ": 31, "ठ": 32, "ख": 33, "'": 34, "भ": 35, "छ": 36, "म": 37, "आ": 38, "ः": 39, "ं": 40, "्": 41, "ह": 42, "ष": 43, "ओ": 44, "ँ": 45, "ू": 46, "च": 47, "स": 48, "ब": 49, "श": 50, "झ": 51, "ो": 52, "m": 53, "ट": 54, "प": 55, "ऑ": 56, "ी": 57, "फ": 58, "घ": 59, "ु": 60, "औ": 61, "ॉ": 62, "l": 63, "ऊ": 64, "त": 65, "थ": 66, "ड": 67, "़": 68, "न": 69, "ध": 70, "ढ": 71, "|": 6, "[UNK]": 72, "[PAD]": 73}