add tokenizer

runs/Nov07_09-45-10_c07766eb6e6a/1636278335.4644344/events.out.tfevents.1636278335.c07766eb6e6a.1686.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc4893ea670560a469b0b3c96ea7d09d687c9b2a75cb3886330866c44617087f
+size 4573

runs/Nov07_09-45-10_c07766eb6e6a/events.out.tfevents.1636278335.c07766eb6e6a.1686.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f48bac05e0d0e9fbde06db1abfc9629a9ef90a12ba2aa0f3bae94f776168374f
+size 4453

vocab.json

@@ -1 +1 @@
-{"र": 0, "l": 1, "ि": 2, "ख": 3, "आ": 4, "ृ": 5, "ी": 6, "ं": 7, "प": 8, "ओ": 9, "ध": 10, "ढ": 11, "अ": 12, "u": 13, "़": 14, "क": 15, "च": 16, "ड़": 17, "p": 18, "उ": 19, "ॉ": 20, "ल": 21, "न": 22, "ै": 23, "्": 24, "ब": 25, "औ": 26, "ु": 27, "ण": 28, "'": 29, "श": 30, "ट": 31, "ः": 32, "i": 33, "ौ": 34, "ऊ": 35, "घ": 36, "ग": 37, "ष": 38, "m": 39, "ड": 40, "ई": 41, "स": 42, "इ": 43, "ए": 44, "े": 45, "a": 46, "।": 47, "r": 48, "ो": 49, "ऑ": 50, "w": 51, "भ": 52, "छ": 53, "f": 54, "त": 55, "ऐ": 56, "य": 57, "थ": 58, "ठ": 59, "झ": 60, "द": 61, "ँ": 62, "ू": 63, "फ": 64, "ह": 66, "म": 67, "ा": 68, "व": 69, "e": 70, "ज": 71, "|": 65, "[UNK]": 72, "[PAD]": 73}
+{"व": 1, "ऊ": 2, "च": 3, "घ": 4, "इ": 5, "र": 6, "ई": 7, "अ": 8, "ृ": 9, "।": 10, "r": 11, "े": 12, "ख": 13, "फ": 14, "w": 15, "़": 16, "ौ": 17, "'": 18, "ी": 19, "e": 20, "झ": 21, "f": 22, "ए": 23, "ॉ": 24, "ड": 25, "ँ": 26, "m": 27, "उ": 28, "ो": 29, "ग": 30, "ा": 31, "आ": 32, "ू": 33, "त": 34, "ष": 35, "p": 36, "ज": 37, "ण": 38, "ह": 39, "प": 40, "म": 41, "ि": 42, "i": 43, "a": 44, "ऑ": 45, "भ": 46, "u": 47, "स": 48, "श": 49, "ध": 50, "न": 51, "य": 52, "ट": 53, "्": 54, "द": 55, "औ": 56, "ं": 57, "ल": 58, "ै": 59, "ढ": 60, "ड़": 61, "ठ": 62, "l": 63, "ः": 64, "ब": 65, "क": 66, "थ": 67, "ओ": 68, "छ": 69, "ऐ": 70, "ु": 71, "|": 0, "[UNK]": 72, "[PAD]": 73}