Upload tokenizer

added_tokens.json

@@ -1,4 +1,4 @@
 {
-  "</s>": 107,
-  "<s>": 106
+  "</s>": 108,
+  "<s>": 107
 }

tokenizer_config.json

@@ -1,6 +1,6 @@
 {
   "added_tokens_decoder": {
-    "104": {
+    "105": {
       "content": "[UNK]",
       "lstrip": true,
       "normalized": false,
@@ -8,7 +8,7 @@
       "single_word": false,
       "special": false
     },
-    "105": {
+    "106": {
       "content": "[PAD]",
       "lstrip": true,
       "normalized": false,
@@ -16,7 +16,7 @@
       "single_word": false,
       "special": false
     },
-    "106": {
+    "107": {
       "content": "<s>",
       "lstrip": false,
       "normalized": false,
@@ -24,7 +24,7 @@
       "single_word": false,
       "special": true
     },
-    "107": {
+    "108": {
       "content": "</s>",
       "lstrip": false,
       "normalized": false,

vocab.json

@@ -1,7 +1,7 @@
 {
   " ": 0,
-  "[PAD]": 105,
-  "[UNK]": 104,
+  "[PAD]": 106,
+  "[UNK]": 105,
   "ँ": 1,
   "ं": 2,
   "अ": 3,
@@ -24,85 +24,86 @@
   "ट": 20,
   "ठ": 21,
   "ड": 22,
-  "ण": 23,
-  "त": 24,
-  "थ": 25,
-  "द": 26,
-  "ध": 27,
-  "न": 28,
-  "प": 29,
-  "फ": 30,
-  "ब": 31,
-  "भ": 32,
-  "म": 33,
-  "य": 34,
-  "र": 35,
-  "ल": 36,
-  "व": 37,
-  "श": 38,
-  "ष": 39,
-  "स": 40,
-  "ह": 41,
-  "़": 42,
-  "ा": 43,
-  "ि": 44,
-  "ी": 45,
-  "ु": 46,
-  "ू": 47,
-  "े": 48,
-  "ै": 49,
-  "ॉ": 50,
-  "ो": 51,
-  "ौ": 52,
-  "्": 53,
-  "ం": 54,
-  "అ": 55,
-  "ఆ": 56,
-  "ఇ": 57,
-  "ఈ": 58,
-  "ఉ": 59,
-  "ఎ": 60,
-  "ఏ": 61,
-  "ఒ": 62,
-  "క": 63,
-  "ఖ": 64,
-  "గ": 65,
-  "ఘ": 66,
-  "చ": 67,
-  "జ": 68,
-  "ట": 69,
-  "డ": 70,
-  "ణ": 71,
-  "త": 72,
-  "థ": 73,
-  "ద": 74,
-  "ధ": 75,
-  "న": 76,
-  "ప": 77,
-  "ఫ": 78,
-  "బ": 79,
-  "భ": 80,
-  "మ": 81,
-  "య": 82,
-  "ర": 83,
-  "ల": 84,
-  "ళ": 85,
-  "వ": 86,
-  "శ": 87,
-  "ష": 88,
-  "స": 89,
-  "హ": 90,
-  "ా": 91,
-  "ి": 92,
-  "ీ": 93,
-  "ు": 94,
-  "ూ": 95,
-  "ృ": 96,
-  "ె": 97,
-  "ే": 98,
-  "ై": 99,
-  "ొ": 100,
-  "ో": 101,
-  "ౌ": 102,
-  "్": 103
+  "ढ": 23,
+  "ण": 24,
+  "त": 25,
+  "थ": 26,
+  "द": 27,
+  "ध": 28,
+  "न": 29,
+  "प": 30,
+  "फ": 31,
+  "ब": 32,
+  "भ": 33,
+  "म": 34,
+  "य": 35,
+  "र": 36,
+  "ल": 37,
+  "व": 38,
+  "श": 39,
+  "ष": 40,
+  "स": 41,
+  "ह": 42,
+  "़": 43,
+  "ा": 44,
+  "ि": 45,
+  "ी": 46,
+  "ु": 47,
+  "ू": 48,
+  "े": 49,
+  "ै": 50,
+  "ॉ": 51,
+  "ो": 52,
+  "ौ": 53,
+  "्": 54,
+  "ం": 55,
+  "అ": 56,
+  "ఆ": 57,
+  "ఇ": 58,
+  "ఈ": 59,
+  "ఉ": 60,
+  "ఎ": 61,
+  "ఏ": 62,
+  "ఒ": 63,
+  "క": 64,
+  "ఖ": 65,
+  "గ": 66,
+  "ఘ": 67,
+  "చ": 68,
+  "జ": 69,
+  "ట": 70,
+  "డ": 71,
+  "ణ": 72,
+  "త": 73,
+  "థ": 74,
+  "ద": 75,
+  "ధ": 76,
+  "న": 77,
+  "ప": 78,
+  "ఫ": 79,
+  "బ": 80,
+  "భ": 81,
+  "మ": 82,
+  "య": 83,
+  "ర": 84,
+  "ల": 85,
+  "ళ": 86,
+  "వ": 87,
+  "శ": 88,
+  "ష": 89,
+  "స": 90,
+  "హ": 91,
+  "ా": 92,
+  "ి": 93,
+  "ీ": 94,
+  "ు": 95,
+  "ూ": 96,
+  "ృ": 97,
+  "ె": 98,
+  "ే": 99,
+  "ై": 100,
+  "ొ": 101,
+  "ో": 102,
+  "ౌ": 103,
+  "్": 104
 }