Upload tokenizer

added_tokens.json

@@ -1,6 +1,6 @@
 {
-  "</s>": 128,
-  "<pad>": 130,
-  "<unk>": 129,
-  "[PAD]": 127
+  "</s>": 112,
+  "<pad>": 114,
+  "<s>": 111,
+  "<unk>": 113
 }

tokenizer_config.json

@@ -1,6 +1,14 @@
 {
   "added_tokens_decoder": {
-    "127": {
+    "109": {
+      "content": "[UNK]",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "110": {
       "content": "[PAD]",
       "lstrip": true,
       "normalized": false,
@@ -8,7 +16,15 @@
       "single_word": false,
       "special": false
     },
-    "128": {
+    "111": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "112": {
       "content": "</s>",
       "lstrip": false,
       "normalized": false,
@@ -16,7 +32,7 @@
       "single_word": false,
       "special": true
     },
-    "129": {
+    "113": {
       "content": "<unk>",
       "lstrip": false,
       "normalized": false,
@@ -24,7 +40,7 @@
       "single_word": false,
       "special": true
     },
-    "130": {
+    "114": {
       "content": "<pad>",
       "lstrip": false,
       "normalized": false,

vocab.json

@@ -1,129 +1,113 @@
 {
-  "\t": 0,
-  "\n": 1,
-  "!": 3,
-  "\"": 4,
-  "%": 5,
-  "'": 6,
-  ",": 7,
-  "-": 8,
-  ".": 9,
-  "/": 10,
-  "0": 11,
-  "1": 12,
-  "2": 13,
-  "3": 14,
-  "4": 15,
-  "5": 16,
-  "6": 17,
-  "7": 18,
-  "8": 19,
-  "9": 20,
-  ":": 21,
-  "B": 22,
-  "L": 23,
-  "T": 24,
-  "[PAD]": 127,
-  "[UNK]": 127,
-  "a": 25,
-  "b": 26,
-  "c": 27,
-  "d": 28,
-  "e": 29,
-  "f": 30,
-  "g": 31,
-  "i": 32,
-  "l": 33,
-  "m": 34,
-  "n": 35,
-  "o": 36,
-  "p": 37,
-  "r": 38,
-  "s": 39,
-  "t": 40,
-  "u": 41,
-  "v": 42,
-  "w": 43,
-  "z": 44,
-  "|": 2,
-  "“": 45,
-  "”": 46,
-  "।": 47,
-  "ঁ": 48,
-  "ং": 49,
-  "ঃ": 50,
-  "অ": 51,
-  "আ": 52,
-  "ই": 53,
-  "ঈ": 54,
-  "উ": 55,
-  "ঋ": 56,
-  "এ": 57,
-  "ঐ": 58,
-  "ও": 59,
-  "ঔ": 60,
-  "ক": 61,
-  "খ": 62,
-  "গ": 63,
-  "ঘ": 64,
-  "ঙ": 65,
-  "চ": 66,
-  "ছ": 67,
-  "জ": 68,
-  "ঝ": 69,
-  "ঞ": 70,
-  "ট": 71,
-  "ঠ": 72,
-  "ড": 73,
-  "ঢ": 74,
-  "ণ": 75,
-  "ত": 76,
-  "থ": 77,
-  "দ": 78,
-  "ধ": 79,
-  "ন": 80,
-  "প": 81,
-  "ফ": 82,
-  "ব": 83,
-  "ভ": 84,
-  "ম": 85,
-  "য": 86,
-  "র": 87,
-  "ল": 88,
-  "শ": 89,
-  "ষ": 90,
-  "স": 91,
-  "হ": 92,
-  "়": 93,
-  "া": 94,
-  "ি": 95,
-  "ী": 96,
-  "ু": 97,
-  "ূ": 98,
-  "ৃ": 99,
-  "ে": 100,
-  "ৈ": 101,
-  "ো": 102,
-  "ৌ": 103,
-  "্": 104,
-  "ৎ": 105,
-  "ড়": 106,
-  "য়": 107,
-  "০": 108,
-  "১": 109,
-  "২": 110,
-  "৩": 111,
-  "৪": 112,
-  "৫": 113,
-  "৬": 114,
-  "৭": 115,
-  "৮": 116,
-  "৯": 117,
-  "‌": 118,
-  "‍": 119,
-  "‘": 120,
-  "’": 121,
-  "“": 122,
-  "”": 123,
-  "…": 124
+  "!": 1,
+  "%": 2,
+  "'": 3,
+  "-": 4,
+  ".": 5,
+  "/": 6,
+  "0": 7,
+  "1": 8,
+  "2": 9,
+  "4": 10,
+  ":": 11,
+  "B": 12,
+  "[PAD]": 110,
+  "[UNK]": 109,
+  "a": 13,
+  "b": 14,
+  "c": 15,
+  "d": 16,
+  "f": 17,
+  "g": 18,
+  "i": 19,
+  "l": 20,
+  "n": 21,
+  "o": 22,
+  "p": 23,
+  "r": 24,
+  "s": 25,
+  "t": 26,
+  "u": 27,
+  "v": 28,
+  "w": 29,
+  "z": 30,
+  "|": 0,
+  "“": 31,
+  "”": 32,
+  "।": 33,
+  "ঁ": 34,
+  "ং": 35,
+  "ঃ": 36,
+  "অ": 37,
+  "আ": 38,
+  "ই": 39,
+  "ঈ": 40,
+  "উ": 41,
+  "ঋ": 42,
+  "এ": 43,
+  "ঐ": 44,
+  "ও": 45,
+  "ঔ": 46,
+  "ক": 47,
+  "খ": 48,
+  "গ": 49,
+  "ঘ": 50,
+  "ঙ": 51,
+  "চ": 52,
+  "ছ": 53,
+  "জ": 54,
+  "ঝ": 55,
+  "ঞ": 56,
+  "ট": 57,
+  "ঠ": 58,
+  "ড": 59,
+  "ঢ": 60,
+  "ণ": 61,
+  "ত": 62,
+  "থ": 63,
+  "দ": 64,
+  "ধ": 65,
+  "ন": 66,
+  "প": 67,
+  "ফ": 68,
+  "ব": 69,
+  "ভ": 70,
+  "ম": 71,
+  "য": 72,
+  "র": 73,
+  "ল": 74,
+  "শ": 75,
+  "ষ": 76,
+  "স": 77,
+  "হ": 78,
+  "়": 79,
+  "া": 80,
+  "ি": 81,
+  "ী": 82,
+  "ু": 83,
+  "ূ": 84,
+  "ৃ": 85,
+  "ে": 86,
+  "ৈ": 87,
+  "ো": 88,
+  "ৌ": 89,
+  "্": 90,
+  "ৎ": 91,
+  "ড়": 92,
+  "য়": 93,
+  "০": 94,
+  "১": 95,
+  "২": 96,
+  "৩": 97,
+  "৪": 98,
+  "৫": 99,
+  "৬": 100,
+  "৭": 101,
+  "৮": 102,
+  "৯": 103,
+  "‌": 104,
+  "‍": 105,
+  "‘": 106,
+  "’": 107,
+  "…": 108
 }