add tokenizer

added_tokens.json

@@ -1 +1 @@
-{"\t\t\t\t\t": 50301, "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t": 50289, "\t\t\t": 50303, "\t\t\t\t\t\t\t\t\t\t": 50296, "         ": 50279, "                         ": 50263, "\t\t\t\t\t\t\t": 50299, "                              ": 50258, "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t": 50287, "\t\t\t\t\t\t\t\t\t\t\t\t": 50294, "\t\t\t\t\t\t\t\t\t\t\t": 50295, "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t": 50290, "                ": 50272, "                   ": 50269, "                           ": 50261, "      ": 50282, "\t\t\t\t\t\t\t\t\t\t\t\t\t": 50293, "                    ": 50268, "             ": 50275, "\t\t\t\t": 50302, "            ": 50276, "                            ": 50260, "\t\t": 50304, "               ": 50273, "                             ": 50259, "                     ": 50267, "           ": 50277, "                               ": 50257, "                  ": 50270, "        ": 50280, "\t\t\t\t\t\t\t\t\t": 50297, "\t\t\t\t\t\t\t\t": 50298, "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t": 50288, "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t": 50291, "     ": 50283, "                 ": 50271, "                       ": 50265, "                      ": 50266, "                        ": 50264, "\t\t\t\t\t\t": 50300, "   ": 50285, "              ": 50274, "                          ": 50262, "    ": 50284, "  ": 50286, "       ": 50281, "\t\t\t\t\t\t\t\t\t\t\t\t\t\t": 50292, "          ": 50278}
+{"     ": 50283, "                 ": 50271, "\t\t": 50304, "\t\t\t\t\t\t\t\t\t": 50297, "                        ": 50264, "             ": 50275, "\t\t\t\t": 50302, "    ": 50284, "                           ": 50261, "                  ": 50270, "              ": 50274, "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t": 50288, "\t\t\t\t\t\t\t": 50299, "\t\t\t\t\t\t\t\t\t\t\t\t": 50294, "        ": 50280, "                         ": 50263, "\t\t\t\t\t": 50301, "                    ": 50268, "                   ": 50269, "                               ": 50257, "                             ": 50259, "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t": 50287, "      ": 50282, "       ": 50281, "                            ": 50260, "\t\t\t": 50303, "\t\t\t\t\t\t\t\t\t\t": 50296, "                ": 50272, "         ": 50279, "<pad>": 50305, "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t": 50291, "  ": 50286, "\t\t\t\t\t\t\t\t\t\t\t": 50295, "                      ": 50266, "\t\t\t\t\t\t\t\t\t\t\t\t\t\t": 50292, "                     ": 50267, "               ": 50273, "\t\t\t\t\t\t": 50300, "   ": 50285, "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t": 50290, "          ": 50278, "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t": 50289, "                          ": 50262, "                              ": 50258, "                       ": 50265, "            ": 50276, "\t\t\t\t\t\t\t\t": 50298, "\t\t\t\t\t\t\t\t\t\t\t\t\t": 50293, "           ": 50277}

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>"}
+{"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>", "pad_token": "<pad>"}

tokenizer.json

@@ -443,6 +443,15 @@
       "rstrip": false,
       "normalized": true,
       "special": false
+    },
+    {
+      "id": 50305,
+      "content": "<pad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,