Upload tokenizer.json

tokenizer.json

@@ -21,6 +21,12 @@
           "id": "A",
           "type_id": 0
         }
+      },
+      {
+        "SpecialToken": {
+          "id": "<eos>",
+          "type_id": 0
+        }
       }
     ],
     "pair": [
@@ -46,6 +52,15 @@
         "tokens": [
           "<cls>"
         ]
+      },
+      "<eos>": {
+        "id": "<eos>",
+        "ids": [
+          2
+        ],
+        "tokens": [
+          "<eos>"
+        ]
       }
     }
   },