Upload tokenizer.json

tokenizer.json

@@ -99,6 +99,15 @@
                 "tokens": [
                     "<s>"
                 ]
+            },
+            "</s>": {
+                "id": "</s>",
+                "ids": [
+                    2
+                ],
+                "tokens": [
+                    "</s>"
+                ]
             }
         }
     },
@@ -129,11 +138,11 @@
     "model": {
         "type": "BPE",
         "dropout": null,
-        "unk_token": "<unk>",
+        "unk_token": null,
         "continuing_subword_prefix": null,
         "end_of_word_suffix": null,
-        "fuse_unk": true,
-        "byte_fallback": true,
+        "fuse_unk": false,
+        "byte_fallback": false,
         "vocab": {
             "<unk>": 0,
             "<s>": 1,