Upload tokenizer

merges.txt

@@ -1,4 +1,4 @@
-#version: 0.2 - Trained by `huggingface/tokenizers`
+#version: 0.2
 Ġ t
 Ġ a
 h e

tokenizer.json

@@ -87,6 +87,7 @@
     "continuing_subword_prefix": "",
     "end_of_word_suffix": "",
     "fuse_unk": false,
+    "byte_fallback": false,
     "vocab": {
       "<s>": 0,
       "<pad>": 1,

tokenizer_config.json

@@ -1,15 +1,61 @@
 {
   "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50264": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": true,
   "cls_token": "<s>",
   "eos_token": "</s>",
   "errors": "replace",
   "mask_token": "<mask>",
+  "max_length": 64,
   "model_max_length": 512,
   "pad_token": "<pad>",
   "sep_token": "</s>",
+  "stride": 0,
   "tokenizer_class": "RobertaTokenizer",
   "trim_offsets": true,
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "<unk>"
 }