Upload tokenizer

special_tokens_map.json

@@ -1 +1,3 @@
-{}
+{
+  "pad_token": "<pad>"
+}

tokenizer.json

@@ -1,6 +1,11 @@
 {
   "version": "1.0",
-  "truncation": null,
+  "truncation": {
+    "direction": "Right",
+    "max_length": 512,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
   "padding": null,
   "added_tokens": [
     {
@@ -58,7 +63,17 @@
     "prepend_scheme": "always",
     "split": true
   },
-  "post_processor": null,
+  "post_processor": {
+    "type": "BertProcessing",
+    "sep": [
+      "</s>",
+      2
+    ],
+    "cls": [
+      "<s>",
+      1
+    ]
+  },
   "decoder": {
     "type": "Metaspace",
     "replacement": "▁",

tokenizer_config.json

@@ -42,6 +42,12 @@
     }
   },
   "clean_up_tokenization_spaces": true,
-  "model_max_length": 1000000000000000019884624838656,
-  "tokenizer_class": "PreTrainedTokenizerFast"
+  "max_length": 512,
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "stride": 0,
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "truncation": true,
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first"
 }