Upload tokenizer

special_tokens_map.json

@@ -12,6 +12,5 @@
     "normalized": false,
     "rstrip": false,
     "single_word": false
-  },
-  "pad_token": "<|eot_id|>"
+  }
 }

tokenizer.json

@@ -1,11 +1,6 @@
 {
   "version": "1.0",
-  "truncation": {
-    "direction": "Right",
-    "max_length": 2048,
-    "strategy": "LongestFirst",
-    "stride": 0
-  },
+  "truncation": null,
   "padding": null,
   "added_tokens": [
     {

tokenizer_config.json

@@ -2059,6 +2059,5 @@
     "attention_mask"
   ],
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "<|eot_id|>",
   "tokenizer_class": "PreTrainedTokenizerFast"
 }