Upload tokenizer

special_tokens_map.json

@@ -1,11 +1,4 @@
 {
-  "additional_special_tokens": [
-    "[PAD]",
-    "<s>",
-    "</s>",
-    "<unk>",
-    "[PAD]"
-  ],
   "bos_token": "<s>",
   "eos_token": "</s>",
   "pad_token": "[PAD]",

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9051199beacf71b5afaf6058198951c587ac0d6702f3871b122d8bf5660d72b7
-size 14501256
+oid sha256:1785bcb6a31ceabdec1237f1c8330b52b368cee31213ab9eab322f1c4e399485
+size 14501099

tokenizer_config.json

@@ -1,12 +1,11 @@
 {
   "add_prefix_space": false,
   "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
-  "max_length": 512,
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "<pad>",
   "padding_side": "left",
-  "special_tokens_map_file": null,
   "tokenizer_class": "BloomTokenizer",
   "unk_token": "<unk>"
 }