Upload tokenizer

tokenizer.json

@@ -159,21 +159,17 @@
   ],
   "normalizer": null,
   "pre_tokenizer": {
-    "type": "Sequence",
-    "pretokenizers": [
-      {
-        "type": "Digits",
-        "individual_digits": true
-      },
-      {
-        "type": "ByteLevel",
-        "add_prefix_space": false,
-        "trim_offsets": true,
-        "use_regex": true
-      }
-    ]
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "post_processor": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": false,
+    "use_regex": true
   },
-  "post_processor": null,
   "decoder": {
     "type": "ByteLevel",
     "add_prefix_space": true,
@@ -184,8 +180,8 @@
     "type": "BPE",
     "dropout": null,
     "unk_token": null,
-    "continuing_subword_prefix": null,
-    "end_of_word_suffix": null,
+    "continuing_subword_prefix": "",
+    "end_of_word_suffix": "",
     "fuse_unk": false,
     "byte_fallback": false,
     "ignore_merges": false,

tokenizer_config.json

@@ -1,4 +1,5 @@
 {
+  "add_bos_token": false,
   "add_prefix_space": false,
   "added_tokens_decoder": {
     "0": {
@@ -146,6 +147,7 @@
   "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
+  "errors": "replace",
   "model_max_length": 2048,
   "pad_token": "<|im_end|>",
   "tokenizer_class": "GPT2Tokenizer",