Upload tokenizer

tokenizer.json

@@ -275,30 +275,10 @@
     "use_regex": true
   },
   "post_processor": {
-    "type": "TemplateProcessing",
-    "single": [
-      {
-        "Sequence": {
-          "id": "A",
-          "type_id": 0
-        }
-      }
-    ],
-    "pair": [
-      {
-        "Sequence": {
-          "id": "A",
-          "type_id": 0
-        }
-      },
-      {
-        "Sequence": {
-          "id": "B",
-          "type_id": 1
-        }
-      }
-    ],
-    "special_tokens": {}
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": true,
+    "use_regex": true
   },
   "decoder": {
     "type": "ByteLevel",
@@ -314,7 +294,6 @@
     "end_of_word_suffix": null,
     "fuse_unk": false,
     "byte_fallback": false,
-    "ignore_merges": false,
     "vocab": {
       "<|endoftext|>": 0,
       "<|padding|>": 1,

tokenizer_config.json

@@ -1,6 +1,4 @@
 {
-  "add_bos_token": false,
-  "add_eos_token": false,
   "add_prefix_space": false,
   "added_tokens_decoder": {
     "0": {