Upload 2 files

special_tokens_map.json

@@ -0,0 +1,4 @@
+{
+  "eos_token": "X",
+  "pad_token": "X"
+}

tokenizer_config.json

@@ -0,0 +1,18 @@
+{
+  "added_tokens_decoder": {},
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenizer.ByteTokenizer",
+      null
+    ]
+  },
+  "byte_level": true,
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "X",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "X",
+  "padding_side": "left",
+  "tokenizer_class": "ByteTokenizer",
+  "truncation_side": "left",
+  "trust_remote_code": true
+}