Upload tokenizer (#28)

- Upload tokenizer (2ca60a10925b961f7b3f37ed3646fd8991bac3f6)


Co-authored-by: Arthur Zucker <ArthurZ@users.noreply.huggingface.co>

merges.txt

@@ -1,4 +1,5 @@
 #version: 0.2
+Ġ t
 Ġ a
 Ġt h
 i n

special_tokens_map.json

@@ -111,22 +111,28 @@
   "bos_token": {
     "content": "<|endoftext|>",
     "lstrip": false,
-    "normalized": true,
+    "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "eos_token": {
     "content": "<|endoftext|>",
     "lstrip": false,
-    "normalized": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "<|endoftext|>",
   "unk_token": {
     "content": "<|endoftext|>",
     "lstrip": false,
-    "normalized": true,
+    "normalized": false,
     "rstrip": false,
     "single_word": false
   }

tokenizer.json

@@ -64848,6 +64848,7 @@
       "<|endoftext|>": 50257
     },
     "merges": [
+      "Ġ t",
       "Ġ a",
       "Ġt h",
       "i n",