Upload tokenizer

added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endofprompt|>": 100276,
+  "<|im_end|>": 100265,
+  "<|im_start|>": 100264
+}

merges.txt

@@ -99998,4 +99998,4 @@ _RECE IVED
 Ġmerc iless
 .Wait For
 Ġday care
-ĠCon veyor
+ĠCon veyor

special_tokens_map.json

@@ -1,5 +1,23 @@
 {
-  "bos_token": "<|endoftext|>",
-  "eos_token": "<|endoftext|>",
-  "unk_token": "<|endoftext|>"
-}
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -40,8 +40,8 @@
       "special": true
     },
     {
-      "id": 100276,
-      "content": "<|endofprompt|>",
+      "id": 100264,
+      "content": "<|im_start|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -49,8 +49,8 @@
       "special": true
     },
     {
-      "id": 100264,
-      "content": "<|im_start|>",
+      "id": 100265,
+      "content": "<|im_end|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -58,8 +58,8 @@
       "special": true
     },
     {
-      "id": 100265,
-      "content": "<|im_end|>",
+      "id": 100276,
+      "content": "<|endofprompt|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -102,6 +102,7 @@
     "end_of_word_suffix": "",
     "fuse_unk": false,
     "byte_fallback": false,
+    "ignore_merges": false,
     "vocab": {
       "!": 0,
       "\"": 1,
@@ -100363,9 +100364,9 @@
       "<|fim_prefix|>": 100258,
       "<|fim_middle|>": 100259,
       "<|fim_suffix|>": 100260,
-      "<|endofprompt|>": 100276,
       "<|im_start|>": 100264,
-      "<|im_end|>": 100265
+      "<|im_end|>": 100265,
+      "<|endofprompt|>": 100276
     },
     "merges": [
       "Ġ Ġ",

tokenizer_config.json

@@ -1,10 +1,68 @@
 {
   "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100264": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
   "bos_token": "<|endoftext|>",
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "model_max_length": 8192,
-  "tokenizer_class": "GPT2TokenizerFast",
-  "unk_token": "<|endoftext|>",
-  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"
-}
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}