Upload 2 files

tokenizer.json

@@ -5,7 +5,7 @@
     "strategy": "BatchLongest",
     "direction": "Right",
     "pad_to_multiple_of": null,
-    "pad_id": 128255,
+    "pad_id": 128257,
     "pad_type_id": 0,
     "pad_token": "<|im_end|>"
   },
@@ -2313,6 +2313,24 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 128256,
+      "content": "<|im_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 128257,
+      "content": "<|im_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,

tokenizer_config.json

@@ -2047,6 +2047,22 @@
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "128256": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128257": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
     }
   },
   "additional_special_tokens": [