Upload tokenizer
Browse files- tokenizer.json +2 -2
- tokenizer_config.json +32 -0
tokenizer.json
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4bfdbe18ca9d8be4e21fe6e8c46a19833b09560053bb9e9dbe71a9b6033e709b
|
| 3 |
+
size 33385589
|
tokenizer_config.json
CHANGED
|
@@ -51321,6 +51321,38 @@
|
|
| 51321 |
"rstrip": false,
|
| 51322 |
"single_word": false,
|
| 51323 |
"special": true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 51324 |
}
|
| 51325 |
},
|
| 51326 |
"boi_token": "<start_of_image>",
|
|
|
|
| 51321 |
"rstrip": false,
|
| 51322 |
"single_word": false,
|
| 51323 |
"special": true
|
| 51324 |
+
},
|
| 51325 |
+
"262145": {
|
| 51326 |
+
"content": "<think>",
|
| 51327 |
+
"lstrip": false,
|
| 51328 |
+
"normalized": true,
|
| 51329 |
+
"rstrip": false,
|
| 51330 |
+
"single_word": false,
|
| 51331 |
+
"special": false
|
| 51332 |
+
},
|
| 51333 |
+
"262146": {
|
| 51334 |
+
"content": "</think>",
|
| 51335 |
+
"lstrip": false,
|
| 51336 |
+
"normalized": true,
|
| 51337 |
+
"rstrip": false,
|
| 51338 |
+
"single_word": false,
|
| 51339 |
+
"special": false
|
| 51340 |
+
},
|
| 51341 |
+
"262147": {
|
| 51342 |
+
"content": "<reasoning_step>",
|
| 51343 |
+
"lstrip": false,
|
| 51344 |
+
"normalized": true,
|
| 51345 |
+
"rstrip": false,
|
| 51346 |
+
"single_word": false,
|
| 51347 |
+
"special": false
|
| 51348 |
+
},
|
| 51349 |
+
"262148": {
|
| 51350 |
+
"content": "</reasoning_step>",
|
| 51351 |
+
"lstrip": false,
|
| 51352 |
+
"normalized": true,
|
| 51353 |
+
"rstrip": false,
|
| 51354 |
+
"single_word": false,
|
| 51355 |
+
"special": false
|
| 51356 |
}
|
| 51357 |
},
|
| 51358 |
"boi_token": "<start_of_image>",
|