Trained with Unsloth
Browse filesUpload model trained with Unsloth 2x faster
- tokenizer.json +6 -12
tokenizer.json
CHANGED
|
@@ -49,19 +49,13 @@
|
|
| 49 |
"special": true
|
| 50 |
}
|
| 51 |
],
|
| 52 |
-
"normalizer":
|
| 53 |
-
|
| 54 |
-
"
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
"String": " "
|
| 59 |
-
},
|
| 60 |
-
"content": "▁"
|
| 61 |
-
}
|
| 62 |
-
]
|
| 63 |
},
|
| 64 |
-
"pre_tokenizer": null,
|
| 65 |
"post_processor": {
|
| 66 |
"type": "TemplateProcessing",
|
| 67 |
"single": [
|
|
|
|
| 49 |
"special": true
|
| 50 |
}
|
| 51 |
],
|
| 52 |
+
"normalizer": null,
|
| 53 |
+
"pre_tokenizer": {
|
| 54 |
+
"type": "Metaspace",
|
| 55 |
+
"replacement": "▁",
|
| 56 |
+
"prepend_scheme": "first",
|
| 57 |
+
"split": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 58 |
},
|
|
|
|
| 59 |
"post_processor": {
|
| 60 |
"type": "TemplateProcessing",
|
| 61 |
"single": [
|