Commit
·
bcb111b
1
Parent(s):
9d9fe6f
tokenizer improved
Browse files- tokenizer.json +3 -1
tokenizer.json
CHANGED
|
@@ -91,6 +91,7 @@
|
|
| 91 |
"pre_tokenizer": {
|
| 92 |
"type": "Metaspace",
|
| 93 |
"replacement": "▁",
|
|
|
|
| 94 |
"prepend_scheme": "never",
|
| 95 |
"split": true
|
| 96 |
},
|
|
@@ -151,6 +152,7 @@
|
|
| 151 |
"decoder": {
|
| 152 |
"type": "Metaspace",
|
| 153 |
"replacement": "▁",
|
|
|
|
| 154 |
"prepend_scheme": "never",
|
| 155 |
"split": true
|
| 156 |
},
|
|
@@ -128161,4 +128163,4 @@
|
|
| 128161 |
],
|
| 128162 |
"byte_fallback": false
|
| 128163 |
}
|
| 128164 |
-
}
|
|
|
|
| 91 |
"pre_tokenizer": {
|
| 92 |
"type": "Metaspace",
|
| 93 |
"replacement": "▁",
|
| 94 |
+
"add_prefix_space": false,
|
| 95 |
"prepend_scheme": "never",
|
| 96 |
"split": true
|
| 97 |
},
|
|
|
|
| 152 |
"decoder": {
|
| 153 |
"type": "Metaspace",
|
| 154 |
"replacement": "▁",
|
| 155 |
+
"add_prefix_space": false,
|
| 156 |
"prepend_scheme": "never",
|
| 157 |
"split": true
|
| 158 |
},
|
|
|
|
| 128163 |
],
|
| 128164 |
"byte_fallback": false
|
| 128165 |
}
|
| 128166 |
+
}
|