Commit
•
4318da9
1
Parent(s):
82ddd5c
tokenizer improved
Browse files- tokenizer.json +3 -1
tokenizer.json
CHANGED
|
@@ -10846,6 +10846,7 @@
|
|
| 10846 |
"pre_tokenizer": {
|
| 10847 |
"type": "Metaspace",
|
| 10848 |
"replacement": "▁",
|
|
|
|
| 10849 |
"prepend_scheme": "never",
|
| 10850 |
"split": true
|
| 10851 |
},
|
|
@@ -10927,6 +10928,7 @@
|
|
| 10927 |
"decoder": {
|
| 10928 |
"type": "Metaspace",
|
| 10929 |
"replacement": "▁",
|
|
|
|
| 10930 |
"prepend_scheme": "never",
|
| 10931 |
"split": true
|
| 10932 |
},
|
|
@@ -210937,4 +210939,4 @@
|
|
| 210937 |
],
|
| 210938 |
"byte_fallback": false
|
| 210939 |
}
|
| 210940 |
-
}
|
|
|
|
| 10846 |
"pre_tokenizer": {
|
| 10847 |
"type": "Metaspace",
|
| 10848 |
"replacement": "▁",
|
| 10849 |
+
"add_prefix_space": false,
|
| 10850 |
"prepend_scheme": "never",
|
| 10851 |
"split": true
|
| 10852 |
},
|
|
|
|
| 10928 |
"decoder": {
|
| 10929 |
"type": "Metaspace",
|
| 10930 |
"replacement": "▁",
|
| 10931 |
+
"add_prefix_space": false,
|
| 10932 |
"prepend_scheme": "never",
|
| 10933 |
"split": true
|
| 10934 |
},
|
|
|
|
| 210939 |
],
|
| 210940 |
"byte_fallback": false
|
| 210941 |
}
|
| 210942 |
+
}
|