Commit
•
9e01eb8
1
Parent(s):
80feb2b
Clear the spaces rather than Gs rubbish
Browse files- tokenizer.json +1 -24
tokenizer.json
CHANGED
|
@@ -48298,30 +48298,7 @@
|
|
| 48298 |
"ibrated": 48235,
|
| 48299 |
"Ġfortified": 48236,
|
| 48300 |
"ferenced": 48237,
|
| 48301 |
-
"ĠOutcomes": 48238
|
| 48302 |
-
" ": 48239,
|
| 48303 |
-
" ": 48240,
|
| 48304 |
-
" ": 48241,
|
| 48305 |
-
" ": 48242,
|
| 48306 |
-
" ": 48243,
|
| 48307 |
-
" ": 48244,
|
| 48308 |
-
" ": 48245,
|
| 48309 |
-
" ": 48246,
|
| 48310 |
-
" ": 48247,
|
| 48311 |
-
" ": 48248,
|
| 48312 |
-
" ": 48249,
|
| 48313 |
-
" ": 48250,
|
| 48314 |
-
" ": 48251,
|
| 48315 |
-
" ": 48252,
|
| 48316 |
-
" ": 48253,
|
| 48317 |
-
" ": 48254,
|
| 48318 |
-
" ": 48255,
|
| 48319 |
-
" ": 48256,
|
| 48320 |
-
" ": 48257,
|
| 48321 |
-
" ": 48258,
|
| 48322 |
-
" ": 48259,
|
| 48323 |
-
" ": 48260,
|
| 48324 |
-
" ": 48261
|
| 48325 |
},
|
| 48326 |
"merges": [
|
| 48327 |
"Ġ Ġ",
|
|
|
|
| 48298 |
"ibrated": 48235,
|
| 48299 |
"Ġfortified": 48236,
|
| 48300 |
"ferenced": 48237,
|
| 48301 |
+
"ĠOutcomes": 48238
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48302 |
},
|
| 48303 |
"merges": [
|
| 48304 |
"Ġ Ġ",
|