fix: fix merges in tokenizer
Browse files
moonshine_tiny_tokenizer.json
CHANGED
@@ -15703,7 +15703,7 @@
|
|
15703 |
"\"?": 8652,
|
15704 |
"▁>>>": 8653,
|
15705 |
"Que": 8654,
|
15706 |
-
"
|
15707 |
"▁plain": 8656,
|
15708 |
"ativa": 8657,
|
15709 |
"ocker": 8658,
|
@@ -24939,7 +24939,7 @@
|
|
24939 |
"▁farm": 17888,
|
24940 |
"▁rôle": 17889,
|
24941 |
"▁статьи": 17890,
|
24942 |
-
"
|
24943 |
"subfigure": 17892,
|
24944 |
"èces": 17893,
|
24945 |
"ziel": 17894,
|
@@ -27294,7 +27294,7 @@
|
|
27294 |
"▁gcc": 20243,
|
27295 |
"▁scène": 20244,
|
27296 |
"Navigation": 20245,
|
27297 |
-
"▁
|
27298 |
"▁кан": 20247,
|
27299 |
"▁towns": 20248,
|
27300 |
"Username": 20249,
|
@@ -37129,7 +37129,7 @@
|
|
37129 |
"æ": 30078,
|
37130 |
"њ": 30079,
|
37131 |
" ": 30080,
|
37132 |
-
"
|
37133 |
"Э": 30082,
|
37134 |
"ë": 30083,
|
37135 |
"õ": 30084,
|
@@ -57967,7 +57967,7 @@
|
|
57967 |
"▁>> >",
|
57968 |
"Qu e",
|
57969 |
"Q ue",
|
57970 |
-
"
|
57971 |
"▁p lain",
|
57972 |
"▁pl ain",
|
57973 |
"▁pla in",
|
@@ -77192,7 +77192,7 @@
|
|
77192 |
"▁fa rm",
|
77193 |
"▁r ôle",
|
77194 |
"▁стать и",
|
77195 |
-
"
|
77196 |
"sub figure",
|
77197 |
"èce s",
|
77198 |
"è ces",
|
@@ -81855,7 +81855,7 @@
|
|
81855 |
"▁ gcc",
|
81856 |
"▁sc ène",
|
81857 |
"N avigation",
|
81858 |
-
"▁
|
81859 |
"▁к ан",
|
81860 |
"▁ка н",
|
81861 |
"▁ кан",
|
@@ -100301,4 +100301,4 @@
|
|
100301 |
"▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
|
100302 |
]
|
100303 |
}
|
100304 |
-
}
|
|
|
15703 |
"\"?": 8652,
|
15704 |
"▁>>>": 8653,
|
15705 |
"Que": 8654,
|
15706 |
+
" ": 8655,
|
15707 |
"▁plain": 8656,
|
15708 |
"ativa": 8657,
|
15709 |
"ocker": 8658,
|
|
|
24939 |
"▁farm": 17888,
|
24940 |
"▁rôle": 17889,
|
24941 |
"▁статьи": 17890,
|
24942 |
+
" ": 17891,
|
24943 |
"subfigure": 17892,
|
24944 |
"èces": 17893,
|
24945 |
"ziel": 17894,
|
|
|
27294 |
"▁gcc": 20243,
|
27295 |
"▁scène": 20244,
|
27296 |
"Navigation": 20245,
|
27297 |
+
"▁ ": 20246,
|
27298 |
"▁кан": 20247,
|
27299 |
"▁towns": 20248,
|
27300 |
"Username": 20249,
|
|
|
37129 |
"æ": 30078,
|
37130 |
"њ": 30079,
|
37131 |
" ": 30080,
|
37132 |
+
" ": 30081,
|
37133 |
"Э": 30082,
|
37134 |
"ë": 30083,
|
37135 |
"õ": 30084,
|
|
|
57967 |
"▁>> >",
|
57968 |
"Qu e",
|
57969 |
"Q ue",
|
57970 |
+
" ",
|
57971 |
"▁p lain",
|
57972 |
"▁pl ain",
|
57973 |
"▁pla in",
|
|
|
77192 |
"▁fa rm",
|
77193 |
"▁r ôle",
|
77194 |
"▁стать и",
|
77195 |
+
" ",
|
77196 |
"sub figure",
|
77197 |
"èce s",
|
77198 |
"è ces",
|
|
|
81855 |
"▁ gcc",
|
81856 |
"▁sc ène",
|
81857 |
"N avigation",
|
81858 |
+
"▁ ",
|
81859 |
"▁к ан",
|
81860 |
"▁ка н",
|
81861 |
"▁ кан",
|
|
|
100301 |
"▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
|
100302 |
]
|
100303 |
}
|
100304 |
+
}
|