michelleyunun commited on
Commit
49837e5
1 Parent(s): 65df252

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer.json +418 -2
tokenizer.json CHANGED
@@ -1114,7 +1114,215 @@
1114
  "Ê·an": 1052,
1115
  "Ġal": 1053,
1116
  "ozoni": 1054,
1117
- "rab": 1055
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1118
  },
1119
  "merges": [
1120
  "a r",
@@ -2081,7 +2289,215 @@
2081
  "Ê· an",
2082
  "Ġa l",
2083
  "oz oni",
2084
- "r ab"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2085
  ]
2086
  }
2087
  }
 
1114
  "Ê·an": 1052,
1115
  "Ġal": 1053,
1116
  "ozoni": 1054,
1117
+ "rab": 1055,
1118
+ "ĠonoÄį": 1056,
1119
+ "ĠÊķArab": 1057,
1120
+ "Ġozuri": 1058,
1121
+ "ĠÊķArabuz": 1059,
1122
+ "usen": 1060,
1123
+ "ĠĦusen": 1061,
1124
+ "ĠroÉ£": 1062,
1125
+ "Ġpik": 1063,
1126
+ "ĠroÆĽik": 1064,
1127
+ "Ġdidur": 1065,
1128
+ "Ã¤ÆĽ": 1066,
1129
+ "ĠmoÄį": 1067,
1130
+ "ĠanuÆĽin": 1068,
1131
+ "ĠurÉ£izi": 1069,
1132
+ "go": 1070,
1133
+ "Ġer": 1071,
1134
+ "Ġbiž": 1072,
1135
+ "Ġgan": 1073,
1136
+ "moÆĽ": 1074,
1137
+ "eneš": 1075,
1138
+ "ĠheneÅ¡": 1076,
1139
+ "yoz": 1077,
1140
+ "Ġnay": 1078,
1141
+ "Ġbertenno": 1079,
1142
+ "Ġdidiw": 1080,
1143
+ "Ġhis": 1081,
1144
+ "Ġmaħorno": 1082,
1145
+ "Ġyiž": 1083,
1146
+ "ĠÅ¡uÅ¡": 1084,
1147
+ "Ġbayrno": 1085,
1148
+ "na": 1086,
1149
+ "ĠÅģ": 1087,
1150
+ "yoÅĤäy": 1088,
1151
+ "Ġaħoz": 1089,
1152
+ "Ġgurow": 1090,
1153
+ "Ġim": 1091,
1154
+ "ĠiduÉ£orno": 1092,
1155
+ "Ġyaq": 1093,
1156
+ "Ġruk": 1094,
1157
+ "Ġžeduq": 1095,
1158
+ "ĠbeÅĤiqan": 1096,
1159
+ "irxo": 1097,
1160
+ "yno": 1098,
1161
+ "äza": 1099,
1162
+ "og": 1100,
1163
+ "Ġyes": 1101,
1164
+ "Ġla": 1102,
1165
+ "uhal": 1103,
1166
+ "Ġceyä": 1104,
1167
+ "ĠÅĤäs": 1105,
1168
+ "ÅĤer": 1106,
1169
+ "ˤoÆĽ": 1107,
1170
+ "Ġyisiqor": 1108,
1171
+ "azir": 1109,
1172
+ "Ġaluk": 1110,
1173
+ "oxt": 1111,
1174
+ "Ġyaynosi": 1112,
1175
+ "Ġkidbed": 1113,
1176
+ "usa": 1114,
1177
+ "ĠiÅĤ": 1115,
1178
+ "Ġwasi": 1116,
1179
+ "Ġhoboy": 1117,
1180
+ "gi": 1118,
1181
+ "ĠyaÅĤza": 1119,
1182
+ "Ġmaduhal": 1120,
1183
+ "ara": 1121,
1184
+ "Ġmix": 1122,
1185
+ "Ġwazir": 1123,
1186
+ "Ġmaqˤ": 1124,
1187
+ "ĠyaÅĤruni": 1125,
1188
+ "ĠwˤaÅĤer": 1126,
1189
+ "ĠdaÊķ": 1127,
1190
+ "ĠyiÄįin": 1128,
1191
+ "Ġesun": 1129,
1192
+ "Ġhudu": 1130,
1193
+ "uza": 1131,
1194
+ "ĠbiÅ¡Ê·a": 1132,
1195
+ "sÆĽin": 1133,
1196
+ "ĠÉ£amas": 1134,
1197
+ "ĠmoÆĽaÅĤ": 1135,
1198
+ "Ġkecno": 1136,
1199
+ "ub": 1137,
1200
+ "ĠoÆĽixno": 1138,
1201
+ "ĠcaÆĽ": 1139,
1202
+ "ĠbiÄįin": 1140,
1203
+ "Ġyisis": 1141,
1204
+ "Ġamma": 1142,
1205
+ "ĠmiÆĽ": 1143,
1206
+ "ĠriguÆĽ": 1144,
1207
+ "Ġrek": 1145,
1208
+ "ĠmaÄį": 1146,
1209
+ "išat": 1147,
1210
+ "ĠAħ": 1148,
1211
+ "onÄį": 1149,
1212
+ "Ġdibir": 1150,
1213
+ "ĠzownÄį": 1151,
1214
+ "Ġħalt": 1152,
1215
+ "Ġezun": 1153,
1216
+ "Ġbabiwz": 1154,
1217
+ "gaħ": 1155,
1218
+ "ÅĤun": 1156,
1219
+ "ĠboqzaÆĽ": 1157,
1220
+ "Ġɣˤutkun": 1158,
1221
+ "ĠÃ¤ÆĽirugon": 1159,
1222
+ "âĢĿ.": 1160,
1223
+ "aÊķ": 1161,
1224
+ "Ġoqnosi": 1162,
1225
+ "ĠboÆĽik": 1163,
1226
+ "ĠyoÆĽ": 1164,
1227
+ "town": 1165,
1228
+ "ĠÊķomoy": 1166,
1229
+ "ĠdunyalÆĽ": 1167,
1230
+ "Ġrow": 1168,
1231
+ "Ġzeyä": 1169,
1232
+ "Ġreqen": 1170,
1233
+ "Ġyayn": 1171,
1234
+ "ixozo": 1172,
1235
+ "Ġbiz": 1173,
1236
+ "Ġkidbes": 1174,
1237
+ "ĠTawad": 1175,
1238
+ "ĠhawaÆĽ": 1176,
1239
+ "ĠÊķAl": 1177,
1240
+ "ĠHowži": 1178,
1241
+ "Ġkidbeq": 1179,
1242
+ "aɣˤ": 1180,
1243
+ "orolay": 1181,
1244
+ "Ġut": 1182,
1245
+ "Ġelus": 1183,
1246
+ "ĠmaÊķiÅ¡at": 1184,
1247
+ "ĠÅ¡uda": 1185,
1248
+ "Ġdowqor": 1186,
1249
+ "Ġrodanix": 1187,
1250
+ "Ġidurno": 1188,
1251
+ "Ġyez": 1189,
1252
+ "omceni": 1190,
1253
+ "ĠeÄįruni": 1191,
1254
+ "Ġpar": 1192,
1255
+ "iliÄį": 1193,
1256
+ "Ġoqxo": 1194,
1257
+ "ĠdÃ¤ÆĽ": 1195,
1258
+ "ĠbertenyoÅĤ": 1196,
1259
+ "ĠbeÄįedawni": 1197,
1260
+ "ʷˤa": 1198,
1261
+ "Ġele": 1199,
1262
+ "Ġkidbabi": 1200,
1263
+ "ubazi": 1201,
1264
+ "aru": 1202,
1265
+ "Ġbegir": 1203,
1266
+ "Ġnedur": 1204,
1267
+ "oxtur": 1205,
1268
+ "asno": 1206,
1269
+ "ĠzownÆĽin": 1207,
1270
+ "ĠiÅĤe": 1208,
1271
+ "xozo": 1209,
1272
+ "Ġdicegon": 1210,
1273
+ "ĠbiÅ¡a": 1211,
1274
+ "Ġhowlotow": 1212,
1275
+ "ĠdiÆĽin": 1213,
1276
+ "Ġbexurno": 1214,
1277
+ "xay": 1215,
1278
+ "izin": 1216,
1279
+ "Ġnagaħ": 1217,
1280
+ "Ġelor": 1218,
1281
+ "ĠBeq": 1219,
1282
+ "ˤur": 1220,
1283
+ "Ġat": 1221,
1284
+ "Ġesiws": 1222,
1285
+ "ĠÅĤox": 1223,
1286
+ "ĠkidbeÆĽ": 1224,
1287
+ "ĠħaÆĽanix": 1225,
1288
+ "xÆĽin": 1226,
1289
+ "Ġiq": 1227,
1290
+ "Ġqoq": 1228,
1291
+ "Ġqˤab": 1229,
1292
+ "ĠÆĽÃ¤xur": 1230,
1293
+ "Ġnaz": 1231,
1294
+ "ude": 1232,
1295
+ "Ġkoyä": 1233,
1296
+ "Ġbabiwqor": 1234,
1297
+ "unÄį": 1235,
1298
+ "Ġdäde": 1236,
1299
+ "ĠaÉ£i": 1237,
1300
+ "ĠħoÅ¡": 1238,
1301
+ "Ġhok": 1239,
1302
+ "ĠneÅĤqor": 1240,
1303
+ "Ġmežus": 1241,
1304
+ "ĠÅĤu": 1242,
1305
+ "ney": 1243,
1306
+ "sar": 1244,
1307
+ "ĠyoÉ£no": 1245,
1308
+ "ĠÊķUney": 1246,
1309
+ "ĠÊķUneyzat": 1247,
1310
+ "ixanu": 1248,
1311
+ "Ġxizaz": 1249,
1312
+ "ĠÄįant": 1250,
1313
+ "ĠAl": 1251,
1314
+ "Ġmuh": 1252,
1315
+ "Ġbigu": 1253,
1316
+ "ĠÆĽuza": 1254,
1317
+ "ĠŽedu": 1255,
1318
+ "ĠÆĽirba": 1256,
1319
+ "oh": 1257,
1320
+ "Ġžax": 1258,
1321
+ "Ġxanzo": 1259,
1322
+ "ĠeloÅ¡": 1260,
1323
+ "ĠbˤaÅĤ": 1261,
1324
+ "Ġdäx": 1262,
1325
+ "Ġpos": 1263
1326
  },
1327
  "merges": [
1328
  "a r",
 
2289
  "Ê· an",
2290
  "Ġa l",
2291
  "oz oni",
2292
+ "r ab",
2293
+ "Ġo noÄį",
2294
+ "ĠÊķA rab",
2295
+ "Ġozur i",
2296
+ "ĠÊķArab uz",
2297
+ "us en",
2298
+ "ĠĦ usen",
2299
+ "Ġro É£",
2300
+ "Ġp ik",
2301
+ "ĠroÆĽ ik",
2302
+ "Ġdid ur",
2303
+ "ä ÆĽ",
2304
+ "Ġm oÄį",
2305
+ "Ġanu ÆĽin",
2306
+ "ĠurÉ£ izi",
2307
+ "g o",
2308
+ "Ġe r",
2309
+ "Ġbi ž",
2310
+ "Ġg an",
2311
+ "m oÆĽ",
2312
+ "en eš",
2313
+ "Ġh eneÅ¡",
2314
+ "yo z",
2315
+ "Ġn ay",
2316
+ "Ġberten no",
2317
+ "Ġdid iw",
2318
+ "Ġh is",
2319
+ "Ġmaħ orno",
2320
+ "Ġyi ž",
2321
+ "ĠÅ¡ uÅ¡",
2322
+ "Ġbay rno",
2323
+ "n a",
2324
+ "ĠÅ ģ",
2325
+ "yoÅĤ äy",
2326
+ "Ġaħ oz",
2327
+ "Ġgur ow",
2328
+ "Ġi m",
2329
+ "ĠiduÉ£or no",
2330
+ "Ġy aq",
2331
+ "Ġr uk",
2332
+ "Ġžedu q",
2333
+ "ĠbeÅĤiq an",
2334
+ "ir xo",
2335
+ "y no",
2336
+ "ä za",
2337
+ "o g",
2338
+ "Ġy es",
2339
+ "Ġ la",
2340
+ "uh al",
2341
+ "Ġcey ä",
2342
+ "ĠÅĤä s",
2343
+ "ÅĤ er",
2344
+ "ˤ oÆĽ",
2345
+ "Ġyisi qor",
2346
+ "az ir",
2347
+ "Ġal uk",
2348
+ "ox t",
2349
+ "Ġyay nosi",
2350
+ "Ġkidb ed",
2351
+ "us a",
2352
+ "Ġi ÅĤ",
2353
+ "Ġw asi",
2354
+ "Ġhob oy",
2355
+ "g i",
2356
+ "ĠyaÅĤ za",
2357
+ "Ġmad uhal",
2358
+ "ar a",
2359
+ "Ġm ix",
2360
+ "Ġw azir",
2361
+ "Ġm aqˤ",
2362
+ "ĠyaÅĤ runi",
2363
+ "ĠwˤaÅĤ er",
2364
+ "Ġda Êķ",
2365
+ "ĠyiÄį in",
2366
+ "Ġes un",
2367
+ "Ġhud u",
2368
+ "u za",
2369
+ "ĠbiÅ¡ Ê·a",
2370
+ "s ÆĽin",
2371
+ "ĠÉ£ama s",
2372
+ "ĠmoÆĽ aÅĤ",
2373
+ "Ġkec no",
2374
+ "u b",
2375
+ "Ġ oÆĽixno",
2376
+ "Ġc aÆĽ",
2377
+ "ĠbiÄį in",
2378
+ "Ġyisi s",
2379
+ "Ġam ma",
2380
+ "Ġmi ÆĽ",
2381
+ "Ġrig uÆĽ",
2382
+ "Ġre k",
2383
+ "Ġma Äį",
2384
+ "iš at",
2385
+ "ĠA ħ",
2386
+ "on Äį",
2387
+ "Ġdib ir",
2388
+ "Ġzown Äį",
2389
+ "Ġħal t",
2390
+ "Ġez un",
2391
+ "Ġbabiw z",
2392
+ "g aħ",
2393
+ "ÅĤ un",
2394
+ "Ġboq zaÆĽ",
2395
+ "Ġɣˤutk un",
2396
+ "ĠÃ¤ÆĽiru gon",
2397
+ "âĢĿ .",
2398
+ "a Êķ",
2399
+ "Ġoq nosi",
2400
+ "ĠboÆĽ ik",
2401
+ "Ġy oÆĽ",
2402
+ "tow n",
2403
+ "ĠÊķom oy",
2404
+ "Ġdunyal ÆĽ",
2405
+ "Ġr ow",
2406
+ "Ġzey ä",
2407
+ "Ġreq en",
2408
+ "Ġy ayn",
2409
+ "ix ozo",
2410
+ "Ġbi z",
2411
+ "Ġkidb es",
2412
+ "ĠTaw ad",
2413
+ "Ġhaw aÆĽ",
2414
+ "ĠÊķA l",
2415
+ "ĠHow ži",
2416
+ "Ġkidb eq",
2417
+ "aɣ ˤ",
2418
+ "orol ay",
2419
+ "Ġu t",
2420
+ "Ġel us",
2421
+ "ĠmaÊķ iÅ¡at",
2422
+ "ĠÅ¡ uda",
2423
+ "Ġdow qor",
2424
+ "Ġrod anix",
2425
+ "Ġid urno",
2426
+ "Ġy ez",
2427
+ "om ceni",
2428
+ "ĠeÄį runi",
2429
+ "Ġp ar",
2430
+ "il iÄį",
2431
+ "Ġoq xo",
2432
+ "Ġdä ÆĽ",
2433
+ "Ġberten yoÅĤ",
2434
+ "ĠbeÄįed awni",
2435
+ "ʷ ˤa",
2436
+ "Ġel e",
2437
+ "Ġkidb abi",
2438
+ "ub azi",
2439
+ "ar u",
2440
+ "Ġbe gir",
2441
+ "Ġned ur",
2442
+ "oxt ur",
2443
+ "a sno",
2444
+ "Ġzown ÆĽin",
2445
+ "Ġi ÅĤe",
2446
+ "xo zo",
2447
+ "Ġdice gon",
2448
+ "ĠbiÅ¡ a",
2449
+ "Ġhowlo tow",
2450
+ "Ġdi ÆĽin",
2451
+ "Ġbex urno",
2452
+ "x ay",
2453
+ "iz in",
2454
+ "Ġna gaħ",
2455
+ "Ġel or",
2456
+ "ĠB eq",
2457
+ "ˤ ur",
2458
+ "Ġa t",
2459
+ "Ġesiw s",
2460
+ "ĠÅĤ ox",
2461
+ "Ġkidb eÆĽ",
2462
+ "ĠħaÆĽ anix",
2463
+ "x ÆĽin",
2464
+ "Ġi q",
2465
+ "Ġq oq",
2466
+ "Ġqˤ ab",
2467
+ "ĠÆĽÃ¤ xur",
2468
+ "Ġn az",
2469
+ "ud e",
2470
+ "Ġk oyä",
2471
+ "Ġbabiw qor",
2472
+ "un Äį",
2473
+ "Ġdä de",
2474
+ "ĠaÉ£ i",
2475
+ "Ġħo Å¡",
2476
+ "Ġh ok",
2477
+ "ĠneÅĤ qor",
2478
+ "Ġmež us",
2479
+ "ĠÅĤ u",
2480
+ "n ey",
2481
+ "s ar",
2482
+ "Ġyo É£no",
2483
+ "ĠÊķU ney",
2484
+ "ĠÊķUney zat",
2485
+ "ix anu",
2486
+ "Ġxiza z",
2487
+ "ĠÄįan t",
2488
+ "ĠA l",
2489
+ "Ġm uh",
2490
+ "Ġbi gu",
2491
+ "ĠÆĽ uza",
2492
+ "ĠŽ edu",
2493
+ "ĠÆĽir ba",
2494
+ "o h",
2495
+ "Ġža x",
2496
+ "Ġxan zo",
2497
+ "Ġelo Å¡",
2498
+ "Ġbˤ aÅĤ",
2499
+ "Ġdä x",
2500
+ "Ġp os"
2501
  ]
2502
  }
2503
  }