Upload tokenizer

tokenizer.json

@@ -293,123 +293,148 @@
       "Ġthe": 258,
       "Ġi": 259,
       "Ġa": 260,
-      "en": 261,
-      "re": 262,
-      "Ġo": 263,
-      "si": 264,
-      "Ġis": 265,
-      "al": 266,
-      "ri": 267,
-      "at": 268,
-      "es": 269,
-      "le": 270,
-      "on": 271,
-      "Ġf": 272,
-      "Ġof": 273,
-      "nd": 274,
-      "an": 275,
-      "he": 276,
-      "Ġb": 277,
-      "Ġc": 278,
-      "Ġe": 279,
-      "Ġs": 280,
-      "Ġt": 281,
-      "Eu": 282,
-      "ion": 283,
-      "la": 284,
-      "mu": 285,
-      "om": 286,
-      "or": 287,
-      "ore": 288,
-      "se": 289,
-      "ten": 290,
-      "ĠT": 291,
-      "Ġsi": 292,
-      "ĠEu": 293,
-      "Ġand": 294,
-      "Ġfu": 295,
-      "mula": 296,
-      "ormula": 297,
-      "Ġsid": 298,
-      "'s": 299,
-      "ag": 300,
-      "et": 301,
-      "hy": 302,
-      "po": 303,
-      "qu": 304,
-      "use": 305,
-      "Ġ-": 306,
-      "Ġ2": 307,
-      "Ġl": 308,
-      "Ġw": 309,
-      "ĠÎ": 310,
-      "Ġth": 311,
-      "Ġre": 312,
-      "ther": 313,
-      "Ġin": 314,
-      "eng": 315,
-      "ent": 316,
-      "Ġother": 317,
-      "rig": 318,
-      "ler": 319,
-      "Ġformula": 320,
-      "ĠThe": 321,
-      "ĠEuler": 322,
-      "Ġsides": 323,
-      "It": 324,
-      "Py": 325,
-      "am": 326,
-      "are": 327,
-      "ct": 328,
-      "gle": 329,
-      "hi": 330,
-      "ht": 331,
-      "in": 332,
-      "li": 333,
-      "lat": 334,
-      "nct": 335,
-      "ple": 336,
-      "ry": 337,
-      "um": 338,
-      "wo": 339,
-      "Ġ+": 340,
-      "Ġ=": 341,
-      "Ġg": 342,
-      "Ġn": 343,
-      "Ġhy": 344,
-      "Ġrig": 345,
-      "ĠIt": 346,
-      "ĠPy": 347,
-      "thag": 348,
-      "Ġan": 349,
-      "ndam": 350,
-      "here": 351,
-      "Ġcom": 352,
-      "Ġex": 353,
-      "Ġsqu": 354,
-      "Ġtwo": 355,
-      "ions": 356,
-      "omet": 357,
-      "orem": 358,
-      "orean": 359,
-      "tenuse": 360,
-      "Ġfunct": 361,
-      "Ġfundam": 362,
-      "potenuse": 363,
-      "Ġleng": 364,
-      "Ġwhere": 365,
-      "Ġthat": 366,
-      "Ġrelat": 367,
-      "ental": 368,
-      "plex": 369,
-      "Ġhypotenuse": 370,
-      "Ġright": 371,
-      "ĠPythag": 372,
-      "Ġcomplex": 373,
-      "Ġsquare": 374,
-      "Ġfundamental": 375,
-      "Ġlength": 376,
-      "ĠPythagorean": 377
+      "ĠÎ": 261,
+      "en": 262,
+      "re": 263,
+      "Ġo": 264,
+      "si": 265,
+      "Ġis": 266,
+      "al": 267,
+      "ri": 268,
+      "at": 269,
+      "es": 270,
+      "le": 271,
+      "on": 272,
+      "ν": 273,
+      "Ïģ": 274,
+      "Ġf": 275,
+      "ĠÏ": 276,
+      "Ġof": 277,
+      "nd": 278,
+      "Ġ2": 279,
+      "ÏģÎ": 280,
+      "an": 281,
+      "he": 282,
+      "Ġb": 283,
+      "Ġc": 284,
+      "Ġe": 285,
+      "Ġs": 286,
+      "Ġt": 287,
+      "Eu": 288,
+      "ion": 289,
+      "la": 290,
+      "mu": 291,
+      "om": 292,
+      "or": 293,
+      "ore": 294,
+      "se": 295,
+      "ten": 296,
+      "ε": 297,
+      "ο": 298,
+      "ĠT": 299,
+      "Ġsi": 300,
+      "ĠEu": 301,
+      "Ġand": 302,
+      "Ġfu": 303,
+      "ĠÏĦ": 304,
+      "mula": 305,
+      "ormula": 306,
+      "Ġsid": 307,
+      "'s": 308,
+      "ag": 309,
+      "et": 310,
+      "hy": 311,
+      "po": 312,
+      "qu": 313,
+      "use": 314,
+      "¹Ï": 315,
+      "ί": 316,
+      "α": 317,
+      "η": 318,
+      "ÏĤ": 319,
+      "Ïħ": 320,
+      "Ġ+": 321,
+      "Ġ-": 322,
+      "Ġ=": 323,
+      "Ġl": 324,
+      "Ġth": 325,
+      "Ġre": 326,
+      "İν": 327,
+      "ther": 328,
+      "Ġin": 329,
+      "Ġγ": 330,
+      "eng": 331,
+      "ent": 332,
+      "Ġother": 333,
+      "rig": 334,
+      "ler": 335,
+      "Ġformula": 336,
+      "ĠThe": 337,
+      "ĠEuler": 338,
+      "Ġsides": 339,
+      "It": 340,
+      "Py": 341,
+      "am": 342,
+      "are": 343,
+      "ct": 344,
+      "gle": 345,
+      "hi": 346,
+      "ht": 347,
+      "in": 348,
+      "li": 349,
+      "lat": 350,
+      "nct": 351,
+      "ple": 352,
+      "ry": 353,
+      "um": 354,
+      "wo": 355,
+      "whe": 356,
+      "³Ï": 357,
+      "γÏ": 358,
+      "ÏĢ": 359,
+      "ÏĦ": 360,
+      "Ġg": 361,
+      "Ġn": 362,
+      "Ġhy": 363,
+      "Ġrig": 364,
+      "ĠIt": 365,
+      "ĠPy": 366,
+      "ĥη": 367,
+      "īν": 368,
+      "thag": 369,
+      "Ġan": 370,
+      "Ġα": 371,
+      "Ġβ": 372,
+      "Ġμ": 373,
+      "ndam": 374,
+      "ÏģιÏ": 375,
+      "Ġcom": 376,
+      "Ġex": 377,
+      "Ġsqu": 378,
+      "Ġtwo": 379,
+      "ions": 380,
+      "omet": 381,
+      "orem": 382,
+      "orean": 383,
+      "tenuse": 384,
+      "οÏħ": 385,
+      "Ġfunct": 386,
+      "Ġfundam": 387,
+      "potenuse": 388,
+      "Ġleng": 389,
+      "Ġthat": 390,
+      "Ġrelat": 391,
+      "ental": 392,
+      "plex": 393,
+      "where": 394,
+      "Ġhypotenuse": 395,
+      "Ġright": 396,
+      "ĠPythag": 397,
+      "Ġcomplex": 398,
+      "Ġsquare": 399,
+      "Ġfundamental": 400,
+      "Ġlength": 401,
+      "ĠPythagorean": 402
     },
     "merges": [
       "t h",
@@ -417,6 +442,7 @@
       "Ġ the",
       "Ġ i",
       "Ġ a",
+      "Ġ Î",
       "e n",
       "r e",
       "Ġ o",
@@ -428,9 +454,14 @@
       "e s",
       "l e",
       "o n",
+      "Î ½",
+      "Ï ģ",
       "Ġ f",
+      "Ġ Ï",
       "Ġo f",
       "n d",
+      "Ġ 2",
+      "Ïģ Î",
       "a n",
       "h e",
       "Ġ b",
@@ -447,11 +478,14 @@
       "o re",
       "s e",
       "t en",
+      "Î µ",
+      "Î ¿",
       "Ġ T",
       "Ġ si",
       "Ġ Eu",
       "Ġa nd",
       "Ġf u",
+      "ĠÏ Ħ",
       "mu la",
       "or mula",
       "Ġsi d",
@@ -462,15 +496,22 @@
       "p o",
       "q u",
       "u se",
+      "¹ Ï",
+      "Î ¯",
+      "Î ±",
+      "Î ·",
+      "Ï Ĥ",
+      "Ï ħ",
+      "Ġ +",
       "Ġ -",
-      "Ġ 2",
+      "Ġ =",
       "Ġ l",
-      "Ġ w",
-      "Ġ Î",
       "Ġ th",
       "Ġ re",
+      "İ Î½",
       "the r",
       "Ġi n",
+      "ĠÎ ³",
       "en g",
       "en t",
       "Ġo ther",
@@ -496,18 +537,26 @@
       "r y",
       "u m",
       "w o",
-      "Ġ +",
-      "Ġ =",
+      "w he",
+      "³ Ï",
+      "Î ³Ï",
+      "Ï Ģ",
+      "Ï Ħ",
       "Ġ g",
       "Ġ n",
       "Ġ hy",
       "Ġ rig",
       "Ġ It",
       "Ġ Py",
+      "ĥ η",
+      "ī ν",
       "th ag",
       "Ġa n",
+      "ĠÎ ±",
+      "ĠÎ ²",
+      "ĠÎ ¼",
       "nd am",
-      "he re",
+      "ÏģÎ ¹Ï",
       "Ġc om",
       "Ġe x",
       "Ġs qu",
@@ -517,15 +566,16 @@
       "ore m",
       "ore an",
       "ten use",
+      "ο Ïħ",
       "Ġfu nct",
       "Ġfu ndam",
       "po tenuse",
       "Ġl eng",
-      "Ġw here",
       "Ġth at",
       "Ġre lat",
       "ent al",
       "ple x",
+      "whe re",
       "Ġhy potenuse",
       "Ġrig ht",
       "ĠPy thag",