Update tokenizer.json

tokenizer.json

@@ -69,12 +69,12 @@
   ],
   "normalizer": null,
   "pre_tokenizer": {
-    "type": "Whitespace"
+    "type": "WhitespaceSplit"
   },
   "post_processor": null,
   "decoder": null,
   "model": {
-    "type": "BPE",
+    "type": "WordLevel",
     "dropout": null,
     "unk_token": "<unk>",
     "continuing_subword_prefix": null,
@@ -90,245 +90,245 @@
       "<sep>": 5,
       "<mask>": 6,
       "0.000": 7,
-"0.0025": 8,
-"-0.0025": 9,
-"0.0050": 10,
-"-0.0050": 11,
-"0.0075": 12,
-"-0.0075": 13,
-"0.0100": 16,
-"-0.0100": 17,
-"0.0125": 18,
-"-0.0125": 19,
-"0.0150": 20,
-"-0.0150": 21,
-"0.0175": 22,
-"-0.0175": 23,
-"0.0200": 24,
-"-0.0200": 25,
-"0.0225": 26,
-"-0.0225": 27,
-"0.0250": 28,
-"-0.0250": 29,
-"0.0275": 30,
-"-0.0275": 31,
-"0.0300": 32,
-"-0.0300": 33,
-"0.0325": 34,
-"-0.0325": 35,
-"0.0350": 36,
-"-0.0350": 37,
-"0.0375": 38,
-"-0.0375": 39,
-"0.0400": 40,
-"-0.0400": 41,
-"0.0425": 42,
-"-0.0425": 43,
-"0.0450": 44,
-"-0.0450": 45,
-"0.0475": 46,
-"-0.0475": 47,
-"0.0500": 48,
-"-0.0500": 49,
-"0.0525": 50,
-"-0.0525": 51,
-"0.0550": 52,
-"-0.0550": 53,
-"0.0575": 54,
-"-0.0575": 55,
-"0.0600": 56,
-"-0.0600": 57,
-"0.0625": 58,
-"-0.0625": 59,
-"0.0650": 60,
-"-0.0650": 61,
-"0.0675": 62,
-"-0.0675": 63,
-"0.0700": 64,
-"-0.0700": 65,
-"0.0725": 66,
-"-0.0725": 67,
-"0.0750": 68,
-"-0.0750": 69,
-"0.0775": 70,
-"-0.0775": 71,
-"0.0800": 72,
-"-0.0800": 73,
-"0.0825": 74,
-"-0.0825": 75,
-"0.0850": 76,
-"-0.0850": 77,
-"0.0875": 78,
-"-0.0875": 79,
-"0.0900": 80,
-"-0.0900": 81,
-"0.0925": 82,
-"-0.0925": 83,
-"0.0950": 84,
-"-0.0950": 85,
-"0.0975": 86,
-"-0.0975": 87,
-"0.1000": 88,
-"-0.1000": 89,
-"0.1025": 90,
-"-0.1025": 91,
-"0.1050": 92,
-"-0.1050": 93,
-"0.1075": 94,
-"-0.1075": 95,
-"0.1100": 96,
-"-0.1100": 97,
-"0.1125": 98,
-"-0.1125": 99,
-"0.1150": 100,
-"-0.1150": 101,
-"0.1175": 102,
-"-0.1175": 103,
-"0.1200": 104,
-"-0.1200": 105,
-"0.1225": 106,
-"-0.1225": 107,
-"0.1250": 108,
-"-0.1250": 109,
-"0.1275": 110,
-"-0.1275": 111,
-"0.1300": 112,
-"-0.1300": 113,
-"0.1325": 114,
-"-0.1325": 115,
-"0.1350": 116,
-"-0.1350": 117,
-"0.1375": 118,
-"-0.1375": 119,
-"0.1400": 120,
-"-0.1400": 121,
-"0.1425": 122,
-"-0.1425": 123,
-"0.1450": 124,
-"-0.1450": 125,
-"0.1475": 126,
-"-0.1475": 127,
-"0.1500": 128,
-"-0.1500": 129,
-"0.1525": 130,
-"-0.1525": 131,
-"0.1550": 132,
-"-0.1550": 133,
-"0.1575": 134,
-"-0.1575": 135,
-"0.1600": 136,
-"-0.1600": 137,
-"0.1625": 138,
-"-0.1625": 139,
-"0.1650": 140,
-"-0.1650": 141,
-"0.1675": 142,
-"-0.1675": 143,
-"0.1700": 144,
-"-0.1700": 145,
-"0.1725": 146,
-"-0.1725": 147,
-"0.1750": 148,
-"-0.1750": 149,
-"0.1775": 150,
-"-0.1775": 151,
-"0.1800": 152,
-"-0.1800": 153,
-"0.1825": 154,
-"-0.1825": 155,
-"0.1850": 156,
-"-0.1850": 157,
-"0.1875": 158,
-"-0.1875": 159,
-"0.1900": 160,
-"-0.1900": 161,
-"0.1925": 162,
-"-0.1925": 163,
-"0.1950": 164,
-"-0.1950": 165,
-"0.1975": 166,
-"-0.1975": 167,
-"0.2000": 168,
-"-0.2000": 169,
-"0.2025": 170,
-"-0.2025": 171,
-"0.2050": 172,
-"-0.2050": 173,
-"0.2075": 174,
-"-0.2075": 175,
-"0.2100": 176,
-"-0.2100": 177,
-"0.2125": 178,
-"-0.2125": 179,
-"0.2150": 180,
-"-0.2150": 181,
-"0.2175": 182,
-"-0.2175": 183,
-"0.2200": 184,
-"-0.2200": 185,
-"0.2225": 186,
-"-0.2225": 187,
-"0.2250": 188,
-"-0.2250": 189,
-"0.2275": 190,
-"-0.2275": 191,
-"0.2300": 192,
-"-0.2300": 193,
-"0.2325": 194,
-"-0.2325": 195,
-"0.2350": 196,
-"-0.2350": 197,
-"0.2375": 198,
-"-0.2375": 199,
-"0.2400": 200,
-"-0.2400": 201,
-"0.2425": 202,
-"-0.2425": 203,
-"0.2450": 204,
-"-0.2450": 205,
-"0.2475": 206,
-"-0.2475": 207,
-"0.2500": 208,
-"-0.2500": 209,
-"0.2525": 210,
-"-0.2525": 211,
-"0.2550": 212,
-"-0.2550": 213,
-"0.2575": 214,
-"-0.2575": 215,
-"0.2600": 216,
-"-0.2600": 217,
-"0.2625": 218,
-"-0.2625": 219,
-"0.2650": 220,
-"-0.2650": 221,
-"0.2675": 222,
-"-0.2675": 223,
-"0.2700": 224,
-"-0.2700": 225,
-"0.2725": 226,
-"-0.2725": 227,
-"0.2750": 228,
-"-0.2750": 229,
-"0.2775": 230,
-"-0.2775": 231,
-"0.2800": 232,
-"-0.2800": 233,
-"0.2825": 234,
-"-0.2825": 235,
-"0.2850": 236,
-"-0.2850": 237,
-"0.2875": 238,
-"-0.2875": 239,
-"0.2900": 240,
-"-0.2900": 241,
-"0.2925": 242,
-"-0.2925": 243,
-"0.2950": 244,
-"-0.2950": 245,
-"0.2975": 246,
-"-0.2975": 247,
-"0.3000": 248
+      "0.0025": 8,
+      "-0.0025": 9,
+      "0.0050": 10,
+      "-0.0050": 11,
+      "0.0075": 12,
+      "-0.0075": 13,
+      "0.0100": 16,
+      "-0.0100": 17,
+      "0.0125": 18,
+      "-0.0125": 19,
+      "0.0150": 20,
+      "-0.0150": 21,
+      "0.0175": 22,
+      "-0.0175": 23,
+      "0.0200": 24,
+      "-0.0200": 25,
+      "0.0225": 26,
+      "-0.0225": 27,
+      "0.0250": 28,
+      "-0.0250": 29,
+      "0.0275": 30,
+      "-0.0275": 31,
+      "0.0300": 32,
+      "-0.0300": 33,
+      "0.0325": 34,
+      "-0.0325": 35,
+      "0.0350": 36,
+      "-0.0350": 37,
+      "0.0375": 38,
+      "-0.0375": 39,
+      "0.0400": 40,
+      "-0.0400": 41,
+      "0.0425": 42,
+      "-0.0425": 43,
+      "0.0450": 44,
+      "-0.0450": 45,
+      "0.0475": 46,
+      "-0.0475": 47,
+      "0.0500": 48,
+      "-0.0500": 49,
+      "0.0525": 50,
+      "-0.0525": 51,
+      "0.0550": 52,
+      "-0.0550": 53,
+      "0.0575": 54,
+      "-0.0575": 55,
+      "0.0600": 56,
+      "-0.0600": 57,
+      "0.0625": 58,
+      "-0.0625": 59,
+      "0.0650": 60,
+      "-0.0650": 61,
+      "0.0675": 62,
+      "-0.0675": 63,
+      "0.0700": 64,
+      "-0.0700": 65,
+      "0.0725": 66,
+      "-0.0725": 67,
+      "0.0750": 68,
+      "-0.0750": 69,
+      "0.0775": 70,
+      "-0.0775": 71,
+      "0.0800": 72,
+      "-0.0800": 73,
+      "0.0825": 74,
+      "-0.0825": 75,
+      "0.0850": 76,
+      "-0.0850": 77,
+      "0.0875": 78,
+      "-0.0875": 79,
+      "0.0900": 80,
+      "-0.0900": 81,
+      "0.0925": 82,
+      "-0.0925": 83,
+      "0.0950": 84,
+      "-0.0950": 85,
+      "0.0975": 86,
+      "-0.0975": 87,
+      "0.1000": 88,
+      "-0.1000": 89,
+      "0.1025": 90,
+      "-0.1025": 91,
+      "0.1050": 92,
+      "-0.1050": 93,
+      "0.1075": 94,
+      "-0.1075": 95,
+      "0.1100": 96,
+      "-0.1100": 97,
+      "0.1125": 98,
+      "-0.1125": 99,
+      "0.1150": 100,
+      "-0.1150": 101,
+      "0.1175": 102,
+      "-0.1175": 103,
+      "0.1200": 104,
+      "-0.1200": 105,
+      "0.1225": 106,
+      "-0.1225": 107,
+      "0.1250": 108,
+      "-0.1250": 109,
+      "0.1275": 110,
+      "-0.1275": 111,
+      "0.1300": 112,
+      "-0.1300": 113,
+      "0.1325": 114,
+      "-0.1325": 115,
+      "0.1350": 116,
+      "-0.1350": 117,
+      "0.1375": 118,
+      "-0.1375": 119,
+      "0.1400": 120,
+      "-0.1400": 121,
+      "0.1425": 122,
+      "-0.1425": 123,
+      "0.1450": 124,
+      "-0.1450": 125,
+      "0.1475": 126,
+      "-0.1475": 127,
+      "0.1500": 128,
+      "-0.1500": 129,
+      "0.1525": 130,
+      "-0.1525": 131,
+      "0.1550": 132,
+      "-0.1550": 133,
+      "0.1575": 134,
+      "-0.1575": 135,
+      "0.1600": 136,
+      "-0.1600": 137,
+      "0.1625": 138,
+      "-0.1625": 139,
+      "0.1650": 140,
+      "-0.1650": 141,
+      "0.1675": 142,
+      "-0.1675": 143,
+      "0.1700": 144,
+      "-0.1700": 145,
+      "0.1725": 146,
+      "-0.1725": 147,
+      "0.1750": 148,
+      "-0.1750": 149,
+      "0.1775": 150,
+      "-0.1775": 151,
+      "0.1800": 152,
+      "-0.1800": 153,
+      "0.1825": 154,
+      "-0.1825": 155,
+      "0.1850": 156,
+      "-0.1850": 157,
+      "0.1875": 158,
+      "-0.1875": 159,
+      "0.1900": 160,
+      "-0.1900": 161,
+      "0.1925": 162,
+      "-0.1925": 163,
+      "0.1950": 164,
+      "-0.1950": 165,
+      "0.1975": 166,
+      "-0.1975": 167,
+      "0.2000": 168,
+      "-0.2000": 169,
+      "0.2025": 170,
+      "-0.2025": 171,
+      "0.2050": 172,
+      "-0.2050": 173,
+      "0.2075": 174,
+      "-0.2075": 175,
+      "0.2100": 176,
+      "-0.2100": 177,
+      "0.2125": 178,
+      "-0.2125": 179,
+      "0.2150": 180,
+      "-0.2150": 181,
+      "0.2175": 182,
+      "-0.2175": 183,
+      "0.2200": 184,
+      "-0.2200": 185,
+      "0.2225": 186,
+      "-0.2225": 187,
+      "0.2250": 188,
+      "-0.2250": 189,
+      "0.2275": 190,
+      "-0.2275": 191,
+      "0.2300": 192,
+      "-0.2300": 193,
+      "0.2325": 194,
+      "-0.2325": 195,
+      "0.2350": 196,
+      "-0.2350": 197,
+      "0.2375": 198,
+      "-0.2375": 199,
+      "0.2400": 200,
+      "-0.2400": 201,
+      "0.2425": 202,
+      "-0.2425": 203,
+      "0.2450": 204,
+      "-0.2450": 205,
+      "0.2475": 206,
+      "-0.2475": 207,
+      "0.2500": 208,
+      "-0.2500": 209,
+      "0.2525": 210,
+      "-0.2525": 211,
+      "0.2550": 212,
+      "-0.2550": 213,
+      "0.2575": 214,
+      "-0.2575": 215,
+      "0.2600": 216,
+      "-0.2600": 217,
+      "0.2625": 218,
+      "-0.2625": 219,
+      "0.2650": 220,
+      "-0.2650": 221,
+      "0.2675": 222,
+      "-0.2675": 223,
+      "0.2700": 224,
+      "-0.2700": 225,
+      "0.2725": 226,
+      "-0.2725": 227,
+      "0.2750": 228,
+      "-0.2750": 229,
+      "0.2775": 230,
+      "-0.2775": 231,
+      "0.2800": 232,
+      "-0.2800": 233,
+      "0.2825": 234,
+      "-0.2825": 235,
+      "0.2850": 236,
+      "-0.2850": 237,
+      "0.2875": 238,
+      "-0.2875": 239,
+      "0.2900": 240,
+      "-0.2900": 241,
+      "0.2925": 242,
+      "-0.2925": 243,
+      "0.2950": 244,
+      "-0.2950": 245,
+      "0.2975": 246,
+      "-0.2975": 247,
+      "0.3000": 248
     },
     "merges": [