diff --git "a/tokenizer.json" "b/tokenizer.json"
deleted file mode 100644--- "a/tokenizer.json"
+++ /dev/null
@@ -1,118922 +0,0 @@
-{
- "version": "1.0",
- "truncation": null,
- "padding": null,
- "added_tokens": [
- {
- "id": 0,
- "content": "",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": true,
- "special": true
- },
- {
- "id": 1,
- "content": "",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": true,
- "special": true
- },
- {
- "id": 2,
- "content": "",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": true,
- "special": true
- },
- {
- "id": 64000,
- "content": "",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": true,
- "special": true
- }
- ],
- "normalizer": {
- "type": "Sequence",
- "normalizers": [
- {
- "type": "Prepend",
- "prepend": "▁"
- },
- {
- "type": "Replace",
- "pattern": {
- "String": " "
- },
- "content": "▁"
- }
- ]
- },
- "pre_tokenizer": null,
- "post_processor": {
- "type": "TemplateProcessing",
- "single": [
- {
- "Sequence": {
- "id": "A",
- "type_id": 0
- }
- }
- ],
- "pair": [
- {
- "Sequence": {
- "id": "A",
- "type_id": 0
- }
- },
- {
- "Sequence": {
- "id": "B",
- "type_id": 1
- }
- }
- ],
- "special_tokens": {}
- },
- "decoder": {
- "type": "Sequence",
- "decoders": [
- {
- "type": "Replace",
- "pattern": {
- "String": "▁"
- },
- "content": " "
- },
- {
- "type": "ByteFallback"
- },
- {
- "type": "Fuse"
- },
- {
- "type": "Strip",
- "content": " ",
- "start": 1,
- "stop": 0
- }
- ]
- },
- "model": {
- "type": "BPE",
- "dropout": null,
- "unk_token": "",
- "continuing_subword_prefix": null,
- "end_of_word_suffix": null,
- "fuse_unk": true,
- "byte_fallback": true,
- "vocab": {
- "": 0,
- "": 1,
- "": 2,
- "": 3,
- "": 4,
- "\n": 5,
- "\t": 6,
- "": 7,
- "": 8,
- "
": 9,
- "
": 10,
- "
": 11,
- "": 12,
- "": 13,
- "": 14,
- "": 15,
- "": 16,
- "": 17,
- "
": 18,
- "": 19,
- "
": 20,
- "
": 21,
- "": 22,
- "": 23,
- "
": 24,
- "": 25,
- "": 26,
- "
": 27,
- "": 30,
- "": 31,
- "": 32,
- "": 33,
- "": 34,
- "
": 35,
- "": 36,
- "": 37,
- "": 38,
- "
": 39,
- "": 40,
- "": 41,
- "": 42,
- "": 43,
- "": 44,
- "": 45,
- "": 46,
- "": 47,
- "": 48,
- "": 49,
- " | | ": 50,
- "": 51,
- "0": 52,
- "1": 53,
- "2": 54,
- "3": 55,
- "4": 56,
- "5": 57,
- "6": 58,
- "7": 59,
- "8": 60,
- "9": 61,
- "0": 62,
- "1": 63,
- "2": 64,
- "3": 65,
- "4": 66,
- "5": 67,
- "6": 68,
- "7": 69,
- "8": 70,
- "9": 71,
- ",": 72,
- "。": 73,
- "!": 74,
- "?": 75,
- "、": 76,
- ":": 77,
- "¥": 78,
- ".": 79,
- "!": 80,
- "?": 81,
- "《": 82,
- "》": 83,
- "【": 84,
- "】": 85,
- "『": 86,
- "』": 87,
- "```": 88,
- "": 90,
- "---": 91,
- "": 92,
- "": 93,
- "": 94,
- "": 95,
- "": 96,
- "": 97,
- "": 98,
- "": 99,
- "": 100,
- "": 101,
- "": 102,
- "": 103,
- "": 104,
- "": 105,
- "": 106,
- "": 107,
- "": 108,
- "": 109,
- "": 110,
- "": 111,
- "": 112,
- "": 113,
- "": 114,
- "": 115,
- "": 116,
- "": 117,
- "": 118,
- "": 119,
- "": 120,
- "": 121,
- "": 122,
- "": 123,
- "": 124,
- "": 125,
- "": 126,
- "": 127,
- "": 128,
- "": 129,
- "": 130,
- "": 131,
- "": 132,
- "": 133,
- "": 134,
- "": 135,
- "": 136,
- "": 137,
- "": 138,
- "": 139,
- "": 140,
- "": 141,
- "": 142,
- "": 143,
- "": 144,
- "": 145,
- "": 146,
- "": 147,
- "": 148,
- "": 149,
- "": 150,
- "": 151,
- "": 152,
- "": 153,
- "": 154,
- "": 155,
- "": 156,
- "": 157,
- "": 158,
- "": 159,
- "": 160,
- "": 161,
- "": 162,
- "": 163,
- "": 164,
- "": 165,
- "": 166,
- "": 167,
- "": 168,
- "": 169,
- "": 170,
- "": 171,
- "": 172,
- "": 173,
- "": 174,
- "": 175,
- "": 176,
- "": 177,
- "": 178,
- "": 179,
- "": 180,
- "": 181,
- "": 182,
- "": 183,
- "": 184,
- "": 185,
- "": 186,
- "": 187,
- "": 188,
- "": 189,
- "": 190,
- "": 191,
- "": 192,
- "": 193,
- "": 194,
- "": 195,
- "": 196,
- "": 197,
- "": 198,
- "": 199,
- "": 200,
- "": 201,
- "": 202,
- "": 203,
- "": 204,
- "": 205,
- "": 206,
- "": 207,
- "": 208,
- "": 209,
- "": 210,
- "": 211,
- "": 212,
- "": 213,
- "": 214,
- "": 215,
- "": 216,
- "": 217,
- "": 218,
- "": 219,
- "": 220,
- "": 221,
- "": 222,
- "": 223,
- "": 224,
- "": 225,
- "": 226,
- "": 227,
- "": 228,
- "": 229,
- "": 230,
- "": 231,
- "": 232,
- "": 233,
- "": 234,
- "": 235,
- "": 236,
- "": 237,
- "": 238,
- "": 239,
- "": 240,
- "": 241,
- "": 242,
- "": 243,
- "": 244,
- "": 245,
- "": 246,
- "": 247,
- "": 248,
- "": 249,
- "": 250,
- "": 251,
- "": 252,
- "": 253,
- "": 254,
- "": 255,
- "": 256,
- "": 257,
- "": 258,
- "": 259,
- "": 260,
- "": 261,
- "": 262,
- "": 263,
- "": 264,
- "": 265,
- "": 266,
- "": 267,
- "": 268,
- "": 269,
- "": 270,
- "": 271,
- "": 272,
- "": 273,
- "": 274,
- "": 275,
- "": 276,
- "": 277,
- "": 278,
- "": 279,
- "": 280,
- "": 281,
- "": 282,
- "": 283,
- "": 284,
- "": 285,
- "": 286,
- "": 287,
- "": 288,
- "": 289,
- "": 290,
- "": 291,
- "": 292,
- "": 293,
- "": 294,
- "": 295,
- "": 296,
- "": 297,
- "": 298,
- "": 299,
- "": 300,
- "": 301,
- "": 302,
- "": 303,
- "": 304,
- "": 305,
- "": 306,
- "": 307,
- "": 308,
- "": 309,
- "": 310,
- "": 311,
- "": 312,
- "": 313,
- "": 314,
- "": 315,
- "": 316,
- "": 317,
- "": 318,
- "": 319,
- "": 320,
- "": 321,
- "": 322,
- "": 323,
- "": 324,
- "": 325,
- "": 326,
- "": 327,
- "": 328,
- "": 329,
- "": 330,
- "": 331,
- "": 332,
- "": 333,
- "": 334,
- "": 335,
- "": 336,
- "": 337,
- "": 338,
- "": 339,
- "": 340,
- "": 341,
- "": 342,
- "": 343,
- "": 344,
- "": 345,
- "