guitar_tab_gpt2 / tokenizer.json

Upload tokenizer

78d9cac almost 2 years ago

13.1 kB

	{
	"version": "1.0",
	"truncation": null,
	"padding": null,
	"added_tokens": [
	{
	"id": 0,
	"content": "<\|endoftext\|>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	},
	{
	"id": 1,
	"content": "<\|pad\|>",
	"single_word": false,
	"lstrip": false,
	"rstrip": false,
	"normalized": false,
	"special": true
	}
	],
	"normalizer": null,
	"pre_tokenizer": {
	"type": "ByteLevel",
	"add_prefix_space": false,
	"trim_offsets": true,
	"use_regex": true
	},
	"post_processor": {
	"type": "ByteLevel",
	"add_prefix_space": true,
	"trim_offsets": false,
	"use_regex": true
	},
	"decoder": {
	"type": "ByteLevel",
	"add_prefix_space": true,
	"trim_offsets": true,
	"use_regex": true
	},
	"model": {
	"type": "BPE",
	"dropout": null,
	"unk_token": null,
	"continuing_subword_prefix": "",
	"end_of_word_suffix": "",
	"fuse_unk": false,
	"byte_fallback": false,
	"vocab": {
	"<\|endoftext\|>": 0,
	"<\|pad\|>": 1,
	"!": 2,
	"\"": 3,
	"#": 4,
	"$": 5,
	"%": 6,
	"&": 7,
	"'": 8,
	"(": 9,
	")": 10,
	"*": 11,
	"+": 12,
	",": 13,
	"-": 14,
	".": 15,
	"/": 16,
	"0": 17,
	"1": 18,
	"2": 19,
	"3": 20,
	"4": 21,
	"5": 22,
	"6": 23,
	"7": 24,
	"8": 25,
	"9": 26,
	":": 27,
	";": 28,
	"<": 29,
	"=": 30,
	">": 31,
	"?": 32,
	"@": 33,
	"A": 34,
	"B": 35,
	"C": 36,
	"D": 37,
	"E": 38,
	"F": 39,
	"G": 40,
	"H": 41,
	"I": 42,
	"J": 43,
	"K": 44,
	"L": 45,
	"M": 46,
	"N": 47,
	"O": 48,
	"P": 49,
	"Q": 50,
	"R": 51,
	"S": 52,
	"T": 53,
	"U": 54,
	"V": 55,
	"W": 56,
	"X": 57,
	"Y": 58,
	"Z": 59,
	"[": 60,
	"\\": 61,
	"]": 62,
	"^": 63,
	"_": 64,
	"`": 65,
	"a": 66,
	"b": 67,
	"c": 68,
	"d": 69,
	"e": 70,
	"f": 71,
	"g": 72,
	"h": 73,
	"i": 74,
	"j": 75,
	"k": 76,
	"l": 77,
	"m": 78,
	"n": 79,
	"o": 80,
	"p": 81,
	"q": 82,
	"r": 83,
	"s": 84,
	"t": 85,
	"u": 86,
	"v": 87,
	"w": 88,
	"x": 89,
	"y": 90,
	"z": 91,
	"{": 92,
	"\|": 93,
	"}": 94,
	"~": 95,
	"¡": 96,
	"¢": 97,
	"£": 98,
	"¤": 99,
	"¥": 100,
	"¦": 101,
	"§": 102,
	"¨": 103,
	"©": 104,
	"ª": 105,
	"«": 106,
	"¬": 107,
	"®": 108,
	"¯": 109,
	"°": 110,
	"±": 111,
	"²": 112,
	"³": 113,
	"´": 114,
	"µ": 115,
	"¶": 116,
	"·": 117,
	"¸": 118,
	"¹": 119,
	"º": 120,
	"»": 121,
	"¼": 122,
	"½": 123,
	"¾": 124,
	"¿": 125,
	"À": 126,
	"Á": 127,
	"Â": 128,
	"Ã": 129,
	"Ä": 130,
	"Å": 131,
	"Æ": 132,
	"Ç": 133,
	"È": 134,
	"É": 135,
	"Ê": 136,
	"Ë": 137,
	"Ì": 138,
	"Í": 139,
	"Î": 140,
	"Ï": 141,
	"Ð": 142,
	"Ñ": 143,
	"Ò": 144,
	"Ó": 145,
	"Ô": 146,
	"Õ": 147,
	"Ö": 148,
	"×": 149,
	"Ø": 150,
	"Ù": 151,
	"Ú": 152,
	"Û": 153,
	"Ü": 154,
	"Ý": 155,
	"Þ": 156,
	"ß": 157,
	"à": 158,
	"á": 159,
	"â": 160,
	"ã": 161,
	"ä": 162,
	"å": 163,
	"æ": 164,
	"ç": 165,
	"è": 166,
	"é": 167,
	"ê": 168,
	"ë": 169,
	"ì": 170,
	"í": 171,
	"î": 172,
	"ï": 173,
	"ð": 174,
	"ñ": 175,
	"ò": 176,
	"ó": 177,
	"ô": 178,
	"õ": 179,
	"ö": 180,
	"÷": 181,
	"ø": 182,
	"ù": 183,
	"ú": 184,
	"û": 185,
	"ü": 186,
	"ý": 187,
	"þ": 188,
	"ÿ": 189,
	"Ā": 190,
	"ā": 191,
	"Ă": 192,
	"ă": 193,
	"Ą": 194,
	"ą": 195,
	"Ć": 196,
	"ć": 197,
	"Ĉ": 198,
	"ĉ": 199,
	"Ċ": 200,
	"ċ": 201,
	"Č": 202,
	"č": 203,
	"Ď": 204,
	"ď": 205,
	"Đ": 206,
	"đ": 207,
	"Ē": 208,
	"ē": 209,
	"Ĕ": 210,
	"ĕ": 211,
	"Ė": 212,
	"ė": 213,
	"Ę": 214,
	"ę": 215,
	"Ě": 216,
	"ě": 217,
	"Ĝ": 218,
	"ĝ": 219,
	"Ğ": 220,
	"ğ": 221,
	"Ġ": 222,
	"ġ": 223,
	"Ģ": 224,
	"ģ": 225,
	"Ĥ": 226,
	"ĥ": 227,
	"Ħ": 228,
	"ħ": 229,
	"Ĩ": 230,
	"ĩ": 231,
	"Ī": 232,
	"ī": 233,
	"Ĭ": 234,
	"ĭ": 235,
	"Į": 236,
	"į": 237,
	"İ": 238,
	"ı": 239,
	"Ĳ": 240,
	"ĳ": 241,
	"Ĵ": 242,
	"ĵ": 243,
	"Ķ": 244,
	"ķ": 245,
	"ĸ": 246,
	"Ĺ": 247,
	"ĺ": 248,
	"Ļ": 249,
	"ļ": 250,
	"Ľ": 251,
	"ľ": 252,
	"Ŀ": 253,
	"ŀ": 254,
	"Ł": 255,
	"ł": 256,
	"Ń": 257,
	"Ġ(": 258,
	"Ġ0": 259,
	"Ġ3": 260,
	"Ġ\|": 261,
	"Ġ1": 262,
	"Ġr": 263,
	"Ġ2": 264,
	"16": 265,
	"Ġ5": 266,
	"Ġ4": 267,
	"Ġ7": 268,
	").": 269,
	"tu": 270,
	"Ġ:": 271,
	"Ġ6": 272,
	"Ġ8": 273,
	"}.": 274,
	"Ġ9": 275,
	"Ġ12": 276,
	"pm": 277,
	"Ġ10": 278,
	"lr": 279,
	"})": 280,
	"32": 281,
	"Ġ36": 282,
	"Ġ14": 283,
	"}).": 284,
	"Ġ15": 285,
	"Ġ11": 286,
	"Ġ42": 287,
	"10": 288,
	"Ġ13": 289,
	"Ġ17": 290,
	"Ġ35": 291,
	"Ġtu": 292,
	"Ġ38": 293,
	"12": 294,
	"Ġ40": 295,
	"be": 296,
	"st": 297,
	"Ġ16": 298,
	"Ġ46": 299,
	"42": 300,
	"46": 301,
	"Ġ19": 302,
	"in": 303,
	"36": 304,
	"11": 305,
	"ĠE": 306,
	"64": 307,
	"14": 308,
	"Ġ24": 309,
	"Ġ18": 310,
	")}": 311,
	"Ġ20": 312,
	"le": 313,
	"it": 314,
	"ĠD": 315,
	"Ġ\"": 316,
	"re": 317,
	"tit": 318,
	"title": 319,
	"en": 320,
	"ing": 321,
	"ra": 322,
	"te": 323,
	"ck": 324,
	"ĠA": 325,
	"um": 326,
	"ent": 327,
	"mp": 328,
	"ts": 329,
	"ning": 330,
	"rack": 331,
	"str": 332,
	"track": 333,
	"fre": 334,
	"ument": 335,
	"mpo": 336,
	"strument": 337,
	"instrument": 338,
	"tuning": 339,
	"tempo": 340,
	"frets": 341,
	"ĠG": 342,
	"15": 343,
	"49": 344,
	"38": 345,
	")}.": 346,
	"13": 347,
	"ah": 348,
	"ĠB": 349,
	"Ġ22": 350,
	"35": 351,
	"Ġ21": 352,
	"Ġ44": 353,
	"44": 354,
	"17": 355,
	"51": 356,
	"Ġ49": 357,
	"Ġ51": 358,
	"40": 359,
	"Ġlr": 360,
	"57": 361,
	"Ġ41": 362,
	"Ġv": 363,
	"Ġbe": 364,
	"Ġ54": 365,
	"Ġ45": 366,
	"Ġ57": 367,
	"Ġ43": 368,
	"Ġg": 369,
	"nh": 370,
	"59": 371,
	"ĠC": 372,
	"Ġ55": 373,
	"Ġ59": 374,
	"19": 375,
	"Ġ37": 376,
	"Ġ47": 377,
	"Ġ53": 378,
	"Ġ30": 379,
	"53": 380,
	"41": 381,
	"Ġ85": 382,
	"Ġ69": 383,
	"55": 384,
	")})": 385,
	"45": 386,
	"18": 387,
	"Ġ48": 388,
	"Ġ23": 389,
	"Ġ82": 390,
	"20": 391,
	"Ġ70": 392,
	"he": 393,
	"Ġ25": 394,
	"43": 395,
	"Ġ29": 396,
	"sh": 397,
	"ĠEb": 398,
	"Ġpm": 399,
	"Ġ33": 400,
	"22": 401,
	"on": 402,
	"39": 403,
	"Ġ64": 404,
	"Ġ27": 405,
	"ĠF": 406,
	"ĠDb": 407,
	"ĠT": 408,
	"Ġ120": 409,
	")}).": 410,
	"54": 411,
	"ĠGb": 412,
	"an": 413,
	"ĠAb": 414,
	"Ġ50": 415,
	"er": 416,
	"Ġ39": 417,
	"Ġ28": 418,
	"Ġ31": 419,
	"33": 420,
	"ĠS": 421,
	"Ġ73": 422,
	"Ġ63": 423,
	"ar": 424,
	"69": 425,
	"Ġ26": 426,
	"ou": 427,
	"21": 428,
	"su": 429,
	"Ġ61": 430,
	"ĠBb": 431,
	"ph": 432,
	"Ġh": 433,
	"24": 434,
	"btitle": 435,
	"subtitle": 436,
	"or": 437,
	"ĠM": 438,
	"Ġf": 439,
	"Ġah": 440,
	"47": 441,
	"Ġ32": 442,
	"Ġst": 443,
	"48": 444,
	"Ġt": 445,
	"ĠÃ": 446,
	"ĠO": 447,
	"37": 448,
	"al": 449,
	"Ġnh": 450,
	"Ġ34": 451,
	"52": 452,
	"me": 453,
	"Ġ52": 454,
	"ĠL": 455,
	"at": 456,
	"Ã¥": 457,
	"ĠThe": 458,
	"Ã®": 459,
	"Ãł": 460,
	"ĠW": 461,
	"The": 462,
	"es": 463,
	"ĠI": 464,
	"ll": 465,
	"ve": 466,
	"Ġ80": 467,
	"Ġ62": 468,
	"23": 469,
	"82": 470,
	"ĠH": 471,
	"70": 472,
	"Ġ100": 473,
	"ĠR": 474,
	"Ã¥Ã": 475,
	"Ã®Ã": 476,
	"th": 477,
	")\"": 478,
	"ig": 479,
	"ad": 480,
	"Ġ\"\"": 481,
	"Ġ56": 482,
	"ic": 483,
	"Ġthe": 484,
	"ow": 485,
	"om": 486,
	"Ġ87": 487,
	"is": 488,
	"Ġo": 489,
	"Ġ74": 490,
	"ed": 491,
	"28": 492,
	"ay": 493,
	"ÃłÃ": 494,
	"ĠP": 495,
	"Ġof": 496,
	"27": 497,
	"ĠOf": 498,
	"un": 499
	},
	"merges": [
	"Ġ (",
	"Ġ 0",
	"Ġ 3",
	"Ġ \|",
	"Ġ 1",
	"Ġ r",
	"Ġ 2",
	"1 6",
	"Ġ 5",
	"Ġ 4",
	"Ġ 7",
	") .",
	"t u",
	"Ġ :",
	"Ġ 6",
	"Ġ 8",
	"} .",
	"Ġ 9",
	"Ġ1 2",
	"p m",
	"Ġ1 0",
	"l r",
	"} )",
	"3 2",
	"Ġ3 6",
	"Ġ1 4",
	"} ).",
	"Ġ1 5",
	"Ġ1 1",
	"Ġ4 2",
	"1 0",
	"Ġ1 3",
	"Ġ1 7",
	"Ġ3 5",
	"Ġ tu",
	"Ġ3 8",
	"1 2",
	"Ġ4 0",
	"b e",
	"s t",
	"Ġ1 6",
	"Ġ4 6",
	"4 2",
	"4 6",
	"Ġ1 9",
	"i n",
	"3 6",
	"1 1",
	"Ġ E",
	"6 4",
	"1 4",
	"Ġ2 4",
	"Ġ1 8",
	") }",
	"Ġ2 0",
	"l e",
	"i t",
	"Ġ D",
	"Ġ \"",
	"r e",
	"t it",
	"tit le",
	"e n",
	"in g",
	"r a",
	"t e",
	"c k",
	"Ġ A",
	"u m",
	"en t",
	"m p",
	"t s",
	"n ing",
	"ra ck",
	"st r",
	"t rack",
	"f re",
	"um ent",
	"mp o",
	"str ument",
	"in strument",
	"tu ning",
	"te mpo",
	"fre ts",
	"Ġ G",
	"1 5",
	"4 9",
	"3 8",
	") }.",
	"1 3",
	"a h",
	"Ġ B",
	"Ġ2 2",
	"3 5",
	"Ġ2 1",
	"Ġ4 4",
	"4 4",
	"1 7",
	"5 1",
	"Ġ4 9",
	"Ġ5 1",
	"4 0",
	"Ġ lr",
	"5 7",
	"Ġ4 1",
	"Ġ v",
	"Ġ be",
	"Ġ5 4",
	"Ġ4 5",
	"Ġ5 7",
	"Ġ4 3",
	"Ġ g",
	"n h",
	"5 9",
	"Ġ C",
	"Ġ5 5",
	"Ġ5 9",
	"1 9",
	"Ġ3 7",
	"Ġ4 7",
	"Ġ5 3",
	"Ġ3 0",
	"5 3",
	"4 1",
	"Ġ8 5",
	"Ġ6 9",
	"5 5",
	") })",
	"4 5",
	"1 8",
	"Ġ4 8",
	"Ġ2 3",
	"Ġ8 2",
	"2 0",
	"Ġ7 0",
	"h e",
	"Ġ2 5",
	"4 3",
	"Ġ2 9",
	"s h",
	"ĠE b",
	"Ġ pm",
	"Ġ3 3",
	"2 2",
	"o n",
	"3 9",
	"Ġ6 4",
	"Ġ2 7",
	"Ġ F",
	"ĠD b",
	"Ġ T",
	"Ġ12 0",
	") }).",
	"5 4",
	"ĠG b",
	"a n",
	"ĠA b",
	"Ġ5 0",
	"e r",
	"Ġ3 9",
	"Ġ2 8",
	"Ġ3 1",
	"3 3",
	"Ġ S",
	"Ġ7 3",
	"Ġ6 3",
	"a r",
	"6 9",
	"Ġ2 6",
	"o u",
	"2 1",
	"s u",
	"Ġ6 1",
	"ĠB b",
	"p h",
	"Ġ h",
	"2 4",
	"b title",
	"su btitle",
	"o r",
	"Ġ M",
	"Ġ f",
	"Ġ ah",
	"4 7",
	"Ġ3 2",
	"Ġ st",
	"4 8",
	"Ġ t",
	"Ġ Ã",
	"Ġ O",
	"3 7",
	"a l",
	"Ġ nh",
	"Ġ3 4",
	"5 2",
	"m e",
	"Ġ5 2",
	"Ġ L",
	"a t",
	"Ã ¥",
	"ĠT he",
	"Ã ®",
	"Ã ł",
	"Ġ W",
	"T he",
	"e s",
	"Ġ I",
	"l l",
	"v e",
	"Ġ8 0",
	"Ġ6 2",
	"2 3",
	"8 2",
	"Ġ H",
	"7 0",
	"Ġ10 0",
	"Ġ R",
	"Ã¥ Ã",
	"Ã® Ã",
	"t h",
	") \"",
	"i g",
	"a d",
	"Ġ\" \"",
	"Ġ5 6",
	"i c",
	"Ġt he",
	"o w",
	"o m",
	"Ġ8 7",
	"i s",
	"Ġ o",
	"Ġ7 4",
	"e d",
	"2 8",
	"a y",
	"Ãł Ã",
	"Ġ P",
	"Ġo f",
	"2 7",
	"ĠO f",
	"u n"
	]
	}
	}