Hugging Face's logo

floriangardin
/
musiclang

Text Generation
Transformers
PyTorch
gpt2
Inference Endpoints
text-generation-inference
Model card Files Community
5
musiclang / tokenizer.json
floriangardin's picture
floriangardin
Upload tokenizer
8593e68
raw history blame contribute delete
No virus
5.14 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 260,
"content": "<|endoftext|>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": true,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {
"type": "ByteLevel",
"add_prefix_space": false,
"trim_offsets": true,
"use_regex": true
},
"post_processor": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": false,
"use_regex": true
},
"decoder": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": true,
"use_regex": true
},
"model": {
"type": "BPE",
"dropout": null,
"unk_token": null,
"continuing_subword_prefix": "",
"end_of_word_suffix": "",
"fuse_unk": false,
"byte_fallback": false,
"vocab": {
"<s>": 0,
"</s>": 1,
"<unk>": 2,
"<mask>": 3,
"!": 4,
"\"": 5,
"#": 6,
"$": 7,
"%": 8,
"&": 9,
"'": 10,
"(": 11,
")": 12,
"*": 13,
"+": 14,
",": 15,
"-": 16,
".": 17,
"/": 18,
"0": 19,
"1": 20,
"2": 21,
"3": 22,
"4": 23,
"5": 24,
"6": 25,
"7": 26,
"8": 27,
"9": 28,
":": 29,
";": 30,
"<": 31,
"=": 32,
">": 33,
"?": 34,
"@": 35,
"A": 36,
"B": 37,
"C": 38,
"D": 39,
"E": 40,
"F": 41,
"G": 42,
"H": 43,
"I": 44,
"J": 45,
"K": 46,
"L": 47,
"M": 48,
"N": 49,
"O": 50,
"P": 51,
"Q": 52,
"R": 53,
"S": 54,
"T": 55,
"U": 56,
"V": 57,
"W": 58,
"X": 59,
"Y": 60,
"Z": 61,
"[": 62,
"\\": 63,
"]": 64,
"^": 65,
"_": 66,
"`": 67,
"a": 68,
"b": 69,
"c": 70,
"d": 71,
"e": 72,
"f": 73,
"g": 74,
"h": 75,
"i": 76,
"j": 77,
"k": 78,
"l": 79,
"m": 80,
"n": 81,
"o": 82,
"p": 83,
"q": 84,
"r": 85,
"s": 86,
"t": 87,
"u": 88,
"v": 89,
"w": 90,
"x": 91,
"y": 92,
"z": 93,
"{": 94,
"|": 95,
"}": 96,
"~": 97,
"¡": 98,
"¢": 99,
"£": 100,
"¤": 101,
"¥": 102,
"¦": 103,
"§": 104,
"¨": 105,
"©": 106,
"ª": 107,
"«": 108,
"¬": 109,
"®": 110,
"¯": 111,
"°": 112,
"±": 113,
"²": 114,
"³": 115,
"´": 116,
"µ": 117,
"¶": 118,
"·": 119,
"¸": 120,
"¹": 121,
"º": 122,
"»": 123,
"¼": 124,
"½": 125,
"¾": 126,
"¿": 127,
"À": 128,
"Á": 129,
"Â": 130,
"Ã": 131,
"Ä": 132,
"Å": 133,
"Æ": 134,
"Ç": 135,
"È": 136,
"É": 137,
"Ê": 138,
"Ë": 139,
"Ì": 140,
"Í": 141,
"Î": 142,
"Ï": 143,
"Ð": 144,
"Ñ": 145,
"Ò": 146,
"Ó": 147,
"Ô": 148,
"Õ": 149,
"Ö": 150,
"×": 151,
"Ø": 152,
"Ù": 153,
"Ú": 154,
"Û": 155,
"Ü": 156,
"Ý": 157,
"Þ": 158,
"ß": 159,
"à": 160,
"á": 161,
"â": 162,
"ã": 163,
"ä": 164,
"å": 165,
"æ": 166,
"ç": 167,
"è": 168,
"é": 169,
"ê": 170,
"ë": 171,
"ì": 172,
"í": 173,
"î": 174,
"ï": 175,
"ð": 176,
"ñ": 177,
"ò": 178,
"ó": 179,
"ô": 180,
"õ": 181,
"ö": 182,
"÷": 183,
"ø": 184,
"ù": 185,
"ú": 186,
"û": 187,
"ü": 188,
"ý": 189,
"þ": 190,
"ÿ": 191,
"Ā": 192,
"ā": 193,
"Ă": 194,
"ă": 195,
"Ą": 196,
"ą": 197,
"Ć": 198,
"ć": 199,
"Ĉ": 200,
"ĉ": 201,
"Ċ": 202,
"ċ": 203,
"Č": 204,
"č": 205,
"Ď": 206,
"ď": 207,
"Đ": 208,
"đ": 209,
"Ē": 210,
"ē": 211,
"Ĕ": 212,
"ĕ": 213,
"Ė": 214,
"ė": 215,
"Ę": 216,
"ę": 217,
"Ě": 218,
"ě": 219,
"Ĝ": 220,
"ĝ": 221,
"Ğ": 222,
"ğ": 223,
"Ġ": 224,
"ġ": 225,
"Ģ": 226,
"ģ": 227,
"Ĥ": 228,
"ĥ": 229,
"Ħ": 230,
"ħ": 231,
"Ĩ": 232,
"ĩ": 233,
"Ī": 234,
"ī": 235,
"Ĭ": 236,
"ĭ": 237,
"Į": 238,
"į": 239,
"İ": 240,
"ı": 241,
"IJ": 242,
"ij": 243,
"Ĵ": 244,
"ĵ": 245,
"Ķ": 246,
"ķ": 247,
"ĸ": 248,
"Ĺ": 249,
"ĺ": 250,
"Ļ": 251,
"ļ": 252,
"Ľ": 253,
"ľ": 254,
"Ŀ": 255,
"ŀ": 256,
"Ł": 257,
"ł": 258,
"Ń": 259
},
"merges": []
}
}