{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 500, "special": true, "content": "<|startoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true }, { "id": 501, "special": true, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false } ], "normalizer": { "type": "Sequence", "normalizers": [ { "type": "NFC" }, { "type": "Replace", "pattern": { "Regex": "\\s+" }, "content": " " }, { "type": "Lowercase" } ] }, "pre_tokenizer": { "type": "Sequence", "pretokenizers": [ { "type": "Split", "pattern": { "Regex": "<\\|startoftext\\|>|<\\|endoftext\\|>|'s|'t|'re|'ve|'m|'ll|'d|[\\p{L}]+|[\\p{N}]|[^\\s\\p{L}\\p{N}]+" }, "behavior": "Removed", "invert": true }, { "type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true } ] }, "post_processor": { "type": "RobertaProcessing", "sep": [ "<|endoftext|>", 49407 ], "cls": [ "<|startoftext|>", 49406 ], "trim_offsets": false, "add_prefix_space": false }, "decoder": { "type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true }, "model": { "type": "BPE", "dropout": null, "unk_token": "<|endoftext|>", "continuing_subword_prefix": "", "end_of_word_suffix": "", "fuse_unk": false, "vocab": { "!": 0, "\"": 1, "#": 2, "$": 3, "%": 4, "&": 5, "'": 6, "(": 7, ")": 8, "*": 9, "+": 10, ",": 11, "-": 12, ".": 13, "/": 14, "0": 15, "1": 16, "2": 17, "3": 18, "4": 19, "5": 20, "6": 21, "7": 22, "8": 23, "9": 24, ":": 25, ";": 26, "<": 27, "=": 28, ">": 29, "?": 30, "@": 31, "A": 32, "B": 33, "C": 34, "D": 35, "E": 36, "F": 37, "G": 38, "H": 39, "I": 40, "J": 41, "K": 42, "L": 43, "M": 44, "N": 45, "O": 46, "P": 47, "Q": 48, "R": 49, "S": 50, "T": 51, "U": 52, "V": 53, "W": 54, "X": 55, "Y": 56, "Z": 57, "[": 58, "\\": 59, "]": 60, "^": 61, "_": 62, "`": 63, "a": 64, "b": 65, "c": 66, "d": 67, "e": 68, "f": 69, "g": 70, "h": 71, "i": 72, "j": 73, "k": 74, "l": 75, "m": 76, "n": 77, "o": 78, "p": 79, "q": 80, "r": 81, "s": 82, "t": 83, "u": 84, "v": 85, "w": 86, "x": 87, "y": 88, "z": 89, "{": 90, "|": 91, "}": 92, "~": 93, "¡": 94, "¢": 95, "£": 96, "¤": 97, "¥": 98, "¦": 99, "§": 100, "¨": 101, "©": 102, "ª": 103, "«": 104, "¬": 105, "®": 106, "¯": 107, "°": 108, "±": 109, "²": 110, "³": 111, "´": 112, "µ": 113, "¶": 114, "·": 115, "¸": 116, "¹": 117, "º": 118, "»": 119, "¼": 120, "½": 121, "¾": 122, "¿": 123, "À": 124, "Á": 125, "Â": 126, "Ã": 127, "Ä": 128, "Å": 129, "Æ": 130, "Ç": 131, "È": 132, "É": 133, "Ê": 134, "Ë": 135, "Ì": 136, "Í": 137, "Î": 138, "Ï": 139, "Ð": 140, "Ñ": 141, "Ò": 142, "Ó": 143, "Ô": 144, "Õ": 145, "Ö": 146, "×": 147, "Ø": 148, "Ù": 149, "Ú": 150, "Û": 151, "Ü": 152, "Ý": 153, "Þ": 154, "ß": 155, "à": 156, "á": 157, "â": 158, "ã": 159, "ä": 160, "å": 161, "æ": 162, "ç": 163, "è": 164, "é": 165, "ê": 166, "ë": 167, "ì": 168, "í": 169, "î": 170, "ï": 171, "ð": 172, "ñ": 173, "ò": 174, "ó": 175, "ô": 176, "õ": 177, "ö": 178, "÷": 179, "ø": 180, "ù": 181, "ú": 182, "û": 183, "ü": 184, "ý": 185, "þ": 186, "ÿ": 187, "Ā": 188, "ā": 189, "Ă": 190, "ă": 191, "Ą": 192, "ą": 193, "Ć": 194, "ć": 195, "Ĉ": 196, "ĉ": 197, "Ċ": 198, "ċ": 199, "Č": 200, "č": 201, "Ď": 202, "ď": 203, "Đ": 204, "đ": 205, "Ē": 206, "ē": 207, "Ĕ": 208, "ĕ": 209, "Ė": 210, "ė": 211, "Ę": 212, "ę": 213, "Ě": 214, "ě": 215, "Ĝ": 216, "ĝ": 217, "Ğ": 218, "ğ": 219, "Ġ": 220, "ġ": 221, "Ģ": 222, "ģ": 223, "Ĥ": 224, "ĥ": 225, "Ħ": 226, "ħ": 227, "Ĩ": 228, "ĩ": 229, "Ī": 230, "ī": 231, "Ĭ": 232, "ĭ": 233, "Į": 234, "į": 235, "İ": 236, "ı": 237, "IJ": 238, "ij": 239, "Ĵ": 240, "ĵ": 241, "Ķ": 242, "ķ": 243, "ĸ": 244, "Ĺ": 245, "ĺ": 246, "Ļ": 247, "ļ": 248, "Ľ": 249, "ľ": 250, "Ŀ": 251, "ŀ": 252, "Ł": 253, "ł": 254, "Ń": 255, "!": 256, "\"": 257, "#": 258, "$": 259, "%": 260, "&": 261, "'": 262, "(": 263, ")": 264, "*": 265, "+": 266, ",": 267, "-": 268, ".": 269, "/": 270, "0": 271, "1": 272, "2": 273, "3": 274, "4": 275, "5": 276, "6": 277, "7": 278, "8": 279, "9": 280, ":": 281, ";": 282, "<": 283, "=": 284, ">": 285, "?": 286, "@": 287, "A": 288, "B": 289, "C": 290, "D": 291, "E": 292, "F": 293, "G": 294, "H": 295, "I": 296, "J": 297, "K": 298, "L": 299, "M": 300, "N": 301, "O": 302, "P": 303, "Q": 304, "R": 305, "S": 306, "T": 307, "U": 308, "V": 309, "W": 310, "X": 311, "Y": 312, "Z": 313, "[": 314, "\\": 315, "]": 316, "^": 317, "_": 318, "`": 319, "a": 320, "b": 321, "c": 322, "d": 323, "e": 324, "f": 325, "g": 326, "h": 327, "i": 328, "j": 329, "k": 330, "l": 331, "m": 332, "n": 333, "o": 334, "p": 335, "q": 336, "r": 337, "s": 338, "t": 339, "u": 340, "v": 341, "w": 342, "x": 343, "y": 344, "z": 345, "{": 346, "|": 347, "}": 348, "~": 349, "¡": 350, "¢": 351, "£": 352, "¤": 353, "¥": 354, "¦": 355, "§": 356, "¨": 357, "©": 358, "ª": 359, "«": 360, "¬": 361, "®": 362, "¯": 363, "°": 364, "±": 365, "²": 366, "³": 367, "´": 368, "µ": 369, "¶": 370, "·": 371, "¸": 372, "¹": 373, "º": 374, "»": 375, "¼": 376, "½": 377, "¾": 378, "¿": 379, "À": 380, "Á": 381, "Â": 382, "Ã": 383, "Ä": 384, "Å": 385, "Æ": 386, "Ç": 387, "È": 388, "É": 389, "Ê": 390, "Ë": 391, "Ì": 392, "Í": 393, "Î": 394, "Ï": 395, "Ð": 396, "Ñ": 397, "Ò": 398, "Ó": 399, "Ô": 400, "Õ": 401, "Ö": 402, "×": 403, "Ø": 404, "Ù": 405, "Ú": 406, "Û": 407, "Ü": 408, "Ý": 409, "Þ": 410, "ß": 411, "à": 412, "á": 413, "â": 414, "ã": 415, "ä": 416, "å": 417, "æ": 418, "ç": 419, "è": 420, "é": 421, "ê": 422, "ë": 423, "ì": 424, "í": 425, "î": 426, "ï": 427, "ð": 428, "ñ": 429, "ò": 430, "ó": 431, "ô": 432, "õ": 433, "ö": 434, "÷": 435, "ø": 436, "ù": 437, "ú": 438, "û": 439, "ü": 440, "ý": 441, "þ": 442, "ÿ": 443, "Ā": 444, "ā": 445, "Ă": 446, "ă": 447, "Ą": 448, "ą": 449, "Ć": 450, "ć": 451, "Ĉ": 452, "ĉ": 453, "Ċ": 454, "ċ": 455, "Č": 456, "č": 457, "Ď": 458, "ď": 459, "Đ": 460, "đ": 461, "Ē": 462, "ē": 463, "Ĕ": 464, "ĕ": 465, "Ė": 466, "ė": 467, "Ę": 468, "ę": 469, "Ě": 470, "ě": 471, "Ĝ": 472, "ĝ": 473, "Ğ": 474, "ğ": 475, "Ġ": 476, "ġ": 477, "Ģ": 478, "ģ": 479, "Ĥ": 480, "ĥ": 481, "Ħ": 482, "ħ": 483, "Ĩ": 484, "ĩ": 485, "Ī": 486, "ī": 487, "Ĭ": 488, "ĭ": 489, "Į": 490, "į": 491, "İ": 492, "ı": 493, "IJ": 494, "ij": 495, "Ĵ": 496, "ĵ": 497, "Ķ": 498, "ķ": 499, "<|startoftext|>": 500, "<|endoftext|>": 501 }, "merges": [] } }